From 43b2d4888a006c1d9c9f84337f1a7c5d8f1cbfbd Mon Sep 17 00:00:00 2001 From: Nicholas openSUSE Software Engineer Date: Thu, 6 Jun 2024 17:08:20 -0300 Subject: [PATCH] [release-v2.8] Forward ported charts release from 2.7.14 (#4030) Co-authored-by: rancherbot --- .../fleet-agent-102.2.5+up0.8.5.tgz | Bin 0 -> 3062 bytes .../fleet-crd/fleet-crd-102.2.5+up0.8.5.tgz | Bin 0 -> 23716 bytes assets/fleet/fleet-102.2.5+up0.8.5.tgz | Bin 0 -> 5228 bytes .../longhorn-crd-102.4.1+up1.6.2.tgz | Bin 0 -> 12737 bytes assets/longhorn/longhorn-102.4.1+up1.6.2.tgz | Bin 0 -> 30586 bytes .../prometheus-federator-3.0.2+up0.3.3.tgz | Bin 0 -> 20601 bytes .../rancher-alerting-drivers-102.1.2.tgz | Bin 0 -> 8879 bytes ...ancher-monitoring-crd-102.0.5+up40.1.2.tgz | Bin 0 -> 195293 bytes .../rancher-monitoring-102.0.5+up40.1.2.tgz | Bin 0 -> 420128 bytes charts/fleet-agent/102.2.5+up0.8.5/Chart.yaml | 15 + charts/fleet-agent/102.2.5+up0.8.5/README.md | 8 + .../102.2.5+up0.8.5/templates/_helpers.tpl | 22 + .../102.2.5+up0.8.5/templates/configmap.yaml | 12 + .../102.2.5+up0.8.5/templates/deployment.yaml | 51 + .../templates/network_policy_allow_all.yaml | 15 + .../patch_default_serviceaccount.yaml | 28 + .../102.2.5+up0.8.5/templates/rbac.yaml | 25 + .../102.2.5+up0.8.5/templates/secret.yaml | 10 + .../templates/serviceaccount.yaml | 4 + .../102.2.5+up0.8.5/templates/validate.yaml | 11 + .../fleet-agent/102.2.5+up0.8.5/values.yaml | 63 + charts/fleet-crd/102.2.5+up0.8.5/Chart.yaml | 13 + charts/fleet-crd/102.2.5+up0.8.5/README.md | 5 + .../102.2.5+up0.8.5/templates/crds.yaml | 3453 ++++++++ .../templates/gitjobs-crds.yaml | 7714 +++++++++++++++++ charts/fleet-crd/102.2.5+up0.8.5/values.yaml | 1 + charts/fleet/102.2.5+up0.8.5/Chart.yaml | 22 + charts/fleet/102.2.5+up0.8.5/README.md | 30 + .../102.2.5+up0.8.5/charts/gitjob/.helmignore | 23 + .../102.2.5+up0.8.5/charts/gitjob/Chart.yaml | 5 + .../charts/gitjob/templates/_helpers.tpl | 7 + .../charts/gitjob/templates/clusterrole.yaml | 38 + .../gitjob/templates/clusterrolebinding.yaml | 12 + .../charts/gitjob/templates/deployment.yaml | 51 + .../charts/gitjob/templates/leases.yaml | 23 + .../charts/gitjob/templates/service.yaml | 12 + .../gitjob/templates/serviceaccount.yaml | 4 + .../102.2.5+up0.8.5/charts/gitjob/values.yaml | 31 + .../102.2.5+up0.8.5/templates/_helpers.tpl | 22 + .../102.2.5+up0.8.5/templates/configmap.yaml | 25 + .../102.2.5+up0.8.5/templates/deployment.yaml | 102 + .../job_cleanup_clusterregistrations.yaml | 29 + .../fleet/102.2.5+up0.8.5/templates/rbac.yaml | 114 + .../templates/serviceaccount.yaml | 12 + charts/fleet/102.2.5+up0.8.5/values.yaml | 83 + .../longhorn-crd/102.4.1+up1.6.2/Chart.yaml | 11 + charts/longhorn-crd/102.4.1+up1.6.2/README.md | 2 + .../102.4.1+up1.6.2/templates/_helpers.tpl | 66 + .../102.4.1+up1.6.2/templates/crds.yaml | 3942 +++++++++ charts/longhorn/102.4.1+up1.6.2/.helmignore | 21 + charts/longhorn/102.4.1+up1.6.2/Chart.yaml | 40 + charts/longhorn/102.4.1+up1.6.2/README.md | 50 + charts/longhorn/102.4.1+up1.6.2/app-readme.md | 27 + .../longhorn/102.4.1+up1.6.2/questions.yaml | 920 ++ .../102.4.1+up1.6.2/templates/NOTES.txt | 5 + .../102.4.1+up1.6.2/templates/_helpers.tpl | 66 + .../templates/clusterrole.yaml | 77 + .../templates/clusterrolebinding.yaml | 49 + .../templates/daemonset-sa.yaml | 167 + .../templates/default-setting.yaml | 229 + .../templates/deployment-driver.yaml | 132 + .../templates/deployment-ui.yaml | 182 + .../102.4.1+up1.6.2/templates/ingress.yaml | 37 + ...king-image-data-source-network-policy.yaml | 27 + .../backing-image-manager-network-policy.yaml | 27 + .../instance-manager-networking.yaml | 27 + .../manager-network-policy.yaml | 35 + .../recovery-backend-network-policy.yaml | 17 + .../ui-frontend-network-policy.yaml | 46 + .../webhook-network-policy.yaml | 33 + .../templates/postupgrade-job.yaml | 56 + .../templates/preupgrade-job.yaml | 55 + .../templates/priorityclass.yaml | 9 + .../102.4.1+up1.6.2/templates/psp.yaml | 66 + .../templates/registry-secret.yaml | 13 + .../templates/serviceaccount.yaml | 40 + .../templates/servicemonitor.yaml | 19 + .../102.4.1+up1.6.2/templates/services.yaml | 71 + .../templates/storageclass.yaml | 50 + .../templates/tls-secrets.yaml | 16 + .../templates/uninstall-job.yaml | 57 + .../102.4.1+up1.6.2/templates/userroles.yaml | 53 + .../templates/validate-install-crd.yaml | 35 + .../templates/validate-psp-install.yaml | 7 + charts/longhorn/102.4.1+up1.6.2/values.yaml | 484 ++ .../3.0.2+up0.3.3/Chart.yaml | 21 + .../3.0.2+up0.3.3/README.md | 120 + .../3.0.2+up0.3.3/app-README.md | 27 + .../charts/helmProjectOperator/Chart.yaml | 15 + .../charts/helmProjectOperator/README.md | 77 + .../charts/helmProjectOperator/app-readme.md | 20 + .../charts/helmProjectOperator/questions.yaml | 43 + .../helmProjectOperator/templates/NOTES.txt | 2 + .../templates/_helpers.tpl | 66 + .../templates/cleanup.yaml | 82 + .../templates/clusterrole.yaml | 57 + .../templates/configmap.yaml | 14 + .../templates/deployment.yaml | 126 + .../helmProjectOperator/templates/psp.yaml | 68 + .../helmProjectOperator/templates/rbac.yaml | 32 + .../system-namespaces-configmap.yaml | 62 + .../templates/validate-psp-install.yaml | 7 + .../charts/helmProjectOperator/values.yaml | 228 + .../3.0.2+up0.3.3/questions.yaml | 43 + .../3.0.2+up0.3.3/templates/NOTES.txt | 3 + .../3.0.2+up0.3.3/templates/_helpers.tpl | 66 + .../3.0.2+up0.3.3/values.yaml | 94 + .../102.1.2/Chart.yaml | 29 + .../102.1.2/README.md | 11 + .../102.1.2/app-readme.md | 29 + .../102.1.2/charts/prom2teams/.helmignore | 22 + .../102.1.2/charts/prom2teams/Chart.yaml | 10 + .../102.1.2/charts/prom2teams/files/teams.j2 | 44 + .../charts/prom2teams/templates/NOTES.txt | 2 + .../charts/prom2teams/templates/_helpers.tpl | 73 + .../prom2teams/templates/configmap.yaml | 39 + .../prom2teams/templates/deployment.yaml | 83 + .../charts/prom2teams/templates/psp.yaml | 61 + .../prom2teams/templates/service-account.yaml | 6 + .../charts/prom2teams/templates/service.yaml | 17 + .../102.1.2/charts/prom2teams/values.yaml | 69 + .../102.1.2/charts/sachet/.helmignore | 23 + .../102.1.2/charts/sachet/Chart.yaml | 11 + .../102.1.2/charts/sachet/files/template.tmpl | 1 + .../102.1.2/charts/sachet/templates/NOTES.txt | 3 + .../charts/sachet/templates/_helpers.tpl | 79 + .../templates/configmap-pre-install.yaml | 34 + .../charts/sachet/templates/deployment.yaml | 75 + .../102.1.2/charts/sachet/templates/psp.yaml | 61 + .../sachet/templates/service-account.yaml | 6 + .../charts/sachet/templates/service.yaml | 17 + .../102.1.2/charts/sachet/values.yaml | 69 + .../102.1.2/questions.yml | 17 + .../102.1.2/templates/NOTES.txt | 2 + .../102.1.2/templates/_helpers.tpl | 117 + .../102.1.2/templates/cluster-role.yaml | 50 + .../102.1.2/templates/hardened.yaml | 126 + .../templates/validate-psp-install.yaml | 7 + .../102.1.2/values.yaml | 29 + .../102.0.5+up40.1.2/Chart.yaml | 10 + .../102.0.5+up40.1.2/README.md | 24 + .../102.0.5+up40.1.2/files/crd-manifest.tgz | Bin 0 -> 191662 bytes .../102.0.5+up40.1.2/templates/_helpers.tpl | 50 + .../102.0.5+up40.1.2/templates/jobs.yaml | 152 + .../102.0.5+up40.1.2/templates/manifest.yaml | 8 + .../102.0.5+up40.1.2/templates/rbac.yaml | 76 + .../templates/validate-psp-install.yaml | 7 + .../102.0.5+up40.1.2/values.yaml | 17 + .../102.0.5+up40.1.2/.helmignore | 28 + .../102.0.5+up40.1.2/CHANGELOG.md | 47 + .../102.0.5+up40.1.2/CONTRIBUTING.md | 12 + .../102.0.5+up40.1.2/Chart.yaml | 128 + .../102.0.5+up40.1.2/README.md | 739 ++ .../102.0.5+up40.1.2/app-README.md | 46 + .../charts/grafana/.helmignore | 23 + .../charts/grafana/Chart.yaml | 29 + .../102.0.5+up40.1.2/charts/grafana/README.md | 574 ++ .../grafana/dashboards/custom-dashboard.json | 1 + .../charts/grafana/templates/NOTES.txt | 54 + .../charts/grafana/templates/_helpers.tpl | 214 + .../charts/grafana/templates/_pod.tpl | 895 ++ .../charts/grafana/templates/clusterrole.yaml | 25 + .../grafana/templates/clusterrolebinding.yaml | 24 + .../configmap-dashboard-provider.yaml | 29 + .../charts/grafana/templates/configmap.yaml | 117 + .../templates/dashboards-json-configmap.yaml | 35 + .../charts/grafana/templates/deployment.yaml | 50 + .../grafana/templates/extra-manifests.yaml | 4 + .../grafana/templates/headless-service.yaml | 22 + .../charts/grafana/templates/hpa.yaml | 21 + .../templates/image-renderer-deployment.yaml | 123 + .../image-renderer-network-policy.yaml | 73 + .../templates/image-renderer-service.yaml | 33 + .../charts/grafana/templates/ingress.yaml | 78 + .../grafana/templates/networkpolicy.yaml | 52 + .../grafana/templates/nginx-config.yaml | 94 + .../templates/poddisruptionbudget.yaml | 22 + .../grafana/templates/podsecuritypolicy.yaml | 45 + .../charts/grafana/templates/pvc.yaml | 35 + .../charts/grafana/templates/role.yaml | 32 + .../charts/grafana/templates/rolebinding.yaml | 25 + .../charts/grafana/templates/secret-env.yaml | 14 + .../charts/grafana/templates/secret.yaml | 26 + .../charts/grafana/templates/service.yaml | 55 + .../grafana/templates/serviceaccount.yaml | 14 + .../grafana/templates/servicemonitor.yaml | 58 + .../charts/grafana/templates/statefulset.yaml | 56 + .../templates/tests/test-configmap.yaml | 17 + .../tests/test-podsecuritypolicy.yaml | 29 + .../grafana/templates/tests/test-role.yaml | 14 + .../templates/tests/test-rolebinding.yaml | 17 + .../templates/tests/test-serviceaccount.yaml | 9 + .../charts/grafana/templates/tests/test.yaml | 51 + .../charts/grafana/values.yaml | 1088 +++ .../charts/hardenedKubelet/.helmignore | 23 + .../charts/hardenedKubelet/Chart.yaml | 14 + .../charts/hardenedKubelet/README.md | 90 + .../hardenedKubelet/templates/_helpers.tpl | 166 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/hardenedKubelet/values.yaml | 166 + .../charts/hardenedNodeExporter/.helmignore | 23 + .../charts/hardenedNodeExporter/Chart.yaml | 14 + .../charts/hardenedNodeExporter/README.md | 90 + .../templates/_helpers.tpl | 166 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/hardenedNodeExporter/values.yaml | 166 + .../charts/k3sServer/.helmignore | 23 + .../charts/k3sServer/Chart.yaml | 14 + .../charts/k3sServer/README.md | 90 + .../charts/k3sServer/templates/_helpers.tpl | 166 + .../templates/pushprox-clients-rbac.yaml | 97 + .../k3sServer/templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../k3sServer/templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/k3sServer/values.yaml | 166 + .../charts/kube-state-metrics/.helmignore | 21 + .../charts/kube-state-metrics/Chart.yaml | 28 + .../charts/kube-state-metrics/README.md | 68 + .../kube-state-metrics/templates/NOTES.txt | 10 + .../kube-state-metrics/templates/_helpers.tpl | 111 + .../templates/clusterrolebinding.yaml | 20 + .../templates/deployment.yaml | 172 + .../templates/kubeconfig-secret.yaml | 12 + .../kube-state-metrics/templates/pdb.yaml | 18 + .../templates/podsecuritypolicy.yaml | 39 + .../templates/psp-clusterrole.yaml | 19 + .../templates/psp-clusterrolebinding.yaml | 16 + .../kube-state-metrics/templates/role.yaml | 193 + .../templates/rolebinding.yaml | 24 + .../kube-state-metrics/templates/service.yaml | 41 + .../templates/serviceaccount.yaml | 15 + .../templates/servicemonitor.yaml | 86 + .../templates/stsdiscovery-role.yaml | 26 + .../templates/stsdiscovery-rolebinding.yaml | 17 + .../charts/kube-state-metrics/values.yaml | 271 + .../kubeAdmControllerManager/.helmignore | 23 + .../kubeAdmControllerManager/Chart.yaml | 14 + .../charts/kubeAdmControllerManager/README.md | 90 + .../templates/_helpers.tpl | 166 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../kubeAdmControllerManager/values.yaml | 166 + .../charts/kubeAdmEtcd/.helmignore | 23 + .../charts/kubeAdmEtcd/Chart.yaml | 14 + .../charts/kubeAdmEtcd/README.md | 90 + .../charts/kubeAdmEtcd/templates/_helpers.tpl | 166 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../kubeAdmEtcd/templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/kubeAdmEtcd/values.yaml | 166 + .../charts/kubeAdmProxy/.helmignore | 23 + .../charts/kubeAdmProxy/Chart.yaml | 14 + .../charts/kubeAdmProxy/README.md | 90 + .../kubeAdmProxy/templates/_helpers.tpl | 166 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/kubeAdmProxy/values.yaml | 166 + .../charts/kubeAdmScheduler/.helmignore | 23 + .../charts/kubeAdmScheduler/Chart.yaml | 14 + .../charts/kubeAdmScheduler/README.md | 90 + .../kubeAdmScheduler/templates/_helpers.tpl | 166 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/kubeAdmScheduler/values.yaml | 166 + .../charts/prometheus-adapter/.helmignore | 21 + .../charts/prometheus-adapter/Chart.yaml | 27 + .../charts/prometheus-adapter/README.md | 138 + .../prometheus-adapter/templates/NOTES.txt | 9 + .../prometheus-adapter/templates/_helpers.tpl | 113 + .../templates/certmanager.yaml | 76 + .../cluster-role-binding-auth-delegator.yaml | 20 + .../cluster-role-binding-resource-reader.yaml | 20 + .../cluster-role-resource-reader.yaml | 24 + .../templates/configmap.yaml | 97 + .../templates/custom-metrics-apiservice.yaml | 32 + ...stom-metrics-cluster-role-binding-hpa.yaml | 24 + .../custom-metrics-cluster-role.yaml | 17 + .../templates/deployment.yaml | 147 + .../external-metrics-apiservice.yaml | 32 + ...rnal-metrics-cluster-role-binding-hpa.yaml | 20 + .../external-metrics-cluster-role.yaml | 21 + .../prometheus-adapter/templates/pdb.yaml | 23 + .../prometheus-adapter/templates/psp.yaml | 66 + .../resource-metrics-apiservice.yaml | 32 + ...resource-metrics-cluster-role-binding.yaml | 20 + .../resource-metrics-cluster-role.yaml | 23 + .../templates/role-binding-auth-reader.yaml | 21 + .../prometheus-adapter/templates/secret.yaml | 17 + .../prometheus-adapter/templates/service.yaml | 27 + .../templates/serviceaccount.yaml | 18 + .../charts/prometheus-adapter/values.yaml | 217 + .../prometheus-node-exporter/.helmignore | 21 + .../prometheus-node-exporter/Chart.yaml | 25 + .../charts/prometheus-node-exporter/README.md | 77 + .../templates/NOTES.txt | 15 + .../templates/_helpers.tpl | 136 + .../templates/daemonset.yaml | 234 + .../templates/endpoints.yaml | 17 + .../templates/psp-clusterrole.yaml | 15 + .../templates/psp-clusterrolebinding.yaml | 17 + .../templates/psp.yaml | 50 + .../templates/service.yaml | 22 + .../templates/serviceaccount.yaml | 14 + .../templates/servicemonitor.yaml | 61 + .../prometheus-node-exporter/values.yaml | 252 + .../charts/rke2ControllerManager/.helmignore | 23 + .../charts/rke2ControllerManager/Chart.yaml | 14 + .../charts/rke2ControllerManager/README.md | 90 + .../templates/_helpers.tpl | 166 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rke2ControllerManager/values.yaml | 166 + .../charts/rke2Etcd/.helmignore | 23 + .../charts/rke2Etcd/Chart.yaml | 14 + .../charts/rke2Etcd/README.md | 90 + .../charts/rke2Etcd/templates/_helpers.tpl | 166 + .../templates/pushprox-clients-rbac.yaml | 97 + .../rke2Etcd/templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../rke2Etcd/templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rke2Etcd/values.yaml | 166 + .../charts/rke2IngressNginx/.helmignore | 23 + .../charts/rke2IngressNginx/Chart.yaml | 14 + .../charts/rke2IngressNginx/README.md | 90 + .../rke2IngressNginx/templates/_helpers.tpl | 166 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rke2IngressNginx/values.yaml | 166 + .../charts/rke2Proxy/.helmignore | 23 + .../charts/rke2Proxy/Chart.yaml | 14 + .../charts/rke2Proxy/README.md | 90 + .../charts/rke2Proxy/templates/_helpers.tpl | 166 + .../templates/pushprox-clients-rbac.yaml | 97 + .../rke2Proxy/templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../rke2Proxy/templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rke2Proxy/values.yaml | 166 + .../charts/rke2Scheduler/.helmignore | 23 + .../charts/rke2Scheduler/Chart.yaml | 14 + .../charts/rke2Scheduler/README.md | 90 + .../rke2Scheduler/templates/_helpers.tpl | 166 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rke2Scheduler/values.yaml | 166 + .../charts/rkeControllerManager/.helmignore | 23 + .../charts/rkeControllerManager/Chart.yaml | 14 + .../charts/rkeControllerManager/README.md | 90 + .../templates/_helpers.tpl | 166 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rkeControllerManager/values.yaml | 166 + .../charts/rkeEtcd/.helmignore | 23 + .../charts/rkeEtcd/Chart.yaml | 14 + .../102.0.5+up40.1.2/charts/rkeEtcd/README.md | 90 + .../charts/rkeEtcd/templates/_helpers.tpl | 166 + .../templates/pushprox-clients-rbac.yaml | 97 + .../rkeEtcd/templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../rkeEtcd/templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rkeEtcd/values.yaml | 166 + .../charts/rkeIngressNginx/.helmignore | 23 + .../charts/rkeIngressNginx/Chart.yaml | 14 + .../charts/rkeIngressNginx/README.md | 90 + .../rkeIngressNginx/templates/_helpers.tpl | 166 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rkeIngressNginx/values.yaml | 166 + .../charts/rkeProxy/.helmignore | 23 + .../charts/rkeProxy/Chart.yaml | 14 + .../charts/rkeProxy/README.md | 90 + .../charts/rkeProxy/templates/_helpers.tpl | 166 + .../templates/pushprox-clients-rbac.yaml | 97 + .../rkeProxy/templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../rkeProxy/templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rkeProxy/values.yaml | 166 + .../charts/rkeScheduler/.helmignore | 23 + .../charts/rkeScheduler/Chart.yaml | 14 + .../charts/rkeScheduler/README.md | 90 + .../rkeScheduler/templates/_helpers.tpl | 166 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rkeScheduler/values.yaml | 166 + .../charts/windowsExporter/.helmignore | 23 + .../charts/windowsExporter/Chart.yaml | 15 + .../charts/windowsExporter/README.md | 17 + .../scripts/check-wins-version.ps1 | 20 + .../windowsExporter/scripts/proxy-entry.ps1 | 11 + .../charts/windowsExporter/scripts/run.ps1 | 78 + .../windowsExporter/templates/_helpers.tpl | 113 + .../windowsExporter/templates/configmap.yaml | 10 + .../windowsExporter/templates/daemonset.yaml | 77 + .../templates/prometheusrule.yaml | 13 + .../windowsExporter/templates/rbac.yaml | 81 + .../windowsExporter/templates/service.yaml | 15 + .../templates/servicemonitor.yaml | 41 + .../charts/windowsExporter/values.yaml | 52 + .../files/ingress-nginx/nginx.json | 1445 +++ .../request-handling-performance.json | 963 ++ .../cluster/rancher-cluster-nodes.json | 793 ++ .../rancher/cluster/rancher-cluster.json | 776 ++ .../rancher/home/rancher-default-home.json | 1290 +++ .../files/rancher/k8s/rancher-etcd-nodes.json | 687 ++ .../files/rancher/k8s/rancher-etcd.json | 669 ++ .../k8s/rancher-k8s-components-nodes.json | 527 ++ .../rancher/k8s/rancher-k8s-components.json | 519 ++ .../rancher/nodes/rancher-node-detail.json | 805 ++ .../files/rancher/nodes/rancher-node.json | 792 ++ .../performance/performance-debugging.json | 1707 ++++ .../rancher/pods/rancher-pod-containers.json | 636 ++ .../files/rancher/pods/rancher-pod.json | 636 ++ .../workloads/rancher-workload-pods.json | 652 ++ .../rancher/workloads/rancher-workload.json | 652 ++ .../delete-workloads-with-old-labels.sh | 14 + .../102.0.5+up40.1.2/templates/NOTES.txt | 4 + .../102.0.5+up40.1.2/templates/_helpers.tpl | 384 + .../templates/alertmanager/alertmanager.yaml | 170 + .../templates/alertmanager/extrasecret.yaml | 20 + .../templates/alertmanager/ingress.yaml | 77 + .../alertmanager/ingressperreplica.yaml | 67 + .../alertmanager/podDisruptionBudget.yaml | 21 + .../templates/alertmanager/psp-role.yaml | 21 + .../alertmanager/psp-rolebinding.yaml | 18 + .../templates/alertmanager/psp.yaml | 45 + .../templates/alertmanager/secret.yaml | 33 + .../templates/alertmanager/service.yaml | 53 + .../alertmanager/serviceaccount.yaml | 20 + .../alertmanager/servicemonitor.yaml | 55 + .../alertmanager/serviceperreplica.yaml | 49 + .../templates/exporters/core-dns/service.yaml | 24 + .../exporters/core-dns/servicemonitor.yaml | 49 + .../kube-api-server/servicemonitor.yaml | 52 + .../kube-controller-manager/endpoints.yaml | 22 + .../kube-controller-manager/service.yaml | 29 + .../servicemonitor.yaml | 60 + .../templates/exporters/kube-dns/service.yaml | 28 + .../exporters/kube-dns/servicemonitor.yaml | 62 + .../exporters/kube-etcd/endpoints.yaml | 20 + .../exporters/kube-etcd/service.yaml | 27 + .../exporters/kube-etcd/servicemonitor.yaml | 66 + .../exporters/kube-proxy/endpoints.yaml | 20 + .../exporters/kube-proxy/service.yaml | 27 + .../exporters/kube-proxy/servicemonitor.yaml | 54 + .../exporters/kube-scheduler/endpoints.yaml | 22 + .../exporters/kube-scheduler/service.yaml | 29 + .../kube-scheduler/servicemonitor.yaml | 60 + .../kube-state-metrics/validate.yaml | 7 + .../exporters/kubelet/servicemonitor.yaml | 229 + .../exporters/node-exporter/validate.yaml | 3 + .../grafana/configmap-dashboards.yaml | 24 + .../grafana/configmaps-datasources.yaml | 63 + .../alertmanager-overview.yaml | 616 ++ .../grafana/dashboards-1.14/apiserver.yaml | 1772 ++++ .../dashboards-1.14/cluster-total.yaml | 1882 ++++ .../dashboards-1.14/controller-manager.yaml | 1196 +++ .../grafana/dashboards-1.14/etcd.yaml | 1229 +++ .../dashboards-1.14/grafana-overview.yaml | 635 ++ .../grafana/dashboards-1.14/k8s-coredns.yaml | 1530 ++++ .../k8s-resources-cluster.yaml | 3088 +++++++ .../k8s-resources-namespace.yaml | 2797 ++++++ .../dashboards-1.14/k8s-resources-node.yaml | 1026 +++ .../dashboards-1.14/k8s-resources-pod.yaml | 2469 ++++++ .../k8s-resources-workload.yaml | 2024 +++++ .../k8s-resources-workloads-namespace.yaml | 2189 +++++ .../grafana/dashboards-1.14/kubelet.yaml | 2256 +++++ .../dashboards-1.14/namespace-by-pod.yaml | 1464 ++++ .../namespace-by-workload.yaml | 1736 ++++ .../node-cluster-rsrc-use.yaml | 1063 +++ .../dashboards-1.14/node-rsrc-use.yaml | 1089 +++ .../grafana/dashboards-1.14/nodes-darwin.yaml | 1073 +++ .../grafana/dashboards-1.14/nodes.yaml | 1066 +++ .../persistentvolumesusage.yaml | 587 ++ .../grafana/dashboards-1.14/pod-total.yaml | 1228 +++ .../prometheus-remote-write.yaml | 1674 ++++ .../grafana/dashboards-1.14/prometheus.yaml | 1235 +++ .../grafana/dashboards-1.14/proxy.yaml | 1276 +++ .../grafana/dashboards-1.14/scheduler.yaml | 1118 +++ .../dashboards-1.14/workload-total.yaml | 1438 +++ .../templates/grafana/namespaces.yaml | 13 + .../job-patch/clusterrole.yaml | 33 + .../job-patch/clusterrolebinding.yaml | 20 + .../job-patch/job-createSecret.yaml | 69 + .../job-patch/job-patchWebhook.yaml | 70 + .../admission-webhooks/job-patch/psp.yaml | 47 + .../admission-webhooks/job-patch/role.yaml | 21 + .../job-patch/rolebinding.yaml | 21 + .../job-patch/serviceaccount.yaml | 17 + .../mutatingWebhookConfiguration.yaml | 42 + .../validatingWebhookConfiguration.yaml | 41 + .../prometheus-operator/certmanager.yaml | 57 + .../prometheus-operator/clusterrole.yaml | 81 + .../clusterrolebinding.yaml | 17 + .../prometheus-operator/deployment.yaml | 164 + .../prometheus-operator/psp-clusterrole.yaml | 20 + .../psp-clusterrolebinding.yaml | 17 + .../templates/prometheus-operator/psp.yaml | 45 + .../prometheus-operator/service.yaml | 58 + .../prometheus-operator/serviceaccount.yaml | 16 + .../prometheus-operator/servicemonitor.yaml | 54 + .../templates/prometheus/_rules.tpl | 36 + .../additionalAlertRelabelConfigs.yaml | 16 + .../additionalAlertmanagerConfigs.yaml | 16 + .../prometheus/additionalPrometheusRules.yaml | 43 + .../prometheus/additionalScrapeConfigs.yaml | 20 + .../templates/prometheus/clusterrole.yaml | 30 + .../prometheus/clusterrolebinding.yaml | 18 + .../templates/prometheus/csi-secret.yaml | 12 + .../templates/prometheus/extrasecret.yaml | 20 + .../templates/prometheus/ingress.yaml | 77 + .../prometheus/ingressThanosSidecar.yaml | 76 + .../prometheus/ingressperreplica.yaml | 67 + .../templates/prometheus/nginx-config.yaml | 68 + .../prometheus/podDisruptionBudget.yaml | 21 + .../templates/prometheus/podmonitors.yaml | 37 + .../templates/prometheus/prometheus.yaml | 388 + .../templates/prometheus/psp-clusterrole.yaml | 20 + .../prometheus/psp-clusterrolebinding.yaml | 18 + .../templates/prometheus/psp.yaml | 56 + .../rules-1.14/alertmanager.rules.yaml | 217 + .../rules-1.14/config-reloaders.yaml | 46 + .../templates/prometheus/rules-1.14/etcd.yaml | 296 + .../prometheus/rules-1.14/general.rules.yaml | 98 + .../prometheus/rules-1.14/k8s.rules.yaml | 173 + .../kube-apiserver-availability.rules.yaml | 136 + .../kube-apiserver-burnrate.rules.yaml | 328 + .../kube-apiserver-histogram.rules.yaml | 37 + .../rules-1.14/kube-apiserver-slos.yaml | 115 + .../kube-prometheus-general.rules.yaml | 31 + .../kube-prometheus-node-recording.rules.yaml | 39 + .../rules-1.14/kube-scheduler.rules.yaml | 65 + .../rules-1.14/kube-state-metrics.yaml | 107 + .../prometheus/rules-1.14/kubelet.rules.yaml | 41 + .../rules-1.14/kubernetes-apps.yaml | 375 + .../rules-1.14/kubernetes-resources.yaml | 193 + .../rules-1.14/kubernetes-storage.yaml | 160 + .../kubernetes-system-apiserver.yaml | 128 + .../kubernetes-system-controller-manager.yaml | 47 + .../kubernetes-system-kube-proxy.yaml | 46 + .../rules-1.14/kubernetes-system-kubelet.yaml | 253 + .../kubernetes-system-scheduler.yaml | 46 + .../rules-1.14/kubernetes-system.yaml | 65 + .../rules-1.14/node-exporter.rules.yaml | 89 + .../prometheus/rules-1.14/node-exporter.yaml | 398 + .../prometheus/rules-1.14/node-network.yaml | 44 + .../prometheus/rules-1.14/node.rules.yaml | 55 + .../rules-1.14/prometheus-operator.yaml | 148 + .../prometheus/rules-1.14/prometheus.yaml | 448 + .../templates/prometheus/service.yaml | 64 + .../prometheus/serviceThanosSidecar.yaml | 39 + .../serviceThanosSidecarExternal.yaml | 46 + .../templates/prometheus/serviceaccount.yaml | 20 + .../templates/prometheus/servicemonitor.yaml | 52 + .../servicemonitorThanosSidecar.yaml | 51 + .../templates/prometheus/servicemonitors.yaml | 38 + .../prometheus/serviceperreplica.yaml | 49 + .../rancher-monitoring/clusterrole.yaml | 134 + .../rancher-monitoring/config-role.yaml | 48 + .../rancher-monitoring/dashboard-role.yaml | 47 + .../addons/ingress-nginx-dashboard.yaml | 18 + .../rancher/cluster-dashboards.yaml | 17 + .../dashboards/rancher/default-dashboard.yaml | 17 + .../dashboards/rancher/k8s-dashboards.yaml | 31 + .../dashboards/rancher/nodes-dashboards.yaml | 17 + .../rancher/performance-dashboards.yaml | 18 + .../dashboards/rancher/pods-dashboards.yaml | 17 + .../rancher/workload-dashboards.yaml | 17 + .../exporters/ingress-nginx/service.yaml | 27 + .../ingress-nginx/servicemonitor.yaml | 49 + .../exporters/rancher/servicemonitor.yaml | 58 + .../rancher-monitoring/hardened.yaml | 128 + .../rancher-monitoring/upgrade/configmap.yaml | 13 + .../rancher-monitoring/upgrade/job.yaml | 46 + .../rancher-monitoring/upgrade/rbac.yaml | 131 + .../templates/thanos-ruler/extrasecret.yaml | 20 + .../templates/thanos-ruler/ingress.yaml | 77 + .../thanos-ruler/podDisruptionBudget.yaml | 21 + .../templates/thanos-ruler/ruler.yaml | 168 + .../templates/thanos-ruler/service.yaml | 53 + .../thanos-ruler/serviceaccount.yaml | 20 + .../thanos-ruler/servicemonitor.yaml | 56 + .../templates/validate-install-crd.yaml | 21 + .../templates/validate-psp-install.yaml | 7 + .../102.0.5+up40.1.2/values.yaml | 4190 +++++++++ index.yaml | 326 + regsync.yaml | 4 + release.yaml | 22 +- 662 files changed, 115468 insertions(+), 4 deletions(-) create mode 100644 assets/fleet-agent/fleet-agent-102.2.5+up0.8.5.tgz create mode 100644 assets/fleet-crd/fleet-crd-102.2.5+up0.8.5.tgz create mode 100644 assets/fleet/fleet-102.2.5+up0.8.5.tgz create mode 100644 assets/longhorn-crd/longhorn-crd-102.4.1+up1.6.2.tgz create mode 100644 assets/longhorn/longhorn-102.4.1+up1.6.2.tgz create mode 100644 assets/prometheus-federator/prometheus-federator-3.0.2+up0.3.3.tgz create mode 100644 assets/rancher-alerting-drivers/rancher-alerting-drivers-102.1.2.tgz create mode 100644 assets/rancher-monitoring-crd/rancher-monitoring-crd-102.0.5+up40.1.2.tgz create mode 100644 assets/rancher-monitoring/rancher-monitoring-102.0.5+up40.1.2.tgz create mode 100644 charts/fleet-agent/102.2.5+up0.8.5/Chart.yaml create mode 100644 charts/fleet-agent/102.2.5+up0.8.5/README.md create mode 100644 charts/fleet-agent/102.2.5+up0.8.5/templates/_helpers.tpl create mode 100644 charts/fleet-agent/102.2.5+up0.8.5/templates/configmap.yaml create mode 100644 charts/fleet-agent/102.2.5+up0.8.5/templates/deployment.yaml create mode 100644 charts/fleet-agent/102.2.5+up0.8.5/templates/network_policy_allow_all.yaml create mode 100644 charts/fleet-agent/102.2.5+up0.8.5/templates/patch_default_serviceaccount.yaml create mode 100644 charts/fleet-agent/102.2.5+up0.8.5/templates/rbac.yaml create mode 100644 charts/fleet-agent/102.2.5+up0.8.5/templates/secret.yaml create mode 100644 charts/fleet-agent/102.2.5+up0.8.5/templates/serviceaccount.yaml create mode 100644 charts/fleet-agent/102.2.5+up0.8.5/templates/validate.yaml create mode 100644 charts/fleet-agent/102.2.5+up0.8.5/values.yaml create mode 100644 charts/fleet-crd/102.2.5+up0.8.5/Chart.yaml create mode 100644 charts/fleet-crd/102.2.5+up0.8.5/README.md create mode 100644 charts/fleet-crd/102.2.5+up0.8.5/templates/crds.yaml create mode 100644 charts/fleet-crd/102.2.5+up0.8.5/templates/gitjobs-crds.yaml create mode 100644 charts/fleet-crd/102.2.5+up0.8.5/values.yaml create mode 100644 charts/fleet/102.2.5+up0.8.5/Chart.yaml create mode 100644 charts/fleet/102.2.5+up0.8.5/README.md create mode 100644 charts/fleet/102.2.5+up0.8.5/charts/gitjob/.helmignore create mode 100644 charts/fleet/102.2.5+up0.8.5/charts/gitjob/Chart.yaml create mode 100644 charts/fleet/102.2.5+up0.8.5/charts/gitjob/templates/_helpers.tpl create mode 100644 charts/fleet/102.2.5+up0.8.5/charts/gitjob/templates/clusterrole.yaml create mode 100644 charts/fleet/102.2.5+up0.8.5/charts/gitjob/templates/clusterrolebinding.yaml create mode 100644 charts/fleet/102.2.5+up0.8.5/charts/gitjob/templates/deployment.yaml create mode 100644 charts/fleet/102.2.5+up0.8.5/charts/gitjob/templates/leases.yaml create mode 100644 charts/fleet/102.2.5+up0.8.5/charts/gitjob/templates/service.yaml create mode 100644 charts/fleet/102.2.5+up0.8.5/charts/gitjob/templates/serviceaccount.yaml create mode 100644 charts/fleet/102.2.5+up0.8.5/charts/gitjob/values.yaml create mode 100644 charts/fleet/102.2.5+up0.8.5/templates/_helpers.tpl create mode 100644 charts/fleet/102.2.5+up0.8.5/templates/configmap.yaml create mode 100644 charts/fleet/102.2.5+up0.8.5/templates/deployment.yaml create mode 100644 charts/fleet/102.2.5+up0.8.5/templates/job_cleanup_clusterregistrations.yaml create mode 100644 charts/fleet/102.2.5+up0.8.5/templates/rbac.yaml create mode 100644 charts/fleet/102.2.5+up0.8.5/templates/serviceaccount.yaml create mode 100644 charts/fleet/102.2.5+up0.8.5/values.yaml create mode 100644 charts/longhorn-crd/102.4.1+up1.6.2/Chart.yaml create mode 100644 charts/longhorn-crd/102.4.1+up1.6.2/README.md create mode 100644 charts/longhorn-crd/102.4.1+up1.6.2/templates/_helpers.tpl create mode 100644 charts/longhorn-crd/102.4.1+up1.6.2/templates/crds.yaml create mode 100644 charts/longhorn/102.4.1+up1.6.2/.helmignore create mode 100644 charts/longhorn/102.4.1+up1.6.2/Chart.yaml create mode 100644 charts/longhorn/102.4.1+up1.6.2/README.md create mode 100644 charts/longhorn/102.4.1+up1.6.2/app-readme.md create mode 100644 charts/longhorn/102.4.1+up1.6.2/questions.yaml create mode 100644 charts/longhorn/102.4.1+up1.6.2/templates/NOTES.txt create mode 100644 charts/longhorn/102.4.1+up1.6.2/templates/_helpers.tpl create mode 100644 charts/longhorn/102.4.1+up1.6.2/templates/clusterrole.yaml create mode 100644 charts/longhorn/102.4.1+up1.6.2/templates/clusterrolebinding.yaml create mode 100644 charts/longhorn/102.4.1+up1.6.2/templates/daemonset-sa.yaml create mode 100644 charts/longhorn/102.4.1+up1.6.2/templates/default-setting.yaml create mode 100644 charts/longhorn/102.4.1+up1.6.2/templates/deployment-driver.yaml create mode 100644 charts/longhorn/102.4.1+up1.6.2/templates/deployment-ui.yaml create mode 100644 charts/longhorn/102.4.1+up1.6.2/templates/ingress.yaml create mode 100644 charts/longhorn/102.4.1+up1.6.2/templates/network-policies/backing-image-data-source-network-policy.yaml create mode 100644 charts/longhorn/102.4.1+up1.6.2/templates/network-policies/backing-image-manager-network-policy.yaml create mode 100644 charts/longhorn/102.4.1+up1.6.2/templates/network-policies/instance-manager-networking.yaml create mode 100644 charts/longhorn/102.4.1+up1.6.2/templates/network-policies/manager-network-policy.yaml create mode 100644 charts/longhorn/102.4.1+up1.6.2/templates/network-policies/recovery-backend-network-policy.yaml create mode 100644 charts/longhorn/102.4.1+up1.6.2/templates/network-policies/ui-frontend-network-policy.yaml create mode 100644 charts/longhorn/102.4.1+up1.6.2/templates/network-policies/webhook-network-policy.yaml create mode 100644 charts/longhorn/102.4.1+up1.6.2/templates/postupgrade-job.yaml create mode 100644 charts/longhorn/102.4.1+up1.6.2/templates/preupgrade-job.yaml create mode 100644 charts/longhorn/102.4.1+up1.6.2/templates/priorityclass.yaml create mode 100644 charts/longhorn/102.4.1+up1.6.2/templates/psp.yaml create mode 100644 charts/longhorn/102.4.1+up1.6.2/templates/registry-secret.yaml create mode 100644 charts/longhorn/102.4.1+up1.6.2/templates/serviceaccount.yaml create mode 100644 charts/longhorn/102.4.1+up1.6.2/templates/servicemonitor.yaml create mode 100644 charts/longhorn/102.4.1+up1.6.2/templates/services.yaml create mode 100644 charts/longhorn/102.4.1+up1.6.2/templates/storageclass.yaml create mode 100644 charts/longhorn/102.4.1+up1.6.2/templates/tls-secrets.yaml create mode 100644 charts/longhorn/102.4.1+up1.6.2/templates/uninstall-job.yaml create mode 100644 charts/longhorn/102.4.1+up1.6.2/templates/userroles.yaml create mode 100644 charts/longhorn/102.4.1+up1.6.2/templates/validate-install-crd.yaml create mode 100644 charts/longhorn/102.4.1+up1.6.2/templates/validate-psp-install.yaml create mode 100644 charts/longhorn/102.4.1+up1.6.2/values.yaml create mode 100644 charts/prometheus-federator/3.0.2+up0.3.3/Chart.yaml create mode 100644 charts/prometheus-federator/3.0.2+up0.3.3/README.md create mode 100644 charts/prometheus-federator/3.0.2+up0.3.3/app-README.md create mode 100644 charts/prometheus-federator/3.0.2+up0.3.3/charts/helmProjectOperator/Chart.yaml create mode 100644 charts/prometheus-federator/3.0.2+up0.3.3/charts/helmProjectOperator/README.md create mode 100644 charts/prometheus-federator/3.0.2+up0.3.3/charts/helmProjectOperator/app-readme.md create mode 100644 charts/prometheus-federator/3.0.2+up0.3.3/charts/helmProjectOperator/questions.yaml create mode 100644 charts/prometheus-federator/3.0.2+up0.3.3/charts/helmProjectOperator/templates/NOTES.txt create mode 100644 charts/prometheus-federator/3.0.2+up0.3.3/charts/helmProjectOperator/templates/_helpers.tpl create mode 100644 charts/prometheus-federator/3.0.2+up0.3.3/charts/helmProjectOperator/templates/cleanup.yaml create mode 100644 charts/prometheus-federator/3.0.2+up0.3.3/charts/helmProjectOperator/templates/clusterrole.yaml create mode 100644 charts/prometheus-federator/3.0.2+up0.3.3/charts/helmProjectOperator/templates/configmap.yaml create mode 100644 charts/prometheus-federator/3.0.2+up0.3.3/charts/helmProjectOperator/templates/deployment.yaml create mode 100644 charts/prometheus-federator/3.0.2+up0.3.3/charts/helmProjectOperator/templates/psp.yaml create mode 100644 charts/prometheus-federator/3.0.2+up0.3.3/charts/helmProjectOperator/templates/rbac.yaml create mode 100644 charts/prometheus-federator/3.0.2+up0.3.3/charts/helmProjectOperator/templates/system-namespaces-configmap.yaml create mode 100644 charts/prometheus-federator/3.0.2+up0.3.3/charts/helmProjectOperator/templates/validate-psp-install.yaml create mode 100644 charts/prometheus-federator/3.0.2+up0.3.3/charts/helmProjectOperator/values.yaml create mode 100644 charts/prometheus-federator/3.0.2+up0.3.3/questions.yaml create mode 100644 charts/prometheus-federator/3.0.2+up0.3.3/templates/NOTES.txt create mode 100644 charts/prometheus-federator/3.0.2+up0.3.3/templates/_helpers.tpl create mode 100644 charts/prometheus-federator/3.0.2+up0.3.3/values.yaml create mode 100644 charts/rancher-alerting-drivers/102.1.2/Chart.yaml create mode 100644 charts/rancher-alerting-drivers/102.1.2/README.md create mode 100644 charts/rancher-alerting-drivers/102.1.2/app-readme.md create mode 100644 charts/rancher-alerting-drivers/102.1.2/charts/prom2teams/.helmignore create mode 100644 charts/rancher-alerting-drivers/102.1.2/charts/prom2teams/Chart.yaml create mode 100644 charts/rancher-alerting-drivers/102.1.2/charts/prom2teams/files/teams.j2 create mode 100644 charts/rancher-alerting-drivers/102.1.2/charts/prom2teams/templates/NOTES.txt create mode 100644 charts/rancher-alerting-drivers/102.1.2/charts/prom2teams/templates/_helpers.tpl create mode 100644 charts/rancher-alerting-drivers/102.1.2/charts/prom2teams/templates/configmap.yaml create mode 100644 charts/rancher-alerting-drivers/102.1.2/charts/prom2teams/templates/deployment.yaml create mode 100644 charts/rancher-alerting-drivers/102.1.2/charts/prom2teams/templates/psp.yaml create mode 100644 charts/rancher-alerting-drivers/102.1.2/charts/prom2teams/templates/service-account.yaml create mode 100644 charts/rancher-alerting-drivers/102.1.2/charts/prom2teams/templates/service.yaml create mode 100644 charts/rancher-alerting-drivers/102.1.2/charts/prom2teams/values.yaml create mode 100644 charts/rancher-alerting-drivers/102.1.2/charts/sachet/.helmignore create mode 100644 charts/rancher-alerting-drivers/102.1.2/charts/sachet/Chart.yaml create mode 100644 charts/rancher-alerting-drivers/102.1.2/charts/sachet/files/template.tmpl create mode 100644 charts/rancher-alerting-drivers/102.1.2/charts/sachet/templates/NOTES.txt create mode 100644 charts/rancher-alerting-drivers/102.1.2/charts/sachet/templates/_helpers.tpl create mode 100644 charts/rancher-alerting-drivers/102.1.2/charts/sachet/templates/configmap-pre-install.yaml create mode 100644 charts/rancher-alerting-drivers/102.1.2/charts/sachet/templates/deployment.yaml create mode 100644 charts/rancher-alerting-drivers/102.1.2/charts/sachet/templates/psp.yaml create mode 100644 charts/rancher-alerting-drivers/102.1.2/charts/sachet/templates/service-account.yaml create mode 100644 charts/rancher-alerting-drivers/102.1.2/charts/sachet/templates/service.yaml create mode 100644 charts/rancher-alerting-drivers/102.1.2/charts/sachet/values.yaml create mode 100644 charts/rancher-alerting-drivers/102.1.2/questions.yml create mode 100644 charts/rancher-alerting-drivers/102.1.2/templates/NOTES.txt create mode 100644 charts/rancher-alerting-drivers/102.1.2/templates/_helpers.tpl create mode 100644 charts/rancher-alerting-drivers/102.1.2/templates/cluster-role.yaml create mode 100644 charts/rancher-alerting-drivers/102.1.2/templates/hardened.yaml create mode 100644 charts/rancher-alerting-drivers/102.1.2/templates/validate-psp-install.yaml create mode 100644 charts/rancher-alerting-drivers/102.1.2/values.yaml create mode 100644 charts/rancher-monitoring-crd/102.0.5+up40.1.2/Chart.yaml create mode 100644 charts/rancher-monitoring-crd/102.0.5+up40.1.2/README.md create mode 100644 charts/rancher-monitoring-crd/102.0.5+up40.1.2/files/crd-manifest.tgz create mode 100644 charts/rancher-monitoring-crd/102.0.5+up40.1.2/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring-crd/102.0.5+up40.1.2/templates/jobs.yaml create mode 100644 charts/rancher-monitoring-crd/102.0.5+up40.1.2/templates/manifest.yaml create mode 100644 charts/rancher-monitoring-crd/102.0.5+up40.1.2/templates/rbac.yaml create mode 100644 charts/rancher-monitoring-crd/102.0.5+up40.1.2/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring-crd/102.0.5+up40.1.2/values.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/.helmignore create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/CHANGELOG.md create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/CONTRIBUTING.md create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/Chart.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/README.md create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/app-README.md create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/.helmignore create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/Chart.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/README.md create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/dashboards/custom-dashboard.json create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/NOTES.txt create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/_pod.tpl create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/clusterrole.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/configmap-dashboard-provider.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/configmap.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/dashboards-json-configmap.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/deployment.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/extra-manifests.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/headless-service.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/hpa.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/image-renderer-deployment.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/image-renderer-network-policy.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/image-renderer-service.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/ingress.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/networkpolicy.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/nginx-config.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/poddisruptionbudget.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/podsecuritypolicy.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/pvc.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/role.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/rolebinding.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/secret-env.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/secret.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/service.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/statefulset.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/tests/test-configmap.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/tests/test-podsecuritypolicy.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/tests/test-role.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/tests/test-rolebinding.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/tests/test-serviceaccount.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/tests/test.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/values.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedKubelet/.helmignore create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedKubelet/Chart.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedKubelet/README.md create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedKubelet/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedKubelet/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedKubelet/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedKubelet/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedKubelet/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedKubelet/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedKubelet/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedKubelet/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedKubelet/values.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedNodeExporter/.helmignore create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedNodeExporter/Chart.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedNodeExporter/README.md create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedNodeExporter/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedNodeExporter/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedNodeExporter/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedNodeExporter/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedNodeExporter/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedNodeExporter/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedNodeExporter/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedNodeExporter/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedNodeExporter/values.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/k3sServer/.helmignore create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/k3sServer/Chart.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/k3sServer/README.md create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/k3sServer/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/k3sServer/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/k3sServer/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/k3sServer/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/k3sServer/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/k3sServer/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/k3sServer/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/k3sServer/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/k3sServer/values.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/.helmignore create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/Chart.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/README.md create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/NOTES.txt create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/deployment.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/kubeconfig-secret.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/pdb.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/podsecuritypolicy.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/psp-clusterrole.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/psp-clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/role.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/rolebinding.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/service.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/stsdiscovery-role.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/stsdiscovery-rolebinding.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/values.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmControllerManager/.helmignore create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmControllerManager/Chart.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmControllerManager/README.md create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmControllerManager/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmControllerManager/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmControllerManager/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmControllerManager/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmControllerManager/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmControllerManager/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmControllerManager/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmControllerManager/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmControllerManager/values.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmEtcd/.helmignore create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmEtcd/Chart.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmEtcd/README.md create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmEtcd/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmEtcd/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmEtcd/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmEtcd/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmEtcd/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmEtcd/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmEtcd/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmEtcd/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmEtcd/values.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmProxy/.helmignore create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmProxy/Chart.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmProxy/README.md create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmProxy/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmProxy/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmProxy/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmProxy/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmProxy/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmProxy/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmProxy/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmProxy/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmProxy/values.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmScheduler/.helmignore create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmScheduler/Chart.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmScheduler/README.md create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmScheduler/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmScheduler/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmScheduler/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmScheduler/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmScheduler/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmScheduler/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmScheduler/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmScheduler/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmScheduler/values.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/.helmignore create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/Chart.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/README.md create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/NOTES.txt create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/certmanager.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/cluster-role-binding-auth-delegator.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/cluster-role-binding-resource-reader.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/cluster-role-resource-reader.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/configmap.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/custom-metrics-apiservice.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/custom-metrics-cluster-role-binding-hpa.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/custom-metrics-cluster-role.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/deployment.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/external-metrics-apiservice.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/external-metrics-cluster-role-binding-hpa.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/external-metrics-cluster-role.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/pdb.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/psp.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/resource-metrics-apiservice.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/resource-metrics-cluster-role-binding.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/resource-metrics-cluster-role.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/role-binding-auth-reader.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/secret.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/service.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/values.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-node-exporter/.helmignore create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-node-exporter/Chart.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-node-exporter/README.md create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-node-exporter/templates/NOTES.txt create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-node-exporter/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-node-exporter/templates/daemonset.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-node-exporter/templates/endpoints.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-node-exporter/templates/psp-clusterrole.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-node-exporter/templates/psp-clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-node-exporter/templates/psp.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-node-exporter/templates/service.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-node-exporter/templates/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-node-exporter/templates/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-node-exporter/values.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2ControllerManager/.helmignore create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2ControllerManager/Chart.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2ControllerManager/README.md create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2ControllerManager/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2ControllerManager/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2ControllerManager/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2ControllerManager/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2ControllerManager/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2ControllerManager/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2ControllerManager/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2ControllerManager/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2ControllerManager/values.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Etcd/.helmignore create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Etcd/Chart.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Etcd/README.md create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Etcd/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Etcd/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Etcd/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Etcd/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Etcd/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Etcd/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Etcd/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Etcd/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Etcd/values.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2IngressNginx/.helmignore create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2IngressNginx/Chart.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2IngressNginx/README.md create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2IngressNginx/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2IngressNginx/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2IngressNginx/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2IngressNginx/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2IngressNginx/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2IngressNginx/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2IngressNginx/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2IngressNginx/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2IngressNginx/values.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Proxy/.helmignore create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Proxy/Chart.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Proxy/README.md create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Proxy/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Proxy/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Proxy/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Proxy/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Proxy/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Proxy/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Proxy/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Proxy/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Proxy/values.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Scheduler/.helmignore create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Scheduler/Chart.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Scheduler/README.md create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Scheduler/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Scheduler/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Scheduler/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Scheduler/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Scheduler/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Scheduler/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Scheduler/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Scheduler/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Scheduler/values.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeControllerManager/.helmignore create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeControllerManager/Chart.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeControllerManager/README.md create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeControllerManager/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeControllerManager/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeControllerManager/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeControllerManager/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeControllerManager/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeControllerManager/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeControllerManager/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeControllerManager/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeControllerManager/values.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeEtcd/.helmignore create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeEtcd/Chart.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeEtcd/README.md create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeEtcd/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeEtcd/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeEtcd/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeEtcd/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeEtcd/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeEtcd/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeEtcd/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeEtcd/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeEtcd/values.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeIngressNginx/.helmignore create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeIngressNginx/Chart.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeIngressNginx/README.md create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeIngressNginx/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeIngressNginx/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeIngressNginx/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeIngressNginx/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeIngressNginx/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeIngressNginx/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeIngressNginx/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeIngressNginx/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeIngressNginx/values.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeProxy/.helmignore create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeProxy/Chart.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeProxy/README.md create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeProxy/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeProxy/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeProxy/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeProxy/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeProxy/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeProxy/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeProxy/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeProxy/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeProxy/values.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeScheduler/.helmignore create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeScheduler/Chart.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeScheduler/README.md create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeScheduler/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeScheduler/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeScheduler/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeScheduler/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeScheduler/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeScheduler/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeScheduler/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeScheduler/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeScheduler/values.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/windowsExporter/.helmignore create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/windowsExporter/Chart.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/windowsExporter/README.md create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/windowsExporter/scripts/check-wins-version.ps1 create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/windowsExporter/scripts/proxy-entry.ps1 create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/windowsExporter/scripts/run.ps1 create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/windowsExporter/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/windowsExporter/templates/configmap.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/windowsExporter/templates/daemonset.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/windowsExporter/templates/prometheusrule.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/windowsExporter/templates/rbac.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/windowsExporter/templates/service.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/windowsExporter/templates/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/charts/windowsExporter/values.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/files/ingress-nginx/nginx.json create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/files/ingress-nginx/request-handling-performance.json create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/files/rancher/cluster/rancher-cluster-nodes.json create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/files/rancher/cluster/rancher-cluster.json create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/files/rancher/home/rancher-default-home.json create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/files/rancher/k8s/rancher-etcd-nodes.json create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/files/rancher/k8s/rancher-etcd.json create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/files/rancher/k8s/rancher-k8s-components-nodes.json create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/files/rancher/k8s/rancher-k8s-components.json create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/files/rancher/nodes/rancher-node-detail.json create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/files/rancher/nodes/rancher-node.json create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/files/rancher/performance/performance-debugging.json create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/files/rancher/pods/rancher-pod-containers.json create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/files/rancher/pods/rancher-pod.json create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/files/rancher/workloads/rancher-workload-pods.json create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/files/rancher/workloads/rancher-workload.json create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/files/upgrade/scripts/delete-workloads-with-old-labels.sh create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/NOTES.txt create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/alertmanager/alertmanager.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/alertmanager/extrasecret.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/alertmanager/ingress.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/alertmanager/ingressperreplica.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/alertmanager/podDisruptionBudget.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/alertmanager/psp-role.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/alertmanager/psp-rolebinding.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/alertmanager/psp.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/alertmanager/secret.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/alertmanager/service.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/alertmanager/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/alertmanager/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/alertmanager/serviceperreplica.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/core-dns/service.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/core-dns/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-api-server/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-controller-manager/endpoints.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-controller-manager/service.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-controller-manager/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-dns/service.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-dns/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-etcd/endpoints.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-etcd/service.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-etcd/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-proxy/endpoints.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-proxy/service.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-proxy/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-scheduler/endpoints.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-scheduler/service.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-scheduler/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-state-metrics/validate.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kubelet/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/node-exporter/validate.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/configmap-dashboards.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/configmaps-datasources.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/alertmanager-overview.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/apiserver.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/cluster-total.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/controller-manager.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/etcd.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/grafana-overview.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/k8s-coredns.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/k8s-resources-cluster.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/k8s-resources-namespace.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/k8s-resources-node.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/k8s-resources-pod.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/k8s-resources-workload.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/k8s-resources-workloads-namespace.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/kubelet.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/namespace-by-pod.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/namespace-by-workload.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/node-cluster-rsrc-use.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/node-rsrc-use.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/nodes-darwin.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/nodes.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/persistentvolumesusage.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/pod-total.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/prometheus-remote-write.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/prometheus.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/proxy.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/scheduler.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/workload-total.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/namespaces.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/admission-webhooks/job-patch/clusterrole.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/admission-webhooks/job-patch/clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/admission-webhooks/job-patch/job-createSecret.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/admission-webhooks/job-patch/job-patchWebhook.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/admission-webhooks/job-patch/psp.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/admission-webhooks/job-patch/role.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/admission-webhooks/job-patch/rolebinding.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/admission-webhooks/job-patch/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/admission-webhooks/mutatingWebhookConfiguration.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/admission-webhooks/validatingWebhookConfiguration.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/certmanager.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/clusterrole.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/deployment.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/psp-clusterrole.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/psp-clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/psp.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/service.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/_rules.tpl create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/additionalAlertRelabelConfigs.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/additionalAlertmanagerConfigs.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/additionalPrometheusRules.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/additionalScrapeConfigs.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/clusterrole.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/csi-secret.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/extrasecret.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/ingress.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/ingressThanosSidecar.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/ingressperreplica.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/nginx-config.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/podDisruptionBudget.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/podmonitors.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/prometheus.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/psp-clusterrole.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/psp-clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/psp.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/alertmanager.rules.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/config-reloaders.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/etcd.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/general.rules.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/k8s.rules.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kube-apiserver-availability.rules.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kube-apiserver-burnrate.rules.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kube-apiserver-histogram.rules.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kube-apiserver-slos.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kube-prometheus-general.rules.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kube-prometheus-node-recording.rules.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kube-scheduler.rules.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kube-state-metrics.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kubelet.rules.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kubernetes-apps.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kubernetes-resources.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kubernetes-storage.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kubernetes-system-controller-manager.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kubernetes-system-kube-proxy.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kubernetes-system-scheduler.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kubernetes-system.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/node-exporter.rules.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/node-exporter.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/node-network.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/node.rules.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/prometheus-operator.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/prometheus.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/service.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/serviceThanosSidecar.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/serviceThanosSidecarExternal.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/servicemonitorThanosSidecar.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/servicemonitors.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/serviceperreplica.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/clusterrole.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/config-role.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/dashboard-role.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/dashboards/addons/ingress-nginx-dashboard.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/dashboards/rancher/cluster-dashboards.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/dashboards/rancher/default-dashboard.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/dashboards/rancher/k8s-dashboards.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/dashboards/rancher/nodes-dashboards.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/dashboards/rancher/performance-dashboards.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/dashboards/rancher/pods-dashboards.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/dashboards/rancher/workload-dashboards.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/exporters/ingress-nginx/service.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/exporters/ingress-nginx/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/exporters/rancher/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/hardened.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/upgrade/configmap.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/upgrade/job.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/upgrade/rbac.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/thanos-ruler/extrasecret.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/thanos-ruler/ingress.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/thanos-ruler/podDisruptionBudget.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/thanos-ruler/ruler.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/thanos-ruler/service.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/thanos-ruler/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/thanos-ruler/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/102.0.5+up40.1.2/values.yaml diff --git a/assets/fleet-agent/fleet-agent-102.2.5+up0.8.5.tgz b/assets/fleet-agent/fleet-agent-102.2.5+up0.8.5.tgz new file mode 100644 index 0000000000000000000000000000000000000000..09188c8abeb8c2752b969408d57e70387e3a0d59 GIT binary patch literal 3062 zcmVDc zVQyr3R8em|NM&qo0PI|CbKADE&S(CL9eO%xdQIxhFB#6gcgELodeh^$9y{%sGifFR zkt+!`2(SPsS5@`j&j27PlA^67b?wuR_Jb?|yTAg(?mk!oo-jlmkSTKAJDm}!!v#s% zvj?Sqzu!MPIIw^F{eJuJpno)YHaIvM^#=!sqvO%D{$MmZ?mvV6!>Z9uxza>F>;K`t z+JpOx5aC>CqN(62@&IC@2@}&WCR#HLsp!QhHJwmQB9Mf~GnAcdGnypGBk;A%(eGrr z$;TMXQ7S5U1keBXe_#*}j>3M>hyR8_IDBcoKJOIeBt?~x7$b0nfm^p~p){tQB0@#L zD9?ZGtj$oSRI8xk+P|ed5x1(7yBK&=`C)im=spbZQC%WKqA)N+h@fU}9?9sBWrOpf zN3yK?*$-cahhBmzmNe70=(UlAw}g`^N;oksKmgxT{e7l@Xt<6ELyyLWccyiwqFygH z2Cc$EeWMLg3bpDnF%_z3l?l~+>TN1l4Rz2ThQsjizw*o}_0$V?M-NJM|34=zNA>Ul zV2l4BjfTUv|34b-{Qsj6P019E=Qp0vFPz}Mb&1A{OU#6#TF6D^{(E&d0iek=g1Pez z-X6T0A!PZO(HMTXItMWUJwy0MK1Rt=qk@ymH=t0?QG(1lr+jKTn(i&lm4-2bLJc#N z=~4( zvskmg*T6i00B&#**~Y?{i9E@qn9~Gh*i<>^lv7O!qrYL|0RV{1v|}{$&wj}X^KGeh zG&N`sfRhQvI)V#v9nUby8G7E7iLqO^Zka}w)2-51I3YRHg^?|+)dJM!zdOzGqlm z=yt{R7^wx^3VFi>Nfg8)XNiTVg##?$mQYx4(fg)X=vPj%tqg2*s5S1!R5)(Gp2aYZ zvUI=|EHC}9zjnZ;c+-8iW~NgORfHp_Lo-CIvnaKeo_fvh=;5f_|Hpz)=rkqSgCqmC z`2YTB)b{^J!{g!3|33!ZP@Y6^YM=L8k~Nc(1208Q5~7L0Xq!pGvc@f!2!0PNR9o6l zfaS6tF$!qCS{1dvEAXS`Qj71D;4RM8YJc61@jPqEav$kO{&HDLm4&O7!>Fqr68r?} zw*U18@F5()7W*F_9{1b!e>fQJ?Ef*SZto<^RBt};oKJYQP6FKT1yqKyO;?oLT5EnO z(edoONEUK8psa)pXqm5D2nzi#PTrnfU!I(v zRW$|5dM!m7)d~S76j@?6{*BV2{F3NdROua7DX5CqWdN;aE_Nk+z$QB6@bAAcf6)+`C&)DCacBlQ+Lt|i# zq}m-CH#Rz-p>%!swN2+b@0}_aU2&oBTH|*vJ(=?pbs@M}POU{`tA0=@BN+7i{bu>M zQsh}tQPbq9{eDT^@t>nMTiuVDU^HHQv_@w>zYGm*#s5dG_kY8|!El%Vc@zqQpn>^| zCr|k_ym@I;02Ls(a8;LfO>|&_BAf+wwV2PMrg(X=Tvj$$zT^m7Z#ax$l;J!RNh?oI z`Bb7RGTgQV+XZgDzFTBil)iB?_WPOp`qVT2pGDpFpBeC0;gceE5f{5Y-v?~N|NX-8j^Sbty5$G{tQ_fs#$L)#El<8CX@~+SDG-k z?|$XkRFVWY*@FZbY7Cq)`w`eemB%q+g2_INaUvuJwm6n(`>p&&b4O9%9DEuClV}!% zc3#dny}3Q{26yZ(+=SRka8v+(|8oF;J1Jhg_o_jEu-N}DK1~dQ@tZna=8To2*c5HC z)X{gjRe<+?(Sg4YzJ2EAOVb;el44lqYmk}Gl}gh>bnKse7M#B^chH6)cV? z1@$6nq-4$vI1hki)b5Q7r2hPy=k~mz61j}AQ8-6At`&G~%<_49{~zTE<seJr1#@bGtYpEGmI}@|5?N?Zw5DTjqx>|YN_V3QdJ(LI7 zw*NmIwa@<^9t{q5_Wu~vdf!(X#Jz_i{pE6ebbvgrpv67x4wrVE+uUfR&Eq7?^Q{}~ zw!-nWPX2DN*6Ujm_MGkA0;ea2zqB}G6kGhpSO{(AAp48prMv9^kL~!Iw9Wnp{kHud zk4C%r|0vY5|IXNQ|Leb}9>I?O2zA^4oG_XYjSqtWw&MTI^FM>ZF8}u^v|Q2&aD+{8 z_(|FNcFdVh~|L}M?*!lm*puN4~TBEyB z{{P5Qu>S|b^Umfd7p^uayc8tjmMblhq|KxCGUq^eaeYkBh_-JLa!zA|Cu}acypHI) ziv`zGForS&>t&rNRq(TMw1f*iLuoI7ib?&}(^^nfmop1bys2F1dDoiogfPL&o^xeH zbA|4Trt|I-A)N++yAq`_Y1|eJ;kA&E3W<=QCX^{qi17YCnGau=H^tmV``R`9LcO5Q zzSnB&Md&$&DTga33W-2vgF#dMeHiB&?5i+To%i?@WKv8eNmHU}Oc+~)p7*YL8i|N= z!{%;Anr4w}=`Mzqv->^~G2G%v<2F+(XpL$Y_ntAwr$(C)3L2Dwr$(C(MfLlKj)6~dT%{^54(2Nw`+_w z)|zY1x$q+)QGWcpz9@cB>5D1Q8;VJ@O1X2e7%-{O8!57wYbbKC%BrZaN~>B~=-U{% zE6LmPh#6a1|G4PXblhi8Am5mNq{8ri(l>c-SZDE0UyVpxd0bImWYI>?NvL?zpRxj# zDmIFu@PqTK-$cIO?k1p6PasW5C*T)gy(?-nMiK)KpAgb1q448~P`=w-Wl=nO$)23I z^$1D)zH)qC^8P*_930$s|Kj_&J;+Su`}^=QI*#x2_cNBy%kl2+JrQ5FD_rJH#0dXj zjs9^`{J0g*cfJU}=$9;KWuQwg+v}PFX53e{*)q88=j(nGcwh+nQYF)IH_KmwuQOV}vhHhrcEJ__`#yhJ#FspAW@u zbZMnat8FFsAg!!049mz+xjyYNAIUDz3XCBO8ihQ(@Hf`WwLux+M># z&aNelVjc~i_81%@Q7P$UKkfd8?&E(EPg54Yt$jVd>eeztTm}F&{`pG8@ACEf?B;_y z_C}P%o=qx2w8ypA|BZq&G55e5;BIo|NeuXb*Z1}pJI$1WDYrXPd5#L-jQeZ* z>tTN!|MTVK@7K*RzAx;tFXT#+5B0vC>2H*;l33f%LKuF4+v{FP80z!Dc#awq1aWbEgd9~Ubz1wT% zkwHp{RF#kSPvLZ6I13=%f#|sAJ-ATj*_wO`N#|Qr+;LbPVSLp#u{)xO6oIC!7jgsf zJ}P*(JZP8FDWi<(L8PjJ6MT&b7?WyU;o&v<#Pj{2b86$~VQ4i6-;|>@zw#SR8nH*r znp4$ zKGomU$pI@_RJ<&)UrP@^Rb6)yV<>4W_XA4h7#BL7KqFi9*`E`D__HmO zxU-tOGknBMQC}&&vAYHCnZv6aBH8)9ab4ly2yLdPp--EbKmKA5zv{Ti+32_1$0j{% z3$r_Y%?BG%x1pH}5wXiSC+D6p!6wqfYjn3!qxSvb7WgkcW{WF#z%5G6kmYh(LzaxS z1^vk^)piK&N@;N~F$k76p_P&9Q($_N44$K>S#Q89RvGS7l)kdH`PW*iGv^RRmxEXY zs(EhPv6t%(jC|bx}>TE=@5a2?jt;s__tYybtQZr(y zQERHR+?*t=c>Rk)xK`R&p{&>%7Ec(eiz^3KbAh!?49BJ_TkxZ3+J^=8-11gAi4CfY z)v<=?u4xiIe|Ce_k=$=4*KE!H`Bh#*x* z0@0#qkz)o{a}2{@M>x8Kwvb(ImjS1AMZ^NQJ*Td=0@KpZRcB+j*J%xhqJT(sgumX| zTprd(XMI0ETz$DTK9{>|eBM5MKd!!T**-s>zdmhQ+Y1R2tI2z6v0t31exF+E^cSo* z&D?N|Y)BEL5SbU|zFAwfZRYaT`)_aHZXT%Sf?~{wQuQy(>!^3d6id#lPQIWz*t%I1 z@jf{YUysn^^9qdeOjJE$(+BXT7>t?Xt3APN_=fOkaGx69+>5Zk{}6A<7q-l-K5;ox z>6~4BAdeOhtn2}`z_2jMDeDCYyzFGyk^Zv9WV0=NtLUhoi^uB9nGRC&e5#r(uXAuv zt98sMgBE9mG$0J?lfQyz`>{G^-E7LA(ezUM@Ory_aus&uE#nrw*h(eNMlA@!ROed7 z4eQZ{D++*98?*X&8?qJk7QUZbSPk_4@{2n3$dRlKZ&Ln0X_51hEW z(buM{1l$|bFgbNsD%YXgj^8BkIyoklw{V;n4w_46;CeFUvey!KFrnH)#MVw-j0_1% zKFliMtoQE(Iy;D>h`Jhua$>&1!z^6A#N4=eF_Nj_#KqiN(;DfWptsE0neJHw$}fprrZez4@OmBRpQD`OH{@WWrd{*0z>R9bv)nDEV>h zMGNDz+Ly_ml;#qag)+jHV}SbfEIC`zA;|OdF&g@IG3tYBnFxD=JA!paKxxT1f?ct* zu#UF|luXvFGv#qf#X8;*GU8}W$BsGL)%55EU_*k6!`Ss0E2+{)8RDfH@{ov(RG#n3 z5rLv}6DHSa8;r-!DWP_sxm0K~{Vwvhg;&Xm)JO%^;kQ-jq@eIZo@E8AAmq>TvkxLQ zH5y|-gz=bTURM}p8@5>cfJ{`;;qR8z%sIX^U==!gJ~Feeh#dvq#;LJ}ShGFx)|t+) z=Ge;=V>60SwR=HzU#lObI(g(27@@22Bj-n8?L#R^LXTYRB0CAo>7r) z!6CAa);G4E2x-o!F;>>N-)9>>Gux{ePpkQ%UQx*4>BAx0{!$Os=b{7@P)_gW3}H3e zv5iv{?uJ!=1GOELm%G;C)yFNbTem%>T~Jlod~A+o394kb!$Q4w2FV>9Yc-BTFumw^ z&y!g)h0v+xC$ajG*tHJ0D9=P}{b*k<1LnWidDX3}a>=v&lw;vr`7A7M0)B*!b5=7L zYiw1LI&nCVywadVz4snPGU@ry8n6Co^Gm+RsdH{?8t46_@S0-`0%vL`1d1jz@q%be z>RGc>>dDvnIe=iWnxouit~TvE?0?$0mRQ&CG#MN2ZCg_|%rQs!tv`H90Wm!W$2@gY z)JSC78fbF1lvzoJN{GXaDYnz#rsA&iPW`Z!x_ix9l~A@(2!5V{3lqqF>H@tPVMTbP(H1?R~-eC zRHPz6-+&HD3`mePv%lFLgAE8BLb;b-m-|?QEh%H2kM}U~w)myj!hQz!4n%}#P$!t^ z74dnW|71(S1}s}0Bp>FkTM_M%M|Ag|gQbUTzx`r*obxp^H{^Ou}sE%gQ(cnzc#mb!e!V(rp6&dXKSADinL?4{|l!WgeitnU6%`~Vd)|C zA8wh!=!Y4Y$bt=Nbx&X=Br=nO=?jK^b;kCgPc7cMlMJ)@qSQYrBpnCmc#u`Lrw6HJ z)7L)UBL#v~%=>i9sdw4NNp`;>Rh8+I9 zt$*~U?4(8WcFQPr+tOEn!>CclJNx<+%T-xv-qVV*A@*oCfDr2=4A%-z_qqOEJiB8f zT|fUG+e!}IU0#3rbR<1%-YdO1Y?3T8MW#|uR=V%rdQK#5XTiTOI3HIa4vXj-*Hkq!G?M zQm#W-62fT}ja{`8KeAB%G;FrTY2q}d8Xg?u=qQ_)F6JpBB2}1RZxowL&DHMEo9Kus z$0302xH1t*ciEJRt>HK=%p+@BEA%`k?wTs})Ph7M>ns!54uG^fs()`@V0 zwLs7N=ZMaLiRPL2Ozrmr~v^N4qD2!9SqN=dHRo={dMm8b=`7%-Tn3KdtP5K2_Snfz%fb|pGYyJ zk}_!OoRs>l;fL0>+*(aiUT##ea?LT3hi?*7L9)8O=&vT2Kte$QjauKAW4MmY*gnjI zb`ph`TAIs3E>Se{o4;SrUJQk~us*63bTy|^bkXm^iq*3GGptSppz33V$|gG*!n3dm zD}PwGShuRRTbEj)ZfHH+6(c#W0H6n4spxxE2}1)Av^cB0Uo&tNLSOXLG9os^3%W84 z=pZUPWA^m7Ot7A-cVjWfoxQ8dKPKd`SpyF?;)!dqJ{0XBz+HFHKwWp>L0$FiAmDku z?I5ae;IDe#LIb(zTR_C!M0#<<>>!?+<9cDxJM5zIQ9JBlI33+}fz$_4|@*8t_&yAiH@DD!qKy<)h%8kE)^x>wDu8Yb_65%1QWl(p1La^854 zei{Jf$Dx=&1Oi~tKeMT(%Bk=|9qLH8WGyHgXPu# zXON?c0K4h?Fw`;Z>t<-pz<_x*T&V)vX>r@YM2}Q?=m$qR-nN|ORC((bTA(!rlcpHA zf}VVb^IUmC?ELw z7-5dVk`JxQY%xNd0dN(qLWv~u2oe6$44`*t?xm3j*eX5E$lJSLsvamoNVm^ zL{2tWoa&ly{3jdSTYq=MLZ>D*_2dad8<&iauN zq%xuWRV>3`g(y z(=|UjX=K0fOw=c0>x_V&g^Iv#^9nzLO7T;OP2C#=M{G*pB}J~b*Qbr#9WT2g9zQz2 zD`WVZslHQSu@!yYLqoPPNWaG37)ManDs<2&*-CVup!$tqLnrZs6rA85!8drk z6fhSa>u%dyGw%jFLcJ9*A9@BYZzHwYQW32i{EDk$qv0n_vJEhn6 zMFhx>&ZTf_?U;BH+A%Fy`ey(1?S z!x3X5(VwG~kH$$$n-u*y+4&i4{Rzf2bRqL$q4<-8b;9d&(})TF?{z-pM+aI(N2{90 z*6sv}qxX&3HQntqNAC*@iY{QMpCqpzzsV%a-5OfCgq+9(Q`>$j--&3mr<2k(#P|JF zM-o*s$59tM7!jcqeM;cKGHx%eXRePgk#Cl5_-w&LL!r|pZi@m8#YJdZE}hfPw9{Jk z*^TLiJ_=qVpWddd(eZc-BU)$lk-^SAL8O|(fd1k?hf(3wq(6jEAy^7fgE6p%`97h~uYV^LK2X-#c8Ak^k!Q7(MyQ5z32o05jG2)$Yyv^+TAKd$_Masj{cfZrJ2BBxjN+G z*x+u@Bfy@iVS=)5SMTY`g@C-8{xFVAP!YImGa=KOg0HQz_L{0e=dr?#Y3OM{0b>ZP zG`0XYD~PV;>eVD7+Y9irr4SmT<_|^f6_)O8x=Rzw{Mib2?4ew%bi}*VY$*uW;d(nB zkNwu>a#r`VBx-q+my<0Lw_e_WN}GVpQsqIB_xN2yQFP$kjt`pg&Cw!NsJDjWq~u=M znypGQJ;1=ETY2%V32dHX0ARD}qf*R2^uWZQ$pXkxpXdkw?&|Zqm@l$6Z{2x)Q9n*g z@MUD$_7@IhTzyiViEZ|h0_m@(-cgv@UQx4>n|H;_q#rq4h8#L;Fm)1Frcre1L2?Ka zb$Pq5k+H#`jLJIrXkPj>5hd#@1#S60F#d6BH-Mq1Xn~p#+N+p|NjYHHlLF=GwgThv zkAl;B%gw147Q-{1sz6N*k4qmJCSolXmxifXa}HseXgkj|y;YOf0;!$-w#6MovxRW* zl)y(Cw&fg!Kuc_0C0Qd1r4nbed2$sf4L+vMVVgPm{TlY$RTg9>%6;l>J~|E@66p)L z;JweqRc0No*G~km;7jyMXvwk_keX^i)p;^8KFFkwH#tMfwtJ_=Ze+wqsRQ`^>a)UrCDh0hjBx>h}jVIl7+Yba$MrbzYRe z`k= zB(s4Nvk(2jxRf}8mm&Bo)G581seE9dc7En=bDNgl zt{!2947@9@?buwLJ2&4$|KLlCI@Q-1{%axnzgq~Z$Cu)k#ypX)zsr&6qN1X7tjvA_ z*nV6f`4g_(U9X9}M|bUm){bx{w|;*ZX0)inb7TWGhW z9a1PhPIr>S$frpDEvomU)+85IOgS@fcBYPmbH6brq02Xur!fk@npN3Q;RiXBDt`4k z1h7YZ<`t3ecNFYfD8^LGtQw3QnkZMO5BlL)|Ly9xEQdg+Q_s+DNL>^iTBbLxuujDc zn~!wW?as0K(E2sW*6alLnBdzD&SL6%%|8(#`}g!+TM>>4Jjo69<9a)0W!CnW=-2D> z*GqGTVtr`UlCT;z6m%ef!IQ|;8m^#I$g4+tDA^fuTxYj|jB-_KzKYS^oSdCVAvvf&%&9v#&;#p-#dyb<(`90Q!vZ= zGYHG*)Sq+9jzT#e*%1)|b8T^l%a?}|fPghdMR<`Hk4UwL0#P`Fu6u@BeFrIy9~B zckd?i)aZlbwJFG+R!!U7&MH{0oF^C+9QOS1h2{U3Dkc@3@FzdATG&0iCQww*jGBKs zgZupya{FMaLq8-!+GrC8KG`KyDOYY?Kv_fhhc3{Dw;%TEZKK-2iY=&?=MK9p!f?Cd z_n{+ep!kH);9lv%kb@)5;yK4Yi#gZkpUFj(M<|C)Q?>fxapRrKotVo|9yisn=D{x5 zQh01?=7H9LQMhT^ZP~7Xy?LIJhEpRn(e_^PWem&C939I`)vAh(mZ7Dezf|-O7-=jQ zIaV43f=^vLz_xACuJgky{+Cswt6ujp>R*7}IOtFoe8#kB?r`j;LA~gdeLx)ta42f> zlx>w`$eoEio{jS_hB_Sc^(^~4gn~4^{9MD_sn;nx6d-`^irS$OMx)6`ry{ES>2Mjo zy-C{p>JP?(N_CRF492P|pyw#B=QywmlG$MDSWTtf8kTWAsE5D2@}o_yUcdv!ygF`6 zAgv$A%VT`*i%ZHwQt&|`)44gG+EXH8^)|32!#e9B8I#R7S=E^^{)xItD!v*2@ZqG}X~K4jjf%!J`{^MA$b%M|LCl{dmaqe(hf{>HUxNtlN;H+(z)W{3AFb z`$&wT&Bf0OX#lQI!K2#23PQ)UIF1W)^OnDsf?f1P%5GWIyU}rdhf~x{SLNH-_d%|X zUnvF34APqW)*p>bb7`kGv=UdUhCU`ltIMa=<1N_d?OgP%3RXW-u~Ks41Pe0+^e)j| ziqi;|;Ca2CTBMLvAd&MKh0aeNxxeb?^vVxP4+zuTiZRaHXEI&ujwmP?}?l=n16gpzT-|n&#SZKI`(| zZ^&Z;=^m$WpElVNHTPI9KHJwf7=zV^D+@qvYxwUF_get3=coAAU;X$)r9uiaU8L&c zl=~ZE!sS=-j38@T^{WNg(zg|{Y#Bwsi_I}ww5By6oMM|(Mq`Hcf?XR$RP@}{mc9}6 z(g5tjB8l8y$Y^H3Zr|Pcr4`muBqy`StJI)SB8OMpwAG8Da=(SG;4a!i24Clv_db2O z_AzImw@n3E0YJY2NWv|?=1EFSKC4Gz=s|b>mytmH5p=p-5C=SySCy2ueO!$Kh19VZ zc-ykoki~|p_cJx zc{WyoYBn7C0mfo#@LwYlavz)_Q2IlY*#X5hbiBJ45EX>w2^pG! ze-H0ZhZ}MzOv1dK<@vl{Zmtp-NBC4IYBl%&KF;fsrIw`Xl9lGEDB0ss=}`@;VzW*+ zf%E*a_Kbl6IFGY3Rw@@#>#I{tH@P=5dH4^t8I;S@r7t`ND|=#f14b`Px}C)6*YQEAZx zsL_zki5SC9F_C?Gh>KcTl=@OKK%vRBcuJD3wOTXlll(E3$lgqGR_2fo9OPC*GYy7y z6mS~i9DNW@3A%#wAnY4mO}$}+<3>Q5 zL$AW;`j&xbrr+LEwwxYzYK}7NVn$1}jAFCTRAp~=JC_z0P*D|2dJZd7I#aJn9Q7vE zrg)kIU49afcszW5xqol znW9a&Fu3a0&?E0|#CJ&pR(9Vna~Mf36!^3G>7Eke^p+i&nRjfxdEZqcba7+zW1NiO z65mbq6&gpYZM889%d+kgiEIdlh!B}wSXx%dKSbowi7*nCigApK{!%R$1E# z#@k*E^~*W~9ADBnIO9Gb2ij-SeOjUxY|d#&YlB3EzbZAC{8!clV!&IJ|80>_oCE(D z4>|?74PVf}^4ej}y0o zR-yJhn&50Gt{v*8G!ilPEdiWoB(NHo=k83i4f46P!Y}qsGeQTg;5fPt`9+>B5C8EN zRr^Mi)MO!0y5rIZL)Y=j9KZ6U=WQ_*qnQBT9PwZuDZoohFfR771@sl;U>|6FI1jc2 zSGBAkFWMlTl`*j_Yx7G0J%ndRQm15x9&c6uVuRP%n!28Bc!zPQ=jD#d%_YF}xYf;t zlcM^Ayb7WUGX8mWDtT$p4fy~SJfH*mZVdY0Zv^nPgid_Q@*cIT+Rt(C+U(hoEt+UD zTzitLaN8wFgP>S?ZlPDT-FB~ELa#dnZVVqcf5GUEumaE<@KSdMVRa@zo(#ZW#R3WN zQ@>A*72s7lOgb`5fRCWY8$ymh&_VdE>IRYLfYi$dljop8>db=-^0`R|E~6myT6+A} znJ1_@BOIytUL2_)sJTHrSOglaKs>xR>q0!-mrO#q<4?inNNuG@mV?3TH^)K$a;31% zy#|N5{NTlV(?!@q6q%I!m1q}Kb^1d(J4Kq;&IxV&HEZ{+nmb1rsP4p_pIV}m=zWGd z=Sm?WYJgmlW?6G!i(DLXwn1p>Dg;fE;jjUL`W@_wuiuFttoM`G+&L9TzKY~eT4=LK zzp6$)s2j^vb<^$UH@2*fbL zfZVb}&?ma(anlxac7ea`a-}H|Ae$m%Zi9~2!^vjX(u4*7l_=ITFhWbZ&h_F-;~?K& zXUBt$gdJl83b?~fx)u@2sf|HOy7opSk)*+a)^Ats5|1tM0s?OAlDiv2=;sj@#J^b6 z%S#TG%Ja5Gg!CPA%b;rMP|2+^j?nS)=uAi-`%BD0DcC?Uk0jhShT*}Ot~qeNrq?cx zS|~_c{|J}}7+lpIhNrQbzI$`l%PS8o&QPdP@5dve?8igndUGDLC~tU?RuoJ4Kw-9o#tj@5;F zYSOd4*N}Ph8u-3-&s|8Wl>2JyC+p&Daifu(pQrqRIfYg58@m|R2M{7WLzgH8NFHGw zW|jz79J;0ClJE8{H-@EF+K0FLS%XyKHDy2aRO(20y{jf1T10FWAE4_vqyNZyWRE;k zMmUAa_H3$v(IRLrJ0(3qEq!#Ar{dGv)m#m@}ouI&ai{vJWfxzg-C~KZnnRRq0}4w6~E=1GS{(7apGzH zL;J=f7ogb6zG#~~v03~yXIn{XfL}zeGCj=lo)4sL<(x}8f0G^g_exQnX@W02wGUV< zcp?NKO%Yz1=+98ISO7Lp)ZWy-l7m$g>bQ11Fs32VUl(pLIMCk4piB@ zH{b*$VH8Vc2a9!xK*qay9>c`DwWJD==%5hYc0BHeigjpwFIUgN#k&zAUEfj>BGC!$ zxzqge=G`Mis=4PLh;;0e@AAyh&&PR6>?u2F%wuEy3t~2D;&)kc!s_XYpp(sfPY)gp zYK@J?IC*RbKiFycxA3rVAaF^|v8RWKj#$zgdZV(m%;_LjR?7I@Ti3iyKxQ;*Qp{e} ze`{GDbD`2FgG1+7wF_w)39#&n*=zic^Ez9l#HKYd#bNPBE5i1n95Jq^N{Hw|ebv9- z9SZV(Pu;$k|J%wjmK2HxZnH?ecjD%dq}I*kL0HNd-!~HqRZF}|8u~k2#dZX>%uJ86 zYsl(+o|k!KQ!HpmYi^_x@VL*9vM){9yy!`5qtO%)#os;#yvj1cU-w|8d{YJnfift^ z4x`x6mFyjvD8qd$vj`s$`1wi=9X5EPDxho2EcCq56QMAfbh5kOH`GMZ@3s83D zSH{x>w((Wsk1j)kT)0aP&L1j=!kNWf&c4^tu%)(b7+X zHU@x1W~v>ocXQw*c#zvlz$I7dahFTH1Xzko07~_^lT;7v+1WL-LE!3uk(I*1Ap}S|iA)%+-Q4BmY&_=Jg4{4DB(lJ61q;aV;*dEfi1Afv0YnwEQ zFV-qW3Ff@==yhJ_O>+ZrW>u3vq~pib1o7U}(!+Tlxe|({Xt1O7H4&iQQKbij_o^zx zjy*#M+(l=zk#AGsAcP2mA7=jGe}x}&CC-;s6NMUk=7mfsLW2*k-K5$s99Hb|n{jGF z6)0=(ybo4z{SMBucC*|!n!E*Va_8np6>?`$cZQ_4_1{-ei@nfAS9sH?5}V%}WzWJw z6uFv=^j`VL|Xtm^;^+WhX%jB>KlGJtZxpyzkX(M=2n~=Q9~k% z%`qe-2czEY-CBa~-D+~}(zEUFM7gk*uWfd(D1-3SqyhEJq;d-Eq*+qfe;?%g^9 zWIeowecw8E7n3ageZLJUjhKbip;BaKrtBL+Ev?g3J5aM!3;AZ8P}VdjUbN3{j8jvO z{Pw+k3~7G8lMB~*@K&#)!<`sZfY79eJZl}e=G1=ZaT1sk-zup6D2rRI zZK5ExPp%&7t@aw#ym08#$H3yM&zeUZcF5Ky=Qwmrz_`K~O)E0CLW1MBBq=BnYcMKr zIV6g2UQ<2d%(QeVCIHfBm;AN*TbZHq!~k8@qVCbZ^LN7u$@bcd5DDQ~A#5O0fF!$__b+0&DhwggG-}&_iHgUERA?_)jJxKr za9WgGXwNx2)XiRoeauysNh+z|2Z=p7i?vbT0{kdytzz2x>L&T<+|UK=CWiR9*{gOpf@UB)6G1Z+MK$1LKwo+1Wx=mgHHmedPI$-~Fl@<#CFP zhD-|dYPrwOrI3JyACscF_f)#!rr~Sn?NK|TJ~4k!RLZewO^$H@GPHwi9?FnnJ1Y4t z>VJx}PFbhI4TexywI;M~;<|+W!AOzIRw(2hlx|k$Nrzm~O($Xe4wt@(tX`DRj&2~j ztv}CfE9jdR>c8k1Y>r%3rL$X-8}U3CWyW-h1skyOB^07v>sxltsXk{Urx&Q{Uug(H z;s3*f7#}Jp=S3kWtT-49R<6%7Bdf1r_#Rzfz|VzGtGM)l;U+(90xfM-`fS1X8Jrsz z?Vf@zyJ8U{ktdlKicn?_*bGz5FKCHGYHccv*rYMAVs(ENJH1<+3eF$4@S%kU<${*j zLj~pE>P<=^N*wf%pt~?$Xkf!3$IRg(ODPf@q7oD+P6{P|EaC0Q2!ersXLbuMh>o_9 zSf$}id&1YEt<|)aDR-IaNNJZZKjU(6$MyL;-^`jL^?#&mNC$!7oo>Cw6W5 z6JI_YU8s+1Z9S~F?Q4NJYJeSikTpN`fhr>ya$wC9M$$DZ=piy3qWp-XXfwV6H6R?wYdw^aPQi1Vs10IF|erdQb>YO)8L2 zO`Z2(p4eAKdbW|@_(@ZmXJy@jaAjR95Bpz6dU{FLvxRwL{hzpBS^w{Uz%sr>P6Brp zb^w&Zf)#H}M$UKj0J8tXQKg+F3KCh2F{-QKZy$;@hI)Beu6mJf$?S)YF-%S9Jqbao zgBY~bPv$+~Fi(=AD@5PwZEpXW^~8Q@5#vDX-qEKNG>9wtVp^{y zZbQ<7ZDIfaPcUV_if8eo1Cp!ljQl}e;y$D#_Zk4?)O+r~By+f@)UC|unz<&~U6&H( zos_S}gTH(c3GP(4=$|6h?@4>tE^9nV|3#t$4MTv+h`q#GL{dGs$i9+H=N_po!wxHG zD4ODkwf9ZQ!jP8W$RhRQ5@R+rMoH%rnnfEupqQsfp*Cxg+^M{eX_-1sPN9$xz?zl-3_t#_a(siVse44aDJap`EAx2hBYqb&PoPG|J=jz+Ultd&14Z}$wkjRcG z>PHH@hWz5Wcs^c+Nf>6Yq9SMxQYKd7L2WP(T~TAXhBxygw?DeC65NstI@`vesh5ri zqRk4Mb_`s{rOpl%U9%Mezk!>P1jxd&AnzKhbhRhA{80?%m*!T;YSS83PrQk01kOXo zQs@XL!czU%honhdABYT8O@Pj@)FSnFDdt6re;gUp10O}qenCy5q((GvQjXj=>DsFz z=M+huGq}wQPNp_bG|nPRh!&ZZeEaI9b%_Ekvx~JfNZtPs-qm?*gYZ!X?|L39+VW{e zN`{kRQdWC_bV~jcNec%i31>*i>kG1ptW?TCSTkVykLj_nmv31}+#|wM9auTg8yJGZ z5MZsi%YPX!aLCt{tVF?3(O@5ws_ogiOoDDnTuPjm9OkiWp=vCJqY|+TA`uP`>^jIX zNraWopdbr;h6M-x8gun(3iH&uv46U@y=t(+vFF| z*9sHx5K#)IFW@P(Lj?%zVB?qjOE5$gEM-oN{CL>{*PfS>p#8AEeX0-{WiHR(m6oi26MEwBRMarc? z#`(M>x3Q8Xa-xkN+C8P@)w|b7fM+bMOSW)$+42Ks;~BT)^7HP3*}!S)ij2)}z5CK` zYS7*N_=dj>iP3-lGzrG$%3Iu)!mf>48C#cL<5Z{n#ZI!N6Z}@#%%mmz&LP9J0$Ej^ zOwh9DLZw?Kot67jl5#&YJbo+|i$yUSj-*lGSADz(recWVoyLiJ+5$DA2fUcD>GhPi zhDJiJY4;)YRBP95fQ%uUt|P-c0rGSh%kJAy^G-i>1zpbeYj9F*$5F_>r9K?$mV}{n zAWCPEV+`BK4A^m2KTpLnBGvB~qa}J+(&SI1?HM9@1=!Wcg~>mmRt!l2*x@yl!i_|@ zLBc?`@p%hucZpSN^+43yks*d4q&hkJXDBq-ifjPHjAdana+cZ;!$vI<_J^FqXeaWc zpYUb-f*v=`cWMz7Io^7uF1Sf+x_?~_78i%PmoQ^NS z4~x=gx@k=ll=v6BYn%p?A3FmPbbsggJ{Aswq;`n9C0e_(L6a>ZlgA}Jk0g+}SwwCt zI#|Fw;Rx*p?rw#)A~cH==3$aFiB76pb|Gf9OK|3bdMKz)a$Q%^m;a}pmlIXzDkAEJ z(svyK&r^a|>rofdG;WY@Fj&NsT=$g2PG$CUWY(dC*z?=UEBCq$Gws4um(5PimXgN!pH_uAken07a+?Hi!n% zpjm|14U%qu-V_%8tw#j~(kCW3lB&fdIb3up6}0uOissDtlYhm9_@dy0^P}&CeRs^6ug;6SgJ?eqCT3Idj3-#e5T`R2&OdAABIOScF9oCw|6Yi2U};mc z0$a-VuaAC(NQW&B#c3=Goq|9zSs6GRT7=r9UBeq^ua_O9bwWe_6BJaX!a%#!^fr*D zrWv{a=uRN|5JdblMrhr$z|h(_Gz|Wcxu#L;F8<5`!%#?a;6tyhUg*!{FZmKn!-eS& zsxu$&iwmHW@Uc@5H6V%{N3G+J;hES(rn?=y&2==j1i$~t6!f7Duya+GyuL@WsN&{T zmENr&`CxF45)Wjrc+957Mz-D@+q$xrPAJ@yvViQA{M5k{cf*SV52qcEaAD_bX-ETm z(@+1w{P>8;+URUoeVluDR1(=y_(z~^L3Ldi1BdiXZee%iY~#}kW>L>$aEVwK-KcST z7La?suX_e1PoYkG&+%LRwrTe=GbxJT_8L;v=KzjiXr`nLk-Tg%W}&(?IkVaEay}v6 zU;Egy5wTh? zbY{%@luy+$^24yqyBW(`W8G-jNh=rlWYWLq#_kFqfx<`a{e%RU z6|k*@8IDN~wskP4gLuttTc3)qth^@c=Qp%+9C63Nc1VFfj+!zo6x=H|r4I^3M!tzT zOmWITtcVuf9;9ZeCHsHKLCFc%X1E;Nf@SEE_1)-B9h(Ng73Yx|SgJZ$&`B@fomRy0 z12%s$?vWG*|6+sxS{Nj33BH@?79iTW=&gTt*4CMUF`xw5`%uj6Xrne+o)i07*&sH5 z3WU82gL|t&nZrHKdu4BixFsVlfi?|9>3@aU>_~9r2RAq4zaU}*SD+N#90l17Pja1+ zGDAp_muhx9WqxW;D#qsZAG&cN**ARpn_QL99c~Do(8Z2Ek7m#x(cdje*VYXon>xzE8 z+XUOzp6;W&GrZ;E+13(k5sJrGr49hJi5B)Yk~SD6RfPGW`7_k7xm8tF zAGZ~6*!dkt)_d3@>=wH0N0_pIR(CIu$zxosZ1GcN5_?D$U&h|EE^*O0h?BIUn=8dQ z5f*wwaGP{fPJ`k?|CW~Q>~W6$>_ZE%p%Cvi$Dv~9Hw~1PymtyUD`RDGhg=NVJ`b9o z?Kd*EGtX;3DB%p*vrdvJKahjWWY?HtJjpf*!NWbs-cLExIB2A^oMg33aZX(f+a}#k zf*$UMFm*j9f$?I%8rh)G+55q=F zfjE;t+0&@?bgjU?3%bx8PPX%OjQI@sj#P+Gey$>D;1v9`wSp%!Y;j04ydCHa#9i(YI`P8UEbz|GEfe?A%zhtnHihJIV8tmxdl?tght9J)rPs_LzbqZ{8Jo=`7~O!3i) z@`akKpUFj_JJM6@hoIgkKR25VP4XQi^HzWI#H2G&;5fbtn98}_tXdVzRfcj>p;Sqb z;|lx!@p@A`)0(xv6=0aC3y@0LR-C+feH<)7_hdEt8dr@7fAQFV?DpdQaC>rm|8Y2m zt7_o3|JWdl&)d;H(DiBO#wPM|UeYZ^D@)fcr2(N{b!A`aS9MUqYL#sb>&h7z?GMfA zRTuFU>B6eNy4y+=Yhkwg`M9I^ed>Xk^&jtC213!Y6>se9me?6#0%o<)?u~G!W_X8v9qOFZGUCq-x|XjivJs%%z_C1~C1`EClykq;H$?uQ;C>;S z>8Ak{;oPX|iNETpebvw(h7LWxavpIW-=i%SyGD)Un(NZc|i3Pe1>An#*3 zQa_VCgr8IC1jIS61az_b&(b|oD}0zUsh4giL|HuC0Vhb^@}C9ypRDxL3Qc?1hd)>6_HA9Y-~}fxPCdL6EIGMvo}(> zQ;~?gd{UrOA9nLQu3-fT#>yB@X1S}Z!fFzGylHQ$2a4|A%2^8~u-3nyh%<(8AX|Zt z&IxQr|Gs5|mk1VdMW3oKCSoKTWkJ)`PX4{yK zEzHyj7S%Me>=qEz3*dVHqp$@nm!l62EhLHg@oU7qMgeo%i!x=$t1OJ84;=j%S5iMo zO3@StAynj48_!`P_AJoH~%{pJ(3-`?l! zy0>nfI{jmHSFb;-YE?aR&hd=#vha%^&s@~Rb0>A)V+8b4c;QcJn5oS4)7`X_3igQ} z(bbil{amw3LR*BO`pX>zljyac+MIISZCnjT|E#7t;0Ej(r+7$Do%Yg?i0~tc2$kiN z_Ytf*ysPoCaeJsu&3UtT@|*Z;^*7XV2h5vuelJJV2L1{CG!wVHCE1HmB%F~H67e7$ zMby0poWUz8oh*1V-!*b&;Cf{=jZXz&cM6RJscjxk`b;=*bjVFwqI&7ry!k4#aklcy znA9w~SfBDwk7wq=8dwMWx^j%scg$s$pVvT1uD=(I-<{v4=?lrXby@ze$0Xj?C8HSt zH(8Nw>(=c=Lo)=Bo-JM>J<}2bpWTpN0H5pNfzLBrDzq-8c5-ZJ@1rrK&k{nVQ^c?U zyGi|7GV5b^#MeQzbr{K={c8h5`Zp{mpwa{gh#g5$dZ!5gw0Md zgxQ4%CsOM0#+G6+7*c_eH_&l0%Cq)wC#FB#cxdV(dR;gc<^^gNXZPj222#A!M#@>V zxc7X5H!9h32rI7Jw92VpyBEXc7=?G{TjboJZCF@6-KTF8Jy4Usdj+$C^k68QvGWLpn3ffTS=!&kSPE&LjRle_Z>q=1V_r5_!FNt z{oapZj2vfB6B2QX(S0DvJ?^51y*;(6`5Sq`O6-sU-dMtlovn%;r$QjPg!>yYQJ3>hIMDMzHh74p|^c` z9ouC8BF>bKKF%}a_4|ueu}{ZyGyw|up|Tv+d(ANu4bBkRXnZMKuP2f0nUP*iG>7H< ztt9lsTkL4!ThSfX&-*~~sW}hD)h8rNRy>B1LX5T-1X;23te#ZEh0Rur5x!diOT1Il z{VItU2m%K{QqDo6K}%?%R>NRc>S)@u^#0HmJUu(PYJeLtg6@pTN0Bk2*>N5D3FIxf z83@*3eTdJNp0RUS`WobpYftEpnku$3jP4mrhHC$jQ>M+tz%ECtT60KMv^fOhu8K+{ z4)({rYMoy5XQC4wwXHGo6P;dDLTh8TpMJjKU1N^eeGRNLk3v0VoT7Z>V2mL3qly<+ zTk3^aS|}Yd{YCrnIVE3&wb?W=-VU(t=p*NTa*}Kwc;C?AIJsS?>v#%2Z_ZaNjsN#R7hLaRHJPM z^}=HQYAJLb#c1_h13c$JYd-1Xx=LFO{buk1I(pD}Ndm((ysBH7^`rED-$p?eGz3Hc zO6k*&kEQ&sz8J7n6{*Y}O7kvlPSsl@Mh_o{oP`IA*f|B(IKpeEf^K^$q9h9Q9RAJ& zy3Y^Vx;j@%hNX3IGO6WT&f?0jgAi7--hEuO@1Fg8YCNvugt>oPsg+ae{p=B#m6~{g zETD2zTrQU*hw+bgx$ty5^VQmo<|7R-7U*dtZ1PzqPv^JZjC^7FRm}5 zgt@Cl6tfm0=pSc5P9Y+@%wZa-to<&|XQ(r|lW@y8lO|JpR0;B~$mTCB{l^-(U_-cD z6meqH+300TvGAqHn^MwTA6|3*!llyJc>6ttD&2m}d-3<|o5;*G(mdSl&<|{{Ay(-2 zq)PiCdbSoFe^s<#x+du|>ZbGwG*DWn^ zKt;1!gv)0DZjRMBYBdpyTd(Sm`nQhR^ zFT0H@qHR+kgeI%|N-m_B!I)*vXBPm&@_Z=EmOr+S)c8>K&zV@^8|JU@P6r2DP=}BH zAy+K9?-lXCvgX-5va!77#_^IDIFb?EE7nY4u_|_YEm)m@AQUYnfrCuyAe}{7pV+Rh z-bZY#B6IkZJyIL91*mH-zN^{@P>1)Xset3Gr<6l393u3!xJ0S%F(Ec4x(9QjAZ2o| zfCmo@DZQDi=%bp_d@Dvr1U;Dmnd#btV31ZU`@U#ks8#lq;$TsTz^(a?F&gV5X zljui5AuF^`h=hKL_Q?Ei0KRB(F;FJ$N~X+rg(HAmh^G77QJL%w9QtyCY;u2ciNsuV!Q<`Bbf{1cs12y(!p z7W4`wK&MU*nRFQ=m}=7;(vTZo=ZwMsjAf#^k+W*Aa~Q6vU{ldBViPZNf`JvC%VnG7 zxZ@8QCM!>{BD7DdeOka9A04X_i0^+{2`fS+;I_kMGIT1(f2KJfm&*9^E(^1Z@vkB$ z^q8e0*fw_e2w{p06N`x2S$Hu%r@d*)(*2Ef;#+f)fwKKu^RAkoF9AH`D}-tyMHt@! z%6uw7aJDm(B7xu{J4SWOtvGFws~;$JSEL7h3Y0LO#GS+=bN;moJYvT>N&W%Yy^NBw z=ZmTjX?S<{KS1ajavcoQPXvBsQ$r;TWxO*Nz8MhT7`-H>AGDWrV=P$i><6onXOy+&BXxQJ%alf?&i0!A2JmB$tz zL39e?7zle6sh$QvCED%*CO9EW;<&~jHs*$VD!mo%!+G0wmn9pFM?#pxTl3e7FZqp| z^zEU5gZ@}?x6@UWLHnC2q#>HJa`oPSS#e-u&Ur3~L|ya>iy(Z*fTomsHjZ!+^!t)0 z*4C8f8}+2NJ!?)qDJ}c=Hq9H+rm)P5xAi&4k7JU&26VT-IAsGAKH@~TWgJDtuZk$t zB(k~38>8RcWJ9>l_{v%B+H+9vz9fClZv2K(#2hk!yQQL*xV~wUFA&6^v_^y&grr5W z7L3y2>$5p6W#$=QkOM8u7Fn}qcON6#_B+fMWEI1}9u^+rnByDr+X6x$6a5_gT8%yV zmo8`J-2fUQ`px#o4yhCh3&i4xR+1o3OCJ3ge8lxaM!H^#R7g&iwmtR$9r%m5vR{#a-G4XNDh z)NIVP47SY34L3=A923FN?+fp8mQXYzF`+_2@CW$@Mp9V$8q^$ z@cDwRGVw(5^|{!(G2|wAy$7{aD`Z3?<|fk2*_iUs@gm;}#P?Obd!?p$Xhdbh;D_x7 z{L_C!n_!hN^U!+q^6I}Nx3$P7{8F{^_LJ`Dx``pvZLogvjYgd1;G~5|rXVz$9B4Ow&E_laloE z%C?*#nAat~vA;#8_C%8;p2@-LhiZEH@YVi4D|*#^QEWn&Ow(1E>|Na4CYFd5mR;G1mudTwPs zvOL;QOJr3@U=J*`7zNV;7D)UYsTcN>J?otLLU(uy%msJ>$&0 zRk!KLCjr9`!WQFMM0~0=kt_VIIHjl+cz4Kc%mZAmtD>thlKz}E3w_+Cc&zlqSgk~u z>dQT3kR!EZ1YTrrF3Q8xs-Vy1oLa^&H)|&G?VLL#YRkk`a+dn9vZJd=U2`1DNF!=v zmhs}reU<97DngUX85DHU#qGfNf?c&gSG>~df=V5P62XH53L-CXBRj%Y7$+J*q%sW! zPDry=yNb+6S@0`-Y!i`N9wcGt$FR8p@psAd?z|ktjTSl2&>Gm;g~|H$ZSA_M`|KUj zFOH#?-T6-^x#%?+OOa)7N5$Nl@IdT{m;~JN#Ga~QC5(kOfRvQN3arCG_yz)|QFCYS zP$Tv_++~;g9{JxL0|;RJS-QuK)cV)r&-3MskDUMIxQPw3D86$PK@(uJ9vz9gUyoNl z1qETB%0&IJfPN0g?c3~Za}}XOH&l+}%e2V~-OhAaEzg4v#%CK_#=cn6Q&45R3>_|^ zGLp~uIOJ}G=QEvk4)#Os{uzF6;HY(Rr1iA_{2}DT{YVfvxBR9ztu`LAMCv4ndG{*h zuj5YYI`U})m+bg^^Zjw$V7x*-Bb+zofFEWkQ9bM~<_pj|%Od@F#AN~O<3xpF2}9cO zdB0T0Rjr0+(wtH4sWavrgv@-UqD0G1M!D)`kL(6EMV?+HLgBY@`F!|Uy#j>-60%?X zCPfHJ^!w{_l6(buiK6!EnM&0S_&3f{K-M{F3+}kA_pf(Pm=7Y6$J2jxn^P9)nm+p- z9WOp1pFJcjc?Ixz5g7>E`J|IWPnY`y54JlOehlmQ!?diY=pHcd;WC{N!5pq6tyq8fl__eN& z=;qZI_gD7?T!%jgU%3dkj7$UCLmmI~e@woVwz))X!x)}#C&(E5e; zC>CZ(g=sH#YC5C!yvoc%vq-SRPQ71d09yQW?)3 zE-)y){I&4KqDPQrwlM=q)5qCKSF|WV67?PO&866im@E4rXj+0>PCB@}oI8wlT#K)# zfY_CZmPKKDO;SYR+>mQ!_GPrD$`7m#-`{Odff(Bv277Aa;>Xf8_bU`SmyukeMXrXK zDd@(#daTRK>4kW08nNPyk6_OxEh~c0V@8^$%K#u58V>W7?mBNqzA4R{Yw3?Cdm3Uv z?_q29j4=ekscWQVMDTA{t$H<>uPQaDZsolOE8*mjTvm;ry!3oX1eN=wo{jV-Mc--%cy-q?2o8hAb!*=;z`!qPb$2O1J2Lbq?-URzk~a#bKf7BpI&zY+e)v%qZCkhyp^ zRVd%3Hco!%%ddzRwTClnrb#JHiLC59h{gY&p;CYML6COD1;MzKxBFh3{m1=PyWqCa z%oiwxHnWcsO-)@b)qL}`tib7RHxK2d&H^Z>PWA(|duFE^Eti^6p1Z)sQRQ&Unr zbB=Aa{47vLpkcrUIrHwOq4XT)wu>bxD&9u=58=by_V(|2c#VL`1RDyS)nH;#-bd_{ zO;vId95b<3=IntSk>7KVV(VGME@lB})bNy-&CmyhXdgp8f0oOY73x-Kv(0L7ohE;A z(okGQ{~pKD4kv+E*ML`3-Vd2c?XJpD(6mU|!;RHp{~mE-@a>t+D!9e?Pd-H5l`N!l zFP_XK(Fb3Cw(;cUolBoRU1+vI2fBnJT*uhwyl=yLGu4Q$K!IEBnQ^?EIZn$01AtNcsB{${O>$gRm~9XCM53lpT>2ng zM23NO4sYx1+_3wP&-WybTZ?_C9K$Gy5})^qxN`{3x-;|rPC9?6-3Rij+um2#P3*A;)QnJAn$CI<|KJKhD4!VMP?HViQH zdXIi)B^aHw8`)iPj;fy9K5M5TwD5tMZ8H)MVTp=a<_}pgvrX3ZQxs;l-+?K??y=4* z8`l+Hs)ZR1&4P{w`Dt-#Vpg5FGZ8_%{*PpNmjdIe9v(qcUhKr7d%FvHbB?)6dtqLD zv1a~c#OrRh3`LI%aEebyvG&HA&te*AuRNRb6ozaXOgS?wF=~G=6)pe?=LXTbwgfgxO5^3*b*D?_>nTMmb}+8I!a!C z2nW8^hOEpvo#>~+R~n3yAw1V2=as#5ItZ_bO8yRN!{5EO zkJHF1<8rzH3VAxBH=>xSg1q{(1%6($nqai(ej9J~wDgQF#WAuly59y4wM4k>Fv+GI zZVjE1&ARPuB9bithuz4Ad5QNmH{rf#3J6o%`SnRt$S1e606P-&7=M8MQE!1QSPD&B zkybTz$fN0Im&_&C%f_Y~R)$R?x6+dr$k;d=W|gp5hf8MCzEJbNbnj>lkds-ltL_II z$vWD6doO_&g;mG}B5bD0HKi@=VOz;;CP$?=aiC=+APIgr2lPR%ChD6VD$8p`B*_Q* zd9)bVY|J08%|QOwYYXKX{q@>tf4uhQ?g^6MqL}h5oU`)rO*6}djIIl_tFaE7YpAen zVCxTzQp{xQPS_c`(;-?7+;@-;C;Kb;gNZYTbi5Chfy#}FK^`LbO_=tymcmJepib2* zuaUv)NgCzbcKs6BMJw#|W?f6Iq=L-Dcqz4jX@ zr7zU=kJo0jImI>nU4Rs5-Y66ecTYq1)!U@pW@t!~Je^Y(U9}u#ft(ZCz3GZ#KL}OI z0>75|8V_Z^YFN>kIYL<%%NCs2HA*N-G8uR;aUizbKQvv4itC1!SxKP3VMRywca==vhCTc{^uf$-`j>!Q#PTnXa%GWye+z zeN151d=G+m3DQZ@cr!@o+#4Fdq5qtsm)<32EoAIYWBOG(pImXue-|(ag+<-UHk=o( zCbB}tEEz|Q0P2I`76u1Lg7reY)7UwkyL6t~cPZaXJ`#=dgY7ThKZWdhQ|l0?9N~=5 zag3sTQE9flAD4XgBM4J0E(v)$9u)fB1B(b$s8{ZOc>Hzq*nhm^H}^on4EsnvUC7>_ zkB*M+gs63g+|LVfh5Y>fkPeH@{`o{5@^gQ8kGlJLhx&7@s{gPx&VMFm_dLgL5)c3i zf+UiIqdch!O!#a}J1c^p{yVn%n)iN>qReJPiq-R0)wmPiowB|`l1>knSXf-Co!S<) z_u_aq*{l;9_){d+-BjF!YRxi%Btqklu*7rMz`kge8=B=Xc5wN2{<%YtX&%!3P|L7k nH7FN|b(B4R9vmWm=ZJy^py}D_d8UR1l0VCvOy9sgqQdDc zVQyr3R8em|NM&qo0PH<$a~rvl`+0svA9YvW^@+oWUOv`l?-WUPoXECLk~f!1r4(?c zNy5SavjB!NSJD0UD?Da6GaQPtB-59K*eb6Ek4B@>=x#L7;D{mW&e4QO9nMI?zWWNP z+wFGucX#doyWMX6|GnLv-S2w4`#as_5arrK5*Q!P>zA}2$JaRA@|kE#A7h5>-Y zR4A&2oDE<^84fy~uoD^aqdLA)j4F~eb?{yrjNl#NWQ-Ep@P_ISsRE+mA|ea}8kt%X zty4AVctD{ih-F`;P^%6TW1%{Zs%kn8%EK+l*6a4eez^DVS?Y%Qt=Hx{l9T^a!ZK7} zO98Bq|GNj>Ze9NG?d)#k|2h&)$QTE;h3VIiTQ32CCgTB2-J$@nP0ljL&IF^;YyfXZ z?}a{-sF3R*AY! zt<~6I0>~o~V+?kM!$?S&h#L?ijU4QhDNqxUvDi+n)IeZ@ zMCu_D?dJYyg3&eQZ@ES}wYt5xmst3JoS>edgqg_ThHwp90AWnr6g13I!E1yRpi+$J zXl4jTB``8Xrow=ZxsZ6|8lIP$oZ1Ql7!jt>bbBN+gZ4;@#3&+zHxP*+GHvET&3FVi zR8L@-@tEN;rcBI?VFF4MjbX3_zc(`?QOg-n4g`3X3{l$FXtbZ141=FULv=1-ERkrq zAu}@oE$P^(FTw$*2CkkE4Y5GQUucL(rVt1)A=1!IH5b}WevBiMF+174y}T<^$J6VC z|B_*b$H>TR0KI-!neiG9{(2i?=v$*L>~?zn?cRTwuUGDKe>+suXgl=74w;Au3xc5# z+RS=tB#k#Y24Kv@0ALF)CkTakS3+%wV6586OcBOFcxKnqUHYdU5`i7`;_?HO+` zIyvTWS?uq|a)SYE>$$~a7t}qS5xxy{w* zlhMqoAOJ_AV@E(QDrR2jZNYmHqn9dndQuS^(MreM$N*F097l7#a9s#mV_(K1QXR9D z+T)DI*x@3^wnB!H8Fwex+6qk8h0PIi0JnDr;;COprVKQpMm`zJ3x;N)!S@n>$*9DN zT({er{T~8f%kJ6W&wsIlIFnRJs%J-xC}o(y55cGh;pf&2Ck)bhO-Bf4xGyfGAjV-f z&ZVjSIK?Tl0d#{@ij<6r#$%Ixfm%~OR+r+?>WLh{))suQ);wWwboLR_REklRKSDT@ z!fXeW70i)fhFln%TAK1qWQxr!6Dcyq*ho=11G9e=v4MZcXGX))7^Z?{3D&qTBHXMl zzt@VlEr)yk4ns5wC^OP)B9uNOdJ=3!X?AAN90Xh7`x!uMzpGj%!*z00?!#7u0)qc^ z%sGILPSTDYcb>0jDGuP6O3N0r5Aw)LqaaA=SeEwVi|Xx&A>mo-sryv>&)NSPlavvS zs&h3#mYN-?PM>Wnu*(0_-LLz9cJ~i@8~cA9>Grm5Gz;YjEhi?gyrQ}CX4tF+?Yq0c zRHP#ae|B3%Cunm=LZg5!nx&F*J%ZN1sZQ%r0FWsxK3a2pkjD#u2e-GKm%$&M%cZJd zjs7dNsFAr=+*==w6~t;gFjbsDaM>kMYvkySkk?F**r+#S0vj7WnqZtUCuj9UC@ez^ zUUocwIgxE)+p38Ya5ttlT%*xTWml6?d3N(K@NG>7jz$<6h4@}v`2J0{NDVxrWm+e{ zWQ4WqlxajmdEMl2sH(TQtE&X*Tl4HDJzuKJf2TMTl0Ks|XodXm_qzKv`M=jY*!X|e zk}AI5UT{r$JOF3eyd!Clpe8ZVWKei={Rx2JS{f@&26?gLoB{mJn%uX6vvXRFlD##6 z*6l6iYmv)H-aaQHHR>&C(jc9M+1ucXtKUV#{dS z`%HkA-O5$*D{6+eu!I^w%S_?2>ZYt(+ifG4)pw=U7H+TE8bEUsS6ppUH~^OQQ&gz7 ztM1;HHLlhwL70>F28Rl!fQ$5LsAJ0$EB#!9kYSUWb-CfHzyZfg& zDf!1tb^TxR6g-;&utNXu9qjDY_5W^vqyN{E%KD$Asp=F)z}yY~5Gz2X7!9ne&O5rS zWU!Y6yg%Mc7?P$9&7sibzG10HlUN1Dawq^fA|{z4LOIH!rP;$w=V01?IPVYNom`w9 z9-R~=e$(|@ieymA1Q=0d@i~sFxjx^zehTF0qJE*n*QX~Zmse-!AI=Wn9A2JW9iRO6 zt@}iE9(YEkQCNvhB|9&SN+}#y0YFqe*GlX*ODym1P zA1^LX&aY11|M$m}j|Sn>!#_1*_dxLZesVc}?)ahsqu*U#o*CNx&;1D8+y3Gy0N#Ii zKx=>8@nvJq{f_rc#M#HIv$OLLudj~Zo-ZOW_cV6CQnzO(=Wjn8KY%fFprP}hei-{sObk4>lI~lqoFdDU33Eej>QW&$_G!>;c)Cq*Kapj3+80 z%pDH+)025ki6s8O+3Z{h{hBg#+*h1k<|)l##fu;!l9C~1RNG5fP)e~BX*H(}ho`6a z(bk=4yt4=8<)&>&Ox{^>ictaU*t(qi0J+it)E|v3g0AK-;O($hi~78Zs69?NOB2o6?gY4^vaI+M(oNJRv@=vSJaP6 zi<(Vmqx0?WLg`!xmuI#?5Y}YXQV}9EeyH9HZp1=O7}%nZ3grNL-EOy<|3-=|^#vtV z9$^31_kW+n@G5dAj91=*D__TX)E&W9=YM%Z=HRE$a_`7meAKw(OkBa$HgP)nPwXNj{f00~nI%T8u`g zG@*I`y)P8YYfE+cFNY+0?t9=X+lPb9!%*yXKP565p|A1crP)1CQ4UKP3vg;- zGLHpg?-kZs7QQ=WKfUxOPI{|H0(ECh4q|cA{t#eGr)IH_T&)V`Q7>f-DUT^1yZm{L zT2T#|1zrDTeDYfd_(AL<7<$)gKK^@p3jh|Ga=C*#8-Bt_TW)mCE-DIndCSV3-`m^p z+=WbqrDM29(dem*sENV8h^ZnWWK8+}>}1FKOkA-ZcKBL+P~W09dbggoRvti=lCNwT zV>qGrAW68Um0d>q>#gxpFOQz(iYs#Uy8nM_)jd4b?Z2v?dmc-0h5gsv>DBN5?)P{1 zHum2-Qse!v+FE&lsD6}ee0+bURr+_o0>5o?>;L@Z@c7+Hn8aVDk5%_S4|eMDzrFpP z&Hewiq%9v||2t#}*k?e3z~xX?3bt1fVHiSD65~9$D6zr$kYr5Ll;Qjl0~`AD#|g15 zY}5Y?f|t1}2jpYQaWDr;=H4w2)COS^9+v``e6tD$Vl+YtnL?=`5jKFxN(%6SxYb}g z11}O>X~H!mf>SMQtgD_#k&P#Q)cIgXBC@eTTxbxS&2pIT?9Il@S>kNNFw;Oa5T*o- z=@ga6h`@x7ClKRUq74Dgu``HFsiJd89vi^VB+_gKH=cOj+lg`Bdmfl)=vj(aol`9* zYeaZHL30|;V_^+ClH8P|%>H!v?sVJngL_V48~%_fEfV@GZi6uNn?WR!q2D>=q0D+B z5kok~F_AGtrSbx2hHl&VzrrZBabEF^lY~Zuu^H%0avNi*MYVH7C7VGlL1Q0joAW^t z-ByE7tAv6>GZ7M1#`LzI7(*N6pG?ywQ?zXvg({6anmJTR7E_&1z8fKxHJ*QK8poLp z0Sp2gJ84JbG4+eV%q1yPzQFaZ=c2s25{bpiMo*UB-Y}$4+0egwVR}M{MU>e`RqWiD zo{?)w82!~!*AnH=v54RXhl*Dw^yWO0F~cJx7ze;SNRLdcB2hrwD(YHlRZQC6pxc5|C9B{jRL#Up=oa6rL6yMJSxeV0|`JwXPHLE

*=pWWS^js9OpTJhG;tLke!M}-CM0>@>>;gc9D7!u`_lhG;mjRO@_ zUjCU@p?>ROZ<0?<>;Ji516p82L&F7ocoSNkSbH4_$(*N79%m0Jj{>J~imh_MI{=>JX=ighpfBl!P z4m`meTxtK+-+%0Pclw+6f7gPi~d~NSSFQ|X?J0-Wv_}l%cF8e?TTNR{iKVW zDGsZBi9Nw?hPfPED&i&JjP>_o{yE0-*Ns8)G5z@~KF>IU`JIk023o&owX_poQ?(Xy zI!Y6r`GQ*4t(UED&A*%UwDe&4@9&3wh5P^A-ManP@9ppJZ{+_v($~2Ci17M>oj-RFVD3qA19XFuM*{-5VHh!yLneJmtQ~Tt=y9WW;FgcxmoJXK^7saTi9a$B&Tz_v?Rm`TwaWfvfEQ{a*e2|DeCOdH-oG>3L#S z9_ILIh&Ng8kZ`QG%cF&;R}F-mq%y0YA14x`pj3fRjb>HiuM1k z>GhA69<=_S7!9;){dW)Q`~P0Ix4Hkfmh`xIoNuMAHtD%j)A|o5$P)LK^Rx7^YW?pm z{Qm1+e|O{mUrXA8Gom$0_xl!Z(&1);9EKTX##V-uMAu}D_IJl@EQ88YtN1}pkTDoD zF?2pZ%E#Lvkr7R&Xn*>qluvjZYyroyi%j@FwQopc?DpgT@d~^XZl`06wRCnyZ4*eh;{wh26-#lY7?wEh_Pc`M8IiMkl zuCvts8mS6ihUz8_UWP+*9lQ*6V*VDAj)Rx~8*IVPL{gC{czb-Jf-sfB82CX*VW0RR6lo=j>0egFV8iwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PKBhbKEwTXg>Q_ptN_&_N==lIgWEzRjFM?vhQfu^0*}1soL38 z7EPkN5wQUdK(^e;i zTqWP^@;N#>I(q%;mHc~jbkzKN^xe_RZ;oHRe);<7`)^+#egDnT@weZ7|J^s>Xs1{> zo}6cpeRH%mtY+sP$pZV3KfFK zfN*^Da{6j|{I7YcK;8EUZuivIe}-0R0yD%9e_5g=MT}3gbbDUVv;M#R{`hrc{l9*B z^fIjfyLf*8{qWCVjaMgtqYTV(g2dGSVI{~CEzt@1^Wh(VJQ1ayy!nzs63ay-Rt%tX z`F|vZ#uHs`0%A1B1c8HceTl*cU{ZaCd6I#t5;pHeOViIV$q|<&|GGwu;TVB`fQ;oN z0`R|uXl1TzF+31glo$p-vAhl%81Vm(XLr%!%Ds&!dD z6EH+KIGCXb<{SY|R|x!9K0}P43~|-rIYvp$0b~dyc!e_*gN%Y~i8**S+el)ZzrPk` zF6(n$h z*u@6}<)Rk??Pr)oNgmrfIep&3c9~`A$>HIj_+$p78${xhfBWv}=)h=nJklIR3}pwP z0rL0%Ie>Y#%v}i5$sX%UtlGT(i4<&n$ z<;}_b#~+jGjTYYEAAq4Wfe``+|9SupevyOMMnKKNcrx3lW{bIgjo2DT$X*Ex9e_lr zUPa?#DPM~F3eD3}%>dmqPwF(k=r#OHF{;&s#q)9sED}0{Nul+qn&+xwF)b`>{P18@ z11RAY$2e#&paw^+dQ^1bmz*WmZgee?PV|RVG|&bmgK-jh+8mr#uMYqGr0AJgAI0Pa zTHyqxo@!epy3j9WAuBz~uPsL1U!pcjwV5i*caECB{CDVo5sUfG48czQ@71g0*RPHn z`rmio1^w@C9@CV9DMnv1M8q#Xz4^b25gtFe!6ZHbXF1R4>I!k1vk0Aw^?;>iv_ct- zVFrc$*f1vnh$zVzO%lW=3q&N7o#i-*5vy_8_0jbE>5;Glg${=)`G{ADXKshpz0r= zmQ7>&MCfv&KMO|lv@#XNC)JGZt)kSAs>KV@;yKLVwQNygf_e5|9fjUup2>n~k~5e% zv6E%^lCtc51$_dNfmiP$(dPHQ469D5_jxkI=2=0y8 z8BOvPDdE-3?2p12$Qcu)=b2Cpw1tq;801`O%PFI4EM=(pqEQ19H)(x$Etva)_gq# zB%v{K_5Btn378>}GL%AwVnB%lBhkvmc@3NQH1_H`<~Q5pAzJzLse#FP{iOvPqnUzx zOj|3z*5>J=-nqCKX!Uu0JVO~gRx>|Zq7^*R#c7Jj>4%HY-(J^VgE~d*s}W!hZkLE5 z@UBo2ct#oG4WgOTh|YdRQ8q2Xq>QGBWw=M_`^@wS1~Q-QbEIFx3V>^j9LVwCXfNS?g>o5 zT4*jW03#6~Hx?BN2PfH_RKewEM4 zd01ee@d3a8F;09ctEM#IV<6aQDMky?iDC=Wb};BX$8n93b~EN6(7gq^rOO=u=I*QU zQA4d>1O?)L6^NHXfd~pjP#_+q0x@7YUZ*IkErb|xEZ9>`x|K`SxkhVn?dwxK(Go>B zJYO~6J1HQ$f%%Q(Y`&EQ!}O{oIQ2&c37;dzQ0%HZqclN~G`=eN;RP>+46aaySc_1g zrYX6|+0(pps)(l&`0Z8^u?^7##PpUV6vmcMdFuFpzND086R}t)YoY8ROCg4&5b&X$ zP~$v^?OHSzb$Q6Lk=1Goht8CW{E$|TsYH5J#Hm;D72;gj0zP%C-QKIMu7raDQu-C7 z>6mkur-TF-enBq%JtV|m8c=24!y|11* z@804BT|$dDcj)YM_e?!nF*|d*r%ikvlNfZlkO$V6f_rRbw_n+8z)o%Ta71Fn^b9+V zIeZujzF2=>^*z@FNJI0GxMG0Sq8T<^J$9V~Vm zpZ{IOX2!%>Li`}a52UcO2M(0ZUkN6jF_@vn<^=qNW=l$MFiBf@2n=EIR@J_TABO z(TlRw6~(ivgt3azM5*2v0!LMnCbN%VJDUX};# zS|4ig2SYy?`oYj|6b`=hEwbY+>{U6@x6OPn(sAdmBxf<44(5Ib`Q-o;fMB$gP7Egz zwoNwIR)>P9NudyVCYJC_@Ikha%NUXhx#v!V3^S&N*XP#z$L9JDA~1-+AObtcyf0t( zHcsvuggs<(-&?Bl-t#}d+GOipIN!JRx+`G&o}6!SqA}g8(>#G(IsW8k16&B}dvl)e z!2i948+cC+RxzIMAkVJQ{50EYIQNwGJ<2n?<4keST@*b|_uA_J+)$L|I(`9empEDi zi>jP1HylU0!W8_ZH%$J97zH`WZ~~-72A-kmVk(+R5P%IXfw|_5m@J;}j7-6`eLt>k z*=nz^I#2sO*27(|dD`QEtrxJmVH+5!p|?3>t=`yr?R)geD)V&KBu7E}7M;cz?|a=B zUI(RUtN8;EQ<^VP3^Lr)`8-vpXvU{mtN-n9qw(TKBBO;ATPHpWodfU@hH?+;a16WYH1F%;7kp`e0HL;TX>^9 zC6>;uC{MjC{SAq()C>}QvPVZ2zy&iFt6k4Kz;Ys6k+fuJMKim^Xroo=w_>7B3?5%S zgNzmXc59B9mUTUSy5MTDon0vn55-`%k@Z2gWHeta!Rdz!KpFUSF$IEFVhTBMv2C?y zx=J07iHYLQ+!u}czoiqyYYCW&*MMo%R_R>x);hioO~)Hxhj!(Khud}2-bkX0S_4vk zKxK z85|tE00$T3gQkl3B?Lr*G8Bs+Z^V=o|9nz8B?n%Em9u0lzMrn%d4LLD&aWteH&3^B zgj$`S4u@xllhf+r+>cw66PWTP&AL6BMXC2yA?;FYe;AY&RY3K0X6kC5PR$Zp$&EU_ zc|~!%5A)jbb*LBf4*ZrH%-y>$w@U-g({7h#a8d>*WpGlq3x~|#AnJmsdt{NiAnJms z3!?6B;&CHla1TD}Rxm96xjW%93IiChaI14PH(@^ z6aZ>7lbMS%kocwdni;p!!n>MxLZsz)qqzo$M$N#W)1_0=PC_h#jh%?sPvt1 zF3`Q43zVlHCHd80H6pX>lOsN>hfNKOElR7GVIxeN72x)?2c=Y?g+Q?QIa3trJs7HY zRVjUdMfuyqzp5cd42ju~VFEac7|I;*t8&+7F@?Zc`EfKJDNPdPPh6L+G;t7RF6u=p zPp=yO!4B;U65~i1>bFajEu|~2i4x`jt|69cxlrQ?wGl@-q053byiOZ$+u6Lmba{Lc z-;X8rI-TLVf59gneB!|;-Y6Wxzd?8g;rSSa=e9BAAUuQc48pUQ%)1XLtA#fztPHZs z9PM6-pp5O6vS_$pvyyUtfxL$7@dnrua(>Q#L+lIyT?nx%1{Eba&kzFyp;)quGJagf zqI!Z*_Y}>(hoIPYL2bPRMuxWiRzWH9dq`d?0RU^Ag9AKw65FPBxF&@C{%XS$+Kt&2 ziYSY@Mg?*q0|~fYZtTiXPD;Gm2;mH^MjxT|fzcW2IykF__Pa95dy7Q#Goc!(Freyp z8#|L|9mi)0T!x+GHSUqLcq2*z?= z75N8xN(z}BqYUDN1306();p^XjK)#Tt3fD!BKYq@W1<92ZYhgjRMNYco^#6TG=9o= zhN+FkI55==2CJo`2UPQvY8Dn@yN^ut>a8#%UjQAN8;xaVJ$lfBk3JgW1Hh$w! z*#3|=$Qax-zHOiAVI%=OyhM+?PjJv|+bho@MEJg{2%ie`Zohr-b9Me`GgPHMIYLz$ zzM2uXP}N#}YJ{rF<4*S%syj~1z%V2bp>Bnqi_fm1<@5m*WHI{g6ImOD~PUMGVeZ2 zqZZz{FpcFZ2)BQAuaNWI(XsVBm40sXVJn+$dpK=Y!_7QHyp@`mxKYHfXZ<$UX zc^)+BP+`1aiuHIceLvnnX9(69Lcrejwz@*)8RTu1BC{Z2rEGbU z&{X)wg7u!D0_-J?@m$6^Tv46Axhk{Tm6o?M8go!#pRVMcZHd%fGj>K|qID!4u>R-V z4C&elX>5b6lkkymh1B!Fb4R4dA&?ae|9yeVwmwTB=L5KgipI$*AQE*XyAU!xT_2|F z2+Xj&kiQBIxzv~tBl+56Bx-SYTs2ephjhgDt@aKTUY@W=YL9!Q-pIkR-P~h*X@#B4 zAys2yi8`U?RXs+n-3OhnO`}4~6Z3Y)eVjh^t|7l%@J3zLUW4{NU(FB`-78rc$a&#e z7}Q;c=4V11ui%#5RGbs{W zdcma^TzbvIAsW>$)6VEBg&CgV1ZNvBW{XLr>ozvZT(m#*?v)9mHHg+bxd}gr)*xDg zXub1jnX&9VgJl-()U^)9gYB1dtuT3q$U-oq+pspXkx?xO zA^TPaN)x(SA(CCf zbld2n9lO0m{5rZ-n09_+<~W72>sZ0=sS)C1cNHJoZxn2sJwi$qRgfA{8b^SpBQ~SH zG(t1|vA8|#6pC`jq*>PXGC3U{y3Y01ZYCMpFY`4DeVv0cEZ6iki3%6zHJI;d>;n=L zoECSu7-$h$I%3=+fl3uSikfFD+lh{+m#e!4rd8^%P7fEa}|T0;X+lsYs+zx z^3HRJ=1!blbe7wUVzjn=!aU;G@}8_$bIWV(#2SdHBaGR#orEsRJ%YX-S)mkpx-HFH zjdtLTbvXY!ysfP0tlElng`~*$?1gSZsJ_=ygm~6pVOfmCRb=h*C>8NM$jlv7YxQI zzPkTu*L4X{m;b%vTAz_b-Y(@}Y_`EbQ9d)SrW`*=eUW?x#d>mW5Alk}aoOX*+V0!r z%m(ahPd$Ct!wp2F`~I_`jo4LzLAAG!(OT<92`#4T{$|m3b3hmoSGyTS%+C@^Fj;iB z;EG&=oy^Fub}EHbq8@}oDpB(ZsYD@_s8QFNN@TcG{qibR#k*2#w_u*yW4~J?!HffV z(XH^;d8RD2T=0JNEyL1PoWJ>!Lb*%b&)5+xK~@!}62d+_K{vl_J`&I)Vo(9KMX4S(Uyq%f@SE|Qh zDzvA9r0^w;(L)rM8ojzUzCT18cY)Yp%b;R$Nr=?ME5@fiETtmuMf@+?H$-Cx~q0VGztwcVs+oG&P-jbOJs=2CJhIZ@!=? z|7wB~Maz3!rqL27F+)zD!B9A6U%F;+?--j6Ic5o{PJbM|2wcZ1s%K2EJAmz~(_1m1 z6bf!Sx;j(gu<^Xhes0@9=s8Dh(2%<^*cam|)~LVb%QMLQd6ml1y!hY+YM0?nD=|Eh zOT$K+JKbngxRLa?ZFbz)l0K`gH;u2YJvdE7)EuyE#=$G~soagzsCh5ja%wT=W>T%5 zynD;}79yAfmJ=c@r`@GAKe#xIL%nbF&32FRhEZ=eZ+@GBF9<9D+JK^O#K5QO)(!KJbMJa@V{ zSa;WY1U;KSy0Qzrb4^BM`Pm@fn-%+#FW##OIMC3sXGy zNFNuY?yHwF`ek!E9&S;8+*wFtvv~^os%5Z8=X`tp53dUb?lr5P9qu}-o&mn`Zl2=V zRF{i=I=}Qr?B9wE)iqpvAL!>fq%RNmzTB0~lFj|8Dx&4mZ3@)4@3Fk?P13OX_u^poZ`W&+z9|sIn*AO+%20Q!kNjKz| z0%;Fab!FIEN$n(ZCwX0j9h)~R8KifztPhXOGx|o3o9_{6;VjQSgu+>#%_p4Y8P4)- z)U^pAp^xnQET{fVfgUkCBpqQOLsQRg*qV9iM54Ji zBHh}M)PfMhG{%v{$9^6*FvdQH8JY;~!map@xH;%IYDu26>97T&l{eXZ4#3Zk_?!xg@N0 zQP<}F3;T~Ai}l)XjxrSGOpMpBbheFAdR4UZpLEtRP0dd&1GNUUX-hlMJk)H7oLpZTiKEWTDo+6)2hh&oVL&l~{Cz(L`aJ8(eHAit>_ z69Y zhRY69rJGYNMG>BFw8<}+k)s2(j|#>+^HrzcW&A!QlPS5qFZFpDoG`%&6Pz%O!eKvB z&?hQ=qSPT?RR2%rltpNw?j)*xh!O-zp4Q&QC_$>2)=kR)>g`CCrAh#pWDKu@<`OiQ zN20lmj~E5bC1@@|bGe7`k-6|ZgGja2ScA~%z%TgVfGCYoT0)06#SAF~g_aYp7>(oL zZ#F2WNV0m4GBGW^fEopYW8@+!`rTJ)>rI?b^30^^jCl z@)|m&K@5T;3WA8{B=hW^eNqZ_&bxo2QWdd$7&*F7N@i9FrUzN zjUDDUelSIi2F!EpCW{O2>t_YTbIeebQMP&Sdrnr@ zWil?e==#|9vVFNm7g#y&{x_N`IqY5KgY}5Gj!H@ya}0*pgpktJb1o?8-ebAtQEhXG z6Z;nKT8H22VuHhRl&K^bSl%LEh|Rg&AwgO5yH;~C2e~|JR|uz^BT=HLNg04-qc<{4 z4S@PAT%D<3>>?46a)tnnq;eALteys*FANBng9P88WFvqt;aZ*oJkJyGEL+liu>>jQ z9M6)C;7bX5t~XqrmJQtEY*_&iR9NL)vUVnYmzfd|5D-cximqkdXOsb)$>V!x5^xN# zAP|SiTJAHM1DGV$L6wjpJv7g!wN6C|f`WI>&|C-(sA0!~=(;hZf)HObr2_~F28rI{ zz?AztR%4T~jl_oNae7+nLG{f(moNo06hSd^xIPt(EGc6z9gKt_mY{f{hc8+}vha&} zEg_4Arhht@j%7<=wyQFNNw(bdPVc`I<%&+pDTZ0GZ>gF^GZfJkQb$@J-ZGq_8tDL> z3sM(Kt{4?PKvNTleNM}rIy{de#JQN5<Q#L`!mhth$UF^Sv}Ke zX?d19S2<^%ay<#&QU=f$xJnabPASRuYGB0R!$cnHE+&CqWYVO-MDg_n#AFKoLT?4O zFB*ee>%t3zFGh5g%Bh!839*`QVm6~B1t~*9y9KcrZbtLUVv})HvE(oToXY9KhyVk* zuw_~uc;vG;G08JvGPXFJ7=w)`cPpo7+1iVC#6tAnyRS;6!YvYT_vZ_btwlPk2MYZSjBBWR?i znzxOks_nFAh*=SZ!1T#}APk_lx9#AIS ztkaOVq5;CMeY-N;)cFfdU!!j{GwapU~ zzS~AwHH{Rb+I~!tn#Wp#1FKqQaXTfL?>Gm%232Nd-xBHcablnyBQ+elQ^FsfX=Kcn z4y~X@nxGp5-5}@&z$u5dSPgrQ0;7LGmS14>fzby>fA?EZ%!TK<55>kFyH+A_=ajZl zsu-Z$hbI%l=d?&1CJQl#YUNTIx0>B~a!ZX`-We`q2v$tu9-6&>du`5ls1ov^%#>Ey zeKK<^Hcq`M(5oHEIDmFO$I5JdmrQ+(3zp!6EVoDFJtY4KuptinwRl8|6HT}AhM4DN z8J-aRy6puiV7fgZ1g1?NNCjjcp;(mxt94YdE{oeTXGrv3$We=Wnxbf0V=p=^VZQ2B zkc+Xd@^K}Xn_wdE%&btk6$%x+LAKrMdYiT}e`*+=wLkRXby03n!OaUqReHGGVN&d6 zlAKMhGJU!nfbO$!6|%C8kMt9-qk7eR-OeGIv@$Y_lYv#9H^HlX1vvgq@;g1Yg-2JR zy4=J}@I$Rk86)0-uk7D7{;i2{^ZgK;2u4{BlM+djz-~*d3A!8R!!v@686-Na3YUq7 zC0_uiA1;E`6Re(K^^CT9^it}#cuM8{+IH$?M}E3IcCx6{qkn4NYZ^BM-MSCj>`EKF zhx1{%dG>p&&iL!Ym1OFnT{ciXnSzgUoyZJX%mH4l@=WHR*LS2&!IhST!`1zEG!&?a zXxp?JZFv?mc8JwEdeSaYbi-}O8+2toFL4qxWXY@dn{xK)yv_VEg}GloHe=Lsy8=f3V-aBqD=ZHzu!jJsHd+@b* z5ArmnESu#dPS92^1l7=wvZ1CQ!Ti>CCe%=HdK6|=%Z5+Eta}%U6$&0Okl9bb4Rf9& z@agKE!)GDCSD*D!ZsO%YAg5r)Fq)e$i%w#rnbvuFRq-0=^Va)EVa}Qpvm>Zi8}%mJ z3+8PwZ-aU3Yu=hG&oipm7B$^Vu?f7}S{uk1d#NR!f=dqw?lc*G^$gOjt>9g!Fto4u zXK0QXlZRbZu`VMDrlVvYr$@L9&b9zC8yaDC)+Um7z;n!83LE?T5DFXnn@`x-|If6s z-@S8xjuT|RXsQFA%!t^r;XfqJx zISyWc19b$jx|}rDCrq|)B^~Rq8SMfw$?fj}@uzrjfb=0093agnI6(fH93bOQHY`-y zA#MV<%G~Z#;VBWacw%VPgZjI`*%@7}@F1+9;I$=tEu0q%oPkKT(Z+4Ohu>7P2jI4i zqCbYD<2zQki-vlsB++ZmDX){RN9-3Wu3fYGjN03TOfl<)+Th(mZFAKN&e~?sbP&5~ zjqh$+2fx-=<=4`gaN9e!^fnEvXG?!`xV8!u+`g^mjor0+dv$JU7`t{bJ-l2SbDVm; zF$GnEJ0`eef;*;DG7fad6r0?}_+w1Y+r1PeI)=EWz1yeN zT~p#c00mR`QQd~1!bNsVw&(T%Wg%2H3eOnK&|-4}enPV)r8n~Jt@^(hL8Z~2Q8LGi zYBCBiII7J~gD|bo<$`n}NQVy0R^+-^YXf1IBS3m197?YxL(w z+#$Q~y+u8TZET-n!ssmL89|&61X)$6!M$p}9+ck_Em52&C^nR{VOUg5=xho{=8QNl@;% z!F2{nJhLR)b{&bM&icKivBj~5i0w?L31GL>`u2;DGj9f?8${x3!D-GJUL(uNNnW?t zRyuMX;|+lr5>{%pMmZ2_m+VW82G@zF^x1@YgRavjytzyQd3Gw*QWXE_u5L2UI;w$$Q6P6(i2K=&5pd>w@H~8Df8(iy4a}=(!)5&gTgxNLRj6ArDLz zJ!|E!j%q?lD^7ZjQHIz`n8f;MXbnr~VxkscRkb;SLdg-;5s{n|PzH$Q98K2(zzafc zr&V_HVhzh@a5eI$c}9PP2_!a#yHgw!1iJ)AmpJL~#<^hObe^4(44=+rpsa6alOA)) z{Q;-*3_y}$0CnxU;VYEwtQ^ve0+=N9wt&v{+2EB~uaj~(Tpt!t1wsNPSw zkwn`yy{JJi!FC=jz5bgLG?He&$!wZEPBa1FArek{EtH*gg`%8^LjOr;yWlswMsKRPS7GP62|pn{{_qida)mzg_Z(*MLWn-*rNHl0oJ_!6@Njvq@R&Ya!Y`7vexlu| zJ^qAPSK>-i$o`Lbg=p@HT259ly@c%Mn!xmI2{CaV&wIb}er(pJ_KtX`YkE0I)a4Mn zKK(aJ1gmx=$#rLWa_2ekokZ4^rSB-9jd8A3`A$w)uoi*pLa^ zsB7JXtb%U-URqu8vd^c{x*o!e7GiD<7ivd|87}m}X@q+PeMuovX?wN}6`)6HEF5Ly zR8GuC#TcqQg>hn13V_dNS7>hAP~AhYvH=>8Z=g!gwAQXoothix!{@WZGC5Jqoh24l z%Eslb8GuW=es4Xia8Om(01T=ejRt&6*-b)W?3@6gn~F3({7E-4kcdug0}RkOWEmQ} zyE~}UWjkG+_Gqaf;!e;i8vEEGEe1B$$?riw&j+1+zq5U@ut$Uos%kcV&{OdoEf|dL zhdXqgX5jDukK^N`E?PlE030O$s?d(t4h@Fng|SyE= z^1@i&`1;0q*=}FWYtPd~!+E3+WHZa%pWC3+PnDmxl6g7fk8V&-gVVAt1--FJqx$UF zu3_M!W=Lb>hTggX)yKx(4QK-I+r`nJV+y(CS1nRz|S(&K(l zD=Y0^qH>v*l|JI=24z+YVdzbOLyVdQB1ri_+!}x5+7&J)3yF}Rm1flATv#Gq8dO}IEae15ieDlTA<)beiY|#p>J^#2wAeY|8a!)lj z+4K~ADzY~4e2$o)h?pAXFQaVf_>;j?9e(ZUMyJ8$O0jYFq~XajrA^5Xx{SpEIHi!m zm9W`6?A+mVc`(>!noX+OH$YXGY2R>-06ksqFlxIUtwa5^ZU~*9U02-N#$HT1lVcT0 z{d!LKFtDDk*pGDqu4`<&n6%YN(#@v5F16hur&fUNlXsU4yDOY#rO*|MD2o9k8+92i zxLqOwGPY4JYB0}}IZl$ntDw`1Hmnco85F~_c4>4huXqM=5#SZu>p^s=Cu#%4@gP4l=5*LTn12+)|e_DgMC{axg=PQ~-LgbX`&pI#nq3Obj)^S)?10 z@v5fdljU{Gg=}*o+ug`_XrO^bad4G zdvtt!^v&_B*DqfmegEz2qwl{tI{xDc zVQyr3R8em|NM&qo0POv1cN;g7IE>HRKR*SQ&irG0)zsDYEccf^k0Qy5Hoovka`NmZ zb7HU?BoQ@%9sn(g@%Z=IpM%1s(KoUwQF3NR{lXTzfkL5BC{z^+g_N`Dj0?7XG{Zs- z7I>ciY17Zn&d$!ug9G?~XJ@DS|K5vNyMNj}c)9m-=hgnpoxMNp?C$UE@BInwY#ft@ zCzlF~KkeMPt!n4~K_1K)S6ETbbet=apmECM zOC%K+cuJ6=J(_TVe#pl}FrtVY^k!W5XQosphuhmzs%H6k5cB!AmrK5{G}i68q5@wJ zOz~V25p$*pQ&2Nz@njndb92lEu4t@3QyELzix8X0o@U=4gK;mIcaAAjm@*=uAu`7_ z9irgv{}b?6525m02@Ccr%prL$VggV*5x85ih_ESTq-$xA94sc$95Xy6qH8@l!(#19 zYZlLG*OH1zC07(4}(DXu`!DD=ptA zG$p#gF^N+w2)e?8;&DnOQXGv5%CM9qL6j+u7CaY7k$IM45c|`Vk8x@h4wVx97eY@^ z6nS*)f+5IMOWDN$N|=|n$eH~#MEyQ|wM+JqS^5l23-z2RMcEXo8CD3VDZiG53{@P7 zoFUE-MrUS`Mn4%?ZXF$1V1(%|CPOsAsU%?w7d#~b6e%F#8?8hj#uH5s-Dr#%w4#@* zC?Q$O7bGcgi1|F@j4&l#Q_2IBt}79`B)YrBD$Zr!enVyeho}$wOYxdaCL~rv)PKj% z;~7cvloTI%rrY3Ri25gg%W>KdyY!AH1WA&TSaGqTUg^5$7Ua;QaapJY(5Y%K5;Nj< z6pMOIls7zL1@KZ_5Yj8BD(VkWNZD@yg<(Oqada@&(?7X$GK)X=ee{0em!ewiS5C5M@(Aq#U59?=!>fsDHUH`&+0lF3Fz#f4BdvhmcWi z;d8GC>#Xj&lU9Z;jCxX<{AVQNk}56?A{sB>zY{2q0GJyc;mdpTWqVGA;DRKPYmM@# zsuG3rCL=MOEQ)qW&<;{~>Lxj!dc9izwrWFX^bG@vJL2z@m1P~6hb=AXK~6g(|KaqG zc;|E}$~j$$ty53mu1WV(TW>=HP}7o^b;}j*JS3n?uiL|OU1>wzD=YKM`UWLCcg(8X zr@C0j{(WnzAgEL69{OoU7|NLhtSCvg0$!3e6u{A6yI*)jsWn8Fl zbC#s!2dcV{x~Ygv&ASc1MIssltmu-elD>qew`<9ViFPB>5K_vaX8`qtg zq7OiBFT5TcEc~Sg9Nfe;Dj7*zE&yecnVBb#4lwJ`= zq#Ox8Ua8z`WLyO;_{QYH$mN=aR+kZL!}z&Lx4Spk>490#K~18YY21r9x7H?d-}?sf!+zNUvWwqu8jHS)QiO z{5AHt#Y=H*3a|EUm>pp<@U}hV4PXIBr&#i~;bfm3wy^4&uQ2OsT94Fu6R9h!P|F+(=2u2G` zcxS|W5JmG25CRpD3J3yxWy;m<^4g6BC zTO1;mr)jTZey^a39gM+UJLTZqQ_uSSUN6&}D-~g8&{z42q@5t4v?)|{v;B3HVhPjT z7J%PFq(ttWL$7H{tQQxk$0ut+d@V14sN_cZsWrcxNR(lL=R^@9*9H0Zaw12%wzNKD zCk9g*V-)k8sdYiZZwui_0~n%x?;XPmqm;)urE1X)a5&2nD#7QbC(@Wm6_LqA&$DMm z>$-(Ewt*pX6^f3z16mnbF;6)I542=Op&M-?-%@KTpFuLMtJ|_rIU^Q0~8BkOr zyA_?3a6KamQ>HOJCMi(_kQAnUCxpzG?!HRBM4~yJTHh|9<*8=zn+QHfoDpDn5Uq@> znVFTQss>f@TAmYy-n~9Yb2H#sL4(FYfiGu2{lB(N*+Ym;WYPD&>1r+nWz+xWqL|72LSS2XtLXcRc3v{g| zm6Z|>elaTt=;_G~o@W}Rk3WCvGdw5#VSkL{%RJL8LgjaY=UIQKrN!sJel+XCZ+!fQ zN;O13|Fu_k?Q|n#E(yXFe+gx`qTIJ7AZ9xYNT6vK3xbWJ!4hrUC=N8X%(1LVBjFh@ zBWWm%uP{wDGFxTwt3`;}$Tvb%HyQ{)phKZNV9*ma#(-swAa-$pID!8g?73B2fw@*D zWo|a}mecbIIyF+RT$kKXsem;4o83~Y>$ZUsr?^(P92ElGTUtmsk%)0+nhHQ0I!KTa zxiqXpkY8hK1U9Ycx99NEtU)cbDz8Mhq}c!P90tw}J4XL~{{9`3N_b&B$60Aoef{Z| zelF5q`omxPE+nDsS|q0IK<M*Ak)@v|d~GM*h9-$`gR9V|LHXwqb1Dw6>-i3c?=B}y5ohQ z^C7Zudc91L57|`UgaE4;UA@y5e@4WFi#f^!iE;zN^6dZ}8-&U-2v#{8`KW&fgQ$NV ziYZ4&$J%5Bl>>c8)%#2$$y3-51vdJPk5AmrX{Z}_Mb;teY)^x*Rm-HpLOGFE+Y&s+ zWX_q^bw)h#eKf^Gb2*jVX^s+otkHK+bm3#b$b!UtJ|`@Z=#r4E2G)+Yrh)B)4Sxf$ zmKZ(jya$q8=3}iOBEh*r2^GM{i(C3cEU+`q)iD)DnzSqXN9U*L(0bl)(I9%--|s)u z%{upe*&tZmc;L^zQ~QDYd|ec*lk-l^TVLwBO$Uy{J2fBpDXS>-$`K?*$ixXtGESK? z%ak@CYz%}|G4f1HSyB>e?_Qs4DRy*f{y+Q){`>9-+#!em`{C>a1e>XTfyI=BWO#?? z-WD!GVUZ-UAPQkAc}%fhT>aW$03ijC1{KIhf+U0~iqms|5VW*P$at}j({y2ZH!%HJ z;38{Q3iXAMO} zG?k6Hs{2YM!bxIyFVK!P2@MPz?RHG%TRO^;`j3VHZ!7c(N7hys0~A^NTEHGHHUlK-Vs@&Wzwx%@z(%?`#5`Y>Q>jbHE8` zyl}V}!3Z$U`9vKuMGq$v%9f#4Vs=5s#-%C?TwwBHR+x>3N(O_NUlC!}UR&WXSKRt# zgEmJ*#DpoWr1Oc7)c0|HV*)gaK^O^=VnwgOwwz&+7|%0`afV|U{bt_U-Pr+VvcTV8 z5#iT5%BBSdt!T|Dqw{afQ={LR)5O(ZI)Axgk<7 zLxww`V9pdx(VPpdt>YLvIM>Qq0ZUK5T#`0w?7awy@mAxD(?eIg12F(Q)3}=j+UPBY z$qJEXnM1CN84A*{z~zicBCbeMH20c|K=K3_S@jTE{m{AZ+2s70AJaJ>L-+a z*UBGS<`!bH_^{L8LK!#4Y_L`p8YMI{;%R!y6cJZAZS2CFvRsjB7qk@!8&sIW>@zy6 zAjVbJicjHsNVLIbB|(Uq0r;&^c`x3yt(nFX3u|OaC50UV?C`N#2Ixd3wK`CDYal=y zWL7tE9+9GhddM~6)&kO47&SN4enW0}i*I0|DrAn#!RnagY0$Q9K_lkfGJub9NDJ6< zAxS%F(Tqyi-EV8b^v@e{DDG-kPgt%WpgX$}_uFig}C zsf9onSlh^41gBEgIP&ns$6VM$MrIksNRl~?dCFNNNrnZiEvfDN$IE<7VwIXh8Xu8d zFokadH~1x4?0y3PcZcSX%YW*3d* zD`_{HIc6lyP$tL~VXCQ`5qM()PPi)yZu*=hM3}CQ1d%cq#6W+?6?M8K7`4|lO+oR8 zV3nkF6r>!n60}%oT3S?UqnUuVz&Ofm$59MDx5F=qLJOjb2$zp;fpx(USxB&t#I`r9Gf13XY1P}QfyV4g(?Ur!rUC+77U-!STD$kM zWlF;yggtc=bH+&Q7CEXHS&?UoB+mlRYH)()dPd_}(U#eXH6DAW?O;;)#3U&zu7O$H zK|=F=%mGO&h|P$>%ycE{}F1;XykQA$LGY6yJ z%9-0fN~=R)7wbUlvzCB-Hb95qMR$6KvHt?J^jeOEDjW}<=m9v0njq4lr4X2vT}KlR>cE(2 zQD97ig#{r9Qr3&y5#!S&Tj;!ZxWX8y4tj;Xv{ zg61d)RZ7UjpCiOZY8GsSFTFSpM1oX#d};cCk+#UlBu~!?XfDUFa>2yZlNlyO;f6Iz zl13Eb|M@i=@#NTw1{c9}H#DKKz$-0PmA<7(pe3B}il#Z0fN33m!Oe%sC z4L30912JL^lM|dwAv1V@ftQ(u3fdqi{=dv7|Z<3 z;L*~t5|8@LW)t&foWW*SP%rgnkriaTXjtoDjZ?w~m}pJxT2MvoId+KS)3V;nCg`}) z;!K~+hw7$cCNuyABHKu~z9|^Z2u`A#o0|chPx}xCfu8#53qK|O-RooKwW4zqdjj9D zu{0h)N|I-={(*g9dvDmb5@P0+3Gn1ERs^_r^HCN{_?ad7k)e1nO!dh122&;DNf4V} zJqWv1I_WG;WctzhsXb>jKu0sorX~#Dhjx{kglITxen1yZumdbRbxfR=RK2^C!;mm193I8=O|7YngopYSwZisd%v|YNpMRef<+vvtui; z>{bBnux+(uZ!{&Cp*#yIHYJQ0>)N|t3{@Qi8UdEr6IhEoIgHXv-Q-ja}YQ8*$$MSGF^PskKGe>cyen^UVD4HlvcoQrIdyAp@VEP9oXHQ7CF7j$y@eQYvSv1 zu>)8R`!ll72E6)NVzpV=monxo)bLRaF67q^Y|9e8mU@x!Yo<-nVoj?ciKm#d!dRCI ztK8TlY3jmDHNFr7hvGth^^JPc*V<)2XP1m$vwpQFXI6-}bfvt%7wk`(Ej=*yk47K7 zBVATP2&_la?B0R6hL5!z0b1qtUHoy^gt5Bh7hY#?-NxO6eob>MFjIsir{%l}N23pI zLcr|fkx&fGtc0|#&GKG;VCdN6G)S|sjB#qBA`0WlelI@zP`f+2FBsw?n_&j3r;YV> zD?CKO%Z#(ctEy#}gH$*zoH5S^=zSFyjGe4tjTK}XH|&+b4{=9nI?h3?$9|C z${tO-1?8Pi8ac`$M{FhVWJ2R^fmY$3rmb(m27AZv%$op(*D0=gD{vfxy^K?oGaFRt z6;Ic&F*kIN*z}0A2^DiFqv3_lXgVi)8FnY2ni9Gg-;1-?&wXsQ;b~X1|Cc_L?C)#fcC-Si3i(#At4&UJ~avDbqJ^}*B$ zHm$7hN&}eZsiIj*P&}h)VrkPg?s{pbC)(HvG{?|XE(CgbHlT5szBTJK%#Ugu@fOS; z7(_`_Nl%RQ*hd97n3<}VWi!*vvaE580ENEUOqLZ>yuN0%Xcw5F?=LP!C{C#!FQa3a zIC!%lNa8k|H5JkMg2f2dW#Kg@&LmBU@T4jB10byGiUfWrOG*t(gZf^n>- zPK4la>h~us;3yzef+K^VI*X$*)&$T7ikrW4qGmh@VmK?p{%e##kB$RS@B%6<@cEpkvJnJOkhC4^*M2=cD5vo*7;&Lp7Nw zwN^pNHJT8ta$&i9#`83>PJe&sC=)y}hg=b1S5$B|cYCW>d+YYqg>`AI+KlWhpAwWo zeq+5>KBWx3rQbgDJ8)&>tF3;YPl>MFW%SBomQNN!t!#7D#&M<1@xsK8h9vv$78XduZpQ?Xd8QVX!(rapYIcj96Bp{A=RS+0v%|SZ)(GOx9_$a*QYl%j79I9x>x=G3UAF6fkb}u;@{=lSQk1f?pA{ zRg#O0OA?-8lW^+CLUTtbt#9II7ae+LY$PK7`gB>+Wb5gOge(5cE}4P&sGfthcCyE3 zl#f#yqqjr}8cTGH~f!sor z#c2)`^>l=AlIXR_Xk0KrM}}^cO3WvnzoQTPu=H&4-z#O}2$6 zaI9Xlz%^#dY*MMPTyvxTi0PJdok5KI&wu^s#tF0(Jmy!#w4yb&oSCDeHss7Y&g}gO z&|RD6`jRY6+*Wg*>L)?YYxv*2Y{_`t67N?xfzE>j44sKuN^FfjeQu>{{XlwGx_

77(^s6LLes*qeU} zN#XXgy9ibyA?h!IYfIMW{k~qx4GH!;q<|0%7@c` zw#o6~be)M|#&8ppVhJU`=sd6&9CXf8;=&)?%?J<%B$9>|@xy7j)>dw4@G(6GikUus zH?thz>RPItTP!SD7{?$s^|Yv9qovJx0va?1c@E%y5R#?l#44%WXyNXDFoy$*Wz@Ux z2o@39;gPaXI!cwk}{>)fbw2&0<&)|iLM72ML_me6 za+NVErfb7w!NA)~rpo|*kfiW%l$Wnu5_1n0zh+@2Y1THF{1$6_I?uovRvFTy@YGD% z2+nhh+4ZKw_3e<8_Mk^X(isdixYT@ zxG<1{GH2(9Xmob^s`ATy25rZqnMXMo1A z*Xy_lWjrywNY$daB=S*?qKf) z(A2K}CWg4khLqD(MU8KLl3K_@+%^Q9wmQk ztjoe_C6?zXNhWz}(|l;OQ-AtaBAJgP7+D|MVRe-Ptm7`wN-Ou$ejN0Cx%UTa2aK$6za)KWkFZQtR?NXr8wMeqYsN@_Sa+#@gzL8IZK6xGz`wZ=Coh513AcH zBLTD~ryDJSmqGgQgMVNC?tg3R2snXU9x%P(yCO!vhQ$hlFROsmZ6eZq6Bez^A z$Z2p=s2W4!a&AyY6~!t2U9aZ7AE~4YY2`QJajv1+G*QYMMPZ^pFnAtiLF3af|9_p4 zG$TR|RJNI`pjXWQ{a3ry`=4JNJkI}zc|LvGevYo_d}xd;ni93h$TxFMmUu>n==t{N z&plnLcXE?CH6(Pbfyz7R*>aH$;4bu#0p$NjQSljkI)fvq2VTO#kLL1mUDBz(=^^>XuFGm=MjH^%meECHSEkKZ(9H0g z?nH7mGeMb}p#Gm_^k>;egEM1!n3@LcTJ&tj*d8W29R(=6UTCzRy7-NzP7a>cu+_qP zxV`;n>D)%c|9G*p(+_~&9%*CT_fe(EPrvkWu4eL={t$iorT=I5hkmL5G_O8?{@nKj zuLLg*f*>IDv>^Nt2*0Y1Bw}SD6S;%_j>az!2#cTjzfjO)!}Sr`e%>p*0eDPO>D6WF z&a<%uiv)d7(z)5GkeV&S8Cf1?Q^pV&XIaJ3XFL#}KKc9HYg2uok(-fpK9I9*5D!p3 zWaiJGqvDOt437Tpw+j0I-bejkVesk**yF#^cwxXgUUyeDHCA#5;>uMlE)C`FS;`Rc z!z^={F6TTj=YO?x_*XZo1;f&bX905&{=i;4-gv!z!yMo4-wFUpWg#*8^$Apws6~%l z7k`v9?aA`hyAQ|s0C^PcM&><9)qTgu*=rb%K8{pt%f(xc|c z@r72GD2-ByQ;iF0*{H3*{x|4#>5OP6(H-}9w|6-)!Zts$^#NTjt##7Rp6NMA3@UVv}Zs|z#PS0+OOD#hq zR96k$GBZGbM~t$BFoh2E>=k)RACy{ic_2?V0Rv zwS3q%N6d9i2#x>{g5NB*i=RA^+bM@rQ+AKW{0U2(!O7lS06)ym{TyduG3%O@&@a<< z^mG4t|F5e%?!{B3@|3_mk13U^4hCnLtb_9T+ze_qDUcO!43~ozgWgErG56h?x&DjVg*rKe0yoSEElG+}< zZiQ{XJJ4G$pG}dSuK8uanz5vw9b-w&O`{z;ZsY$4t5CG_#}=<)>d!CKU&WlMau2Fs zTsKxB8+eIY5e!-hsAF?-1b+q}T^L*VcktP5H1xOZ6|(NJpiKCE0owd-k{a+w@WE-h z)=+DF=Zp6cs;Dh~<742WH$Hz}hT`^wGH}X?0Gsl=TI=Cwi-9#~Dh{fZ*RsO`Bi-fXJ*Q(1Q6YsVXmOJFuy4+Y@#t4sfR&{EvtL13ojC%E?1RcO`A-km2 z;k%vtPOZ#y1t*}c$#};3Whr9O3?NQvvZP3B$3cT2SP@wA-STU~Nb6X)>dJMLwK8mG z7*ENVLgE;+suqS;%j@Y5s z1@n(LBbrw9FEK$nHk8wQZ_91IK0zhpB*Ig|RD~-|)&QR|0N;*Fzo5R+u>O08T_o{F zHUO5d|GWDKud4AMFZW-(cwGM<;t7Qx&N5lZIUh`15^uMZT-xBqSr$xW>sqPL@bZT> zW|GWEY+{YWtK#R{l6zyYwjO{sMBEgx6@y;uXZ@c4CG={wHL4#C5(0rC$u_s%I@Bpg^3@V5slNzM5{a@Z5)?@MMlkW;t7*s<@27+W< zQn=FV^XK8}@(NEwYSL6e+=t6tMSNKjl;kvR4ju2yq2TVe2!UCl#SZS6QL+6 zjSZb3@;8rK^_ruUPY33NhfOUK;u@jow-T(1pER;p<%AU?^xq! zxEZc~2{q>mc0}7ClC+!*tTVRNdQE79+xZ@F2Ds}YT&mQBt#v~hJnsSPFtzo%$T!t( zqk9Od3^6CG%LidY0ovG)>fnz^y9M0+iP_PHk+GODBx{b zo<%O-n+>lrnOTr2i@iPADT%jszc*aGjbdm3>D_OP4>qe6_o6t`z#c{I#jm2(lAu z48Fg(7@hksW(VXgL=J?6cgE3G00ZteGq=NU&$v|E3Ax&?)yV{pmsQf=>$My^w$!X$ zT3;(Cc!sAo*YvkEp~9T5!f9($(NtuyJx$rvrb=Z5Cyut;=0x=%QhB{Ltxb6ZN0_37 zik61LBHPQR2VqzBF&e!;{`KAA+mrLr;n7LZ0vw@!E%?05I}p}v-3qn$1|g>l7h!9S zjh>D+(wf;k8)Q5Q0J?VGy+1yI-fgC-jKl-R6WAxX%bOJ({bt4SkiDaVG-oK9-?EI@?0$5 z3v)D}*&^y92TTlBh=F8+mgA~isWf1nX`1zDp-1+}FY zvlg|oYPO{kGRuMd9sMomimb+oi<Xex>Hn=xfRne(mlIsRy`(nVhPE0JCI0sGSOj zIGgBZXUMNx`|rq_{Vzk&)UaHv7~GW@EvwKGiCYYAm>uetqgzTa^)SO*!g~GjEmill z{mb13cr{K-Yji}T_M)6z;N7aNgFH*HBIinAMWzcgT!N=5Wz!Gv$&KI~-+W-$T$+}W zA?ok$?EJZ3+tl$cP3!P~V25U*+OO%*TsfTe^qQtx$y2y`j2poCkv=F8u|w({h-Qrq zq`b>T=B>{~W`%m$BilwcZ|X%2W?WXb_DqbNv8uvi6@pb!(%cM54D1NH-%I;Zt@KY7 zA=0$}V|KG58>V{qEuiK5Kl?9U?N`tLzuI~A82|AgPotcKySJuqakgw{yTT^75pTDY zy$+!!MjfKRN1-HOT%o7BSEmy7DTC<1;`o_`VDpd2feGcj0O9h?&3E%MG9srWDV9}l zkQN(3>qro|asXQ+pc&05_QfJ=yL3go+Q23|0=Zqn+d6-m(Ug=g!S@}4!jTV4j(+3g z=vtSL?A^SHt+BLgU7Flv7flSQJ%V{moI@N8FZ-z$KFfWuwd& zUfPkBP2EGk-cm;AI6ZV`#FBo^P0`4k9BcNgy{UWX*O|Sbsc7Ui7lF*E_0HYs6m%+@ za8VLcU3$5y;Z626FKs~iF1c-{*t}ngbs5AxNKc#f9B#v@Nl-V}P1?NtYb8WW9_GUS7D%Mah+zn7Ra!haY!>;a1ds~~=^x)vq zh^MK$FuJ1T!S^WRX$tpWnk%JOs&_T0n?&5)TWKy|zqokg5jO85r<|(gh*qF%cD;rE z8^tfw2W7DC8p2xjvZQ$~=aF{co9%)He4azD#Yvv(CcdY-@dex>81%&I3V zqtx6wxU^R*+q;+U{Wl-K#W!a}X)}bghCXRiV2*Dh;mUODM+NY`ApaUzxWl;EnQ_}O zX>MnudPErPVw!90QYBo+_N#lBRvcs6+0l% z$FFRq<5Hs+O z>b+Y&G~aZEyb|@VsukR*ccbqLJb}%AS^m(JgAey)ehhURP+ zmRdEBTl#*BrdCgE*2Fs8ebbWz6HBpZk_o{v$(HfomJYleO9qxbBHi-a@dnl6lTQ!Xl!Bt@J>u`LP$e7Q-!yq$Y+)BZ1KWTlh& zch1NyJYu6 zgl!!wy`&LkZgQ`1wRLR#9;U__$x_0ogseY5VXA7rH#a{-STefCRCV*ht!ikayO5m4 zxe$aofAJX^=QK@5L}&?FHsFiGQPD>z9%V!reR{PS-=)d>>7ro}<25`pp14*0v@5H1 zOFGBjt$Mqn^?O)f&xqL>gFS#?*TXHV!hScK3M;lhnqh9#j}+?BBc909^5M-klP=?l zwCC8eJWaa@##K#iJeI)>{*k?`=qIXX#{{;M61&MqOB*RBq`@+uyW%LirfL=?ww~Ku zq$~GW)<9s>+U-$FFw3)ZAoq7R-FR|t^S_mq+Srt}w_HUoIe8Mfba5*++2yU>Q{O@_ zzM0(1yH|EjTfwq!7uy@XAIW*>?qxJrvDo8$tAwY&&C#7UcH6DDlAo(_*z^XQJA-P8 z-niJ8&(XR;?!MTsXy?AVbVlGj%%Ve|ge`i@PBpc($zDx!EHG1qB&X#=+DD@gzDGr& z`swmG%a?X$Sp%EwNO7-#^`mKqmoZL>IH~wFi=vTFp5zNh{vx?0om$=4Cc9?xfgU<% zL%3IGK1GFFW7Q^I9o9X>#JnTwnu~hq3p4M#M7LNFo3*)ghuIXuZtZoSuTwk?Oe|9% zih~l-B*oJe=2SD>P2z4=0ITs|&^h6Gv1Zui!5J0Gd|rxk6TFQwf?ZFKvBIZtYh$RN zxvwVDg}U#{`JxHpy`y|OpOb`QW#X0&CyEHCDmZ0uRj(rIV3;_m2x}-_XYu;!`n~Uf z+`SX%$Yg_GNd%iUe*I3`1~%)8nDZ;*NF|SR*yyJzF}m}Gp!2{Rq09Qwl2MYIumWE^ zQi9G`cqCS~x7ipWS@8{27eHO2=7Z?6?zMux2TzeE(D{PJ-z$|JrIavLm3yuSdMqNj zU~x24DvM$$+(`_sYHE|+O8N8+xgv$0nDS|q>NiVKbk#N={&vLYnIMo(<1JA$o>b2` z#6^K<4kbE-zty1L2jb4CDcepSZEr%YHjo-2-a|lK8_v!rwieROFgJ4yXiq~EfUVGi~+PiLrnbA>{sL$NPhM3Ltb&V$$s?pO9}d3*P<1#fk_ zQ7)zp`|eH`&h+O_TkRpd`;t%Jb#eJr#MNCqKwaCu4_oCez9}CF@nbT_HyuucRN>Td z>~{XQ@%*>Dk}EQoKHc>M_=^0$2fI7F)$`v6`#X>4zaQdRlKnH!MqyKAZ9QUoMX#X&|dW#IY=;yEH z?CUM`brkD=;V?J)8GZFM&P9r%Iod^$L=iX1%vheLD7r@dKY!BZ^RF>a$mc$?mqu69 ztH9OK+gH2K(7%0y`g=P&ee~D=L6YuMq=G7+%GrNnNnt}QCF+cG zRmt2~p!~y!Z%^KwT>N_W{{6+T-yVL~F!00m=Uf|#P4le*(f*xijtu;5bioWDC9 zoqzxS;$q_@YQeRQOm&R!Myixv5ueF>IkCz`my+wv>5nJxPR`G@IQ#Zw!xU>pxs7H` zhZ)@zWhVq+FXKfpK%w~PpY}QSL>@vJx zB9}EM9?K(t&I@!&ToeCZ90$LCJZ^8LS@ZsG#amHU77`#)d2 zeEGQl_Ylvg4wodimdWWr@Z2amPrz3$huhmzs%H6k5cB!AFS^Zfu4Yjt_|0N_obvJZ z9P5Q)JLXIYI^LEvAu%>v&AlhcNE?I^)Nn!p9APpS1W|>qX_}&35~OAX8HD;! zuOC7JDNInz88a6_A;qDFY2u1vgiY=8jnMi{(C6?f=rbsNFSiNg)bF(hT4yuqZ+%m$ z;$}rC-;RJ#!>!A@wA(~gM9e9(vEhOc7K=ruzREiD!(|qL zPiz{+L@1kJJF4GQty1Y}UAx_&jIF!jplz`ASM+y()AYf?z9UC?6+&xmVDX~*Uwr*# zS2N@k#xz*$gFn{}9QebHcs+oi!M^)&>SD2F6Twyn7?_#!a4`jznZhuFr5bDFed++# zrTy>?WjqP8G^OhqkPMdhN3wDI?^y#`Hvgxth(>{ zbn=d?ksy);7nH8LEOdSG=pK8YUOrA~aibm<(^6{@9P^M7P7=*`-wbcto(2Uz-h_y6 ze9h8l|F$}Pzga!C-Bqko34-U}_!x&6r1cF8OnA~vP$CW>)93#nMW{uRD&L)q%}u>}mc%4Qi5D9{XLh_0}R zM850}T9F4=&#n?crJ3cJLeD?{vL%e@&HH!XegFRK-LG#C-yME;a#p@-wb<-i7O&J_ z2KU41ucP;8jhi5Sjdj1-?R2l}QuA!-d>QB3bBvhN2V#gm6`X2NsWW7k;vw%XEnjA{ zf4rTP52ttAMXA-l%MQwi)5qPDJMErS2)NX>UMaizU6T)|k2@xJ-Z7~Xan+FCai=7T zqKfOM;upFlI zzX{6r?U|=?{WjDptt>@y9TZkHbq>bC8D>*=qgvZn!?~5Y)CFpdp-bIb;h1Z1D0)3I zv$|%@J?Kk(YQ82lr|aF2+EBjCb76g!r)hQeeCgef$ERWckFu#CQf^}B&pgZb|919v z_V+9Me=iR9_a68E9^$D;0yp-70u?UZHcu91)4}CmWN9z#)YM#dR?q@}z%zn%(OKY7 z+wN&0h1oa!))>;>vTz}4%$5nStV zNx)kJf1A1$qv=!q&hC~j+W)s6MU9@QoE7t$ry;nQs^?&cx?%u1%_5UHBitMoiID!>t(}=-&3zUSSe~=PyB(_sE$etI(5Y) zJYu~boR+Rt)1hn*a+zrN3_cu_$Hi&m&-(J;9hL7U`~TEM{=cls|CjswyN~k!A)YTR z{|n*qrDcDC$)ntVl>7fOPZ#-LORN@5|1M6#mGb|f692z<@bcxsqx^q}=fUNFgLry) z(cgf?qtt(t`u~DY7x}-w#$SF`$^REGUsUA({>zsy9_9Z-JP$7a8${qe%KtX0`;Q{+ z8zWrq0wR_)%F=vF*&Ri=yQOcuXz3_gaSS6upUe@A(#>guppC8I9LC8Fl}e(|E3sbH zTG)r*_&8EnUf!yqRg|u#i~M6QXIAT|VZT`jU6+uKYm zaY=UjpPdro(eA8@l*5ur$%2Otw+K$Dc=5HCo>Pf?3ha4E=)lvolmxn7!}%+kv1zW; zhP{tD?MT@@AM(9%(&~@)rsFS8Sl!cTUFx<}T8qS&`|=-$!Un1mm`v!U51$SWPi6I! zdrj+qR1UjtF80R@wCb|{Uz72S^UHhl|LpBm_kZ_azIyckJjnCq^#7Q%D+4dG4EE1r z0qoXN;cS9lIGI!V&q%JlKZ-2;S9}`ef5xS9H~Rm^$9F#mvO@kJ?C@M^8nw3n)$EK00z8@-w>2{!Rr*Jj{!yj>7g6aSwfl`fjrzaws>CVYOctn> z@t-fN`JY}s?*Bf}6YA&Er|qxpQjS7TTf;78zu49lRM8lx{oAX+36Zg&nYmx2e`4-= zd1I@gj3*M!@RhmlWk#nnB9(U>BE_|abw%Kom<7rixyeYZNP@@}HTS-Z7f|&F9|z)k zP6WzXLIenS1aaiKApM>Zt;Z(g0SPzu$j}f?a4JbJ6NJnGNvAfA%$qm-ng~a*o=u{> z>zIA#k0rXA=D*A~m;3Ec9pXP<)b4*e(61in{{uXY_sxyYN7aZAV^(ip?T(4UUMle9 z+}>@sI({LO+1;&TZ70x_Oi40CO61x=!D-5`NACTJjB#qOCb9(t`CCo}Nsa~2j&O#@ zG^L8V@P&6L7r&kz{^S6Wq+aCwf6jkBJbrunu4k^&vmqN!5SW*zqoeYR(dlu4g1_hw zlp2wzWQfl6%R{z+WS?&+%Wqn~PULrj=UH3bJaZ|{QYa@|3I?g$Au;`gPT%4TekJ^x zU1O0Pj!t`Fs&e>YGKuwU9LH}UOno{sZCKXAUBnp21DvZF7xZ@+jM{>7##6F_TOy&$ z+#&bhnl2lgr!GP`qXi)&%lg1b=47gWpSYVHU8tO_&-G+b{Zeskn^{$t1;w8aIsOgv-wIAEH}8M{ zZ*RYr|M}I<}czu=Vquz$4+m(97aLth;(5AJWlg?7R1bWu+GpFe-y zMAFSMqqZ5jP|-MNN&1kx+YiZzUs|;OLp<&4|C}?bxVZoIf9KW9n*P6Y@EHI9Adi)< zem2gD5;XR+@~ve8C%B`EN^hBjvgsh^f^Z4fzE>rZg|gIFA^4oA8Odc-=D#}#r{cu* zdN)RNM$LI8*Sr(DA@fngaV+mIvdR8C^qa6G= ztLptPFLxi~{~qLd_zU}6&wbw0+0gpEXv?nF3J*F~?$>HWs&woVRqopgYt+5u4@2&@ zW25(TvUFp_Zo;W_X0%G{pu(3TbxTI0d&zmUF+w+>R7z%4r8AWGg&&5{6)F-orECKU zWMfCYA$rFdsnKwqRI6dxTE1j6SO)DcLoz|Kl*afUj%F6`Kfp%GKk3t?|0yo;lx!^g zYxVh$>irM9FArWk#(zA>(z#R^mN`N(1id(lOp zoB0^*tD}{NCY&W#^+th5F~p^kj!8y#qd0=kw& zu`z=E(%cca0#jeD-|_5o1B|zq6FJgE3(h{z{kxSYl&aqb zjX9kPtnfGux?j9&Z@*Np-mw0)x^Xs<@3Ycr5>{VE)d_YW-uZu(|K~qI&aoE3~9kXBFI0G3(ZZUes)E15WSBeBKSNi?DUIx@QHb zh#r#$`oH+da`Vt=>oq2^DnJ0{j)s{`oBu04gK1X3wYK3&x=>J^B;TrkMUm*@~po9 zySS*j;bv^jB)1FmDi+S}mcv);SPoyUbl5epQ9-HBj2ywF#mG~&X_daQ2K|}`QHwz`QKi?IC!-GALQ9| zeg>0;p+kCx_IwOJXG22}m1E6+vV;s_ztDrGW2mdyM~jkY~-4r&c<&oZju1LW)?w5{lUlA5W)(OtB)7;->Bp^_9p;zv)yi z`Z-XBDet4dmJa8lpZoTxbAJoDKewgADu*{F4kP?C_-F$TN`D8RB~gm9sqAl6bT|tN zl&S!<`5gez!mRis_}~H{td83F&KIvXE@xPjYL;FEtysT(foADNEBvrj*i6&=mew?< z*b!RtsZu%58WFC%UW#^}ndqgUvVBo&E8v>ai(0|(&(_BM=EHA%Y}qMz5y0dkn_(9I z88V!6k(Yk7GlDtTVa|1!I;A%uWbm{H9V)%371Ra@7o_1$D>8vpriKi<{u5T%xbk#@k^eCPl z#nZp8c$yNWOQ%%A|JPW>Gdkw=}R+y`Da5y@(rictm(g~Vj z`2$(F7N079t~BGJ!82R#|5rh(R74vR0X)Kz)?WszuYTP^Uy&OsmDJ+j!YpC*l>E&X zfv%M>aS1z7|IGb$bapHe76ic|Pc|K(5rlaltD`_d-N&znvE^D(C0oJeTQXAG}_L5HJL?*Q?F zC7}{x@ki$)i6{$1hLb0m;4vXu)(rY(DbfD-mmcoga-ASV%J*U*ai|gWeN#p=W`0iby19Bu&vY z zB&44T4Dx3p_#9z#(}zGdp^lbT7^VEWcxA!AsyWXG&#qxV`NH+s7%N!jEV#8k3mMiHvgQD{Ma-^m3$2A15#v6@JR!P<%ml#+!m;2|B5aRY zBR)Y8ZawVvb_WGjjVlF35jxUhJ?2aao~A@>+szr2@+pmx>(`X3Sw4moZeA`_vaa;@ z0{G75f~;S_v{Y-vZN#9IPDs3nQ(}{nY1_72ugh zERt@3atbqLBnTM)k}4vBmk*0W3`pcm(Yfa1d4?6$+BxVB0oG5CA^A4C+8ylf|NB-^ zInl572e1Bp3lS9$o@qV~>iyR@`upCl=Bu~(k|4Nq=;Ij6*~cyPG0sJ*|DRBn%$d@E zP6f&I|M>dyBM6j_A&HicT5`gsrJk#m+lQ>JGSJKuG_D0hL}e^# z6Hrj7jM#-T$w!o6qV*XnHKnwOMZ8%tQ*<_K#Z zh_X~Ks`eUDGqT=1g*oxjTiqYQv!^)v2%ff&+WMxd;HXTHE3Mn63tv!)oO}$H)8X^y zz5ha|6SUyDZ*;+P;mz!(!F6^r&{vZQOzA6%5k?>Ns_8&|L{G_;=55o!9J3s!=>o+m z!3^bD5uNZ%lb`A?rYTB^GwEU7wgSa}7WrT!PQ5?Q9k7s6CnB+ofy#iqH z%pWI!CKy-fnu|;H)FSMT3+b6(>Vg$315GJUD`7@NqA5{Gsyx2*o&>;b%l3C+y%W6X zGSn{h%N@z?eaJXNHk6_?&Wamn>XzxMWrj4h91i!Pjm}2}IZ;BglrKo4`IIWn5Hp-k zikaXA1lWC4!b_WOz@wlSj=-Vnrwi~XEb+R89{lQrK6E+z?lPl&;=k=@NX~ekCTL9b zBucz5C+1Ag-LmScVFPA!&7=uYn5I$_V#cpwC?E-vmc+$K`oZvuEtt)~A~5ZA3yfT& z=5B(>##xpw$_369W(gLFv@+9i8L(oncTrqv=dp(&8WYfFGh1p zGd)idKr_KgYkX2Y+d|iRqA0#*D~k%b5QHklwY?&8leB-If^ke z!pg(Qr6z@A$xJU(l$DecF3?BKU9n10#8AanK^a!@3`H6G`cogy@AX{})#tBK1nKC$ znG-Q3D4oeAB>GqyuLQ%2sd&rM3j^U7T@Myk?Vz+~u$1dzH2tjUuun%WxrQ>EM_Z`+ z<|xHHYXeR0j)a!zgx@yynk@;=>L6? zr||!jT@x6kbUIVlME?s;T?*SFdIcj`!ab|G*(Onfj&rT0j#D0Ax@36B%GI)2IY95V z?cnqdr|fEZo{}p96@8u{<;5Eq^V%jEFA8@}fUvoyN@Jan1X6S)9*@H70k{wx&n#T% zj9jpICO9)1uyt8zSXyAaa>m^<&r?OSl(F zSK2=FHabbHoDT`-J6pKlT|ncO^cVMcLI=O6J*dC{RygkQyA=`%j6i)DGy+`Zy(RzKfUYA?1 zS>3#H3L;onficy=u6;CY?_}7PDE~*ZYrnLb^Z2wpelh%8P9(^V&EEa~r&ap@?*9JH zPF4Ru*xB2A)c+si(Yo%G3u;b@S&g+9O(ENEu#XYYTIh0raG4|7<_9ey(#xyQB?K$iagi!tYVVd)_)Ag_Ow`PiZNE1eYA zu}{q<)cs=P(zg**vDac$`?Msn6&%^|ofJj@Z_lX^+AN4%P0FLj6jWtmoE7=9%)&$J zTY3f~lNQxbKti0ZxlX~5t5(VEL!OnfZXpRecSrQP(L&*AnFv?AgO`K7hVGqbB&L%^ z8+sRbdaG_-SOo6aAG5iAx1qHgU~QfI7*|V&I}B?Hx$dJ{t+t8ITtCi4&SbybPY`rx{@=XA%fXNwO8}WlBrXHZpg@!-hXlbt}FqU#=ZpQnjvlu^w@Q#x3Pc8O9G(-H))T60FW8JA<9U z{;LKOT;S;g_x)~M5=%NTnHO#;aAHX#2cR2E9G#ycSL5Eri*D@DHP($;bd_ThK3bU6 zD=1gzkb}XC!R`tM`O=hVWt}xh5QHS(ntuWq-C1Z9#6mv=?ZWmp&PX8@>%6o-csV%e zz)N4AB%9zU#uH+S+T7gu_MD|*MJ5(J5)N8-9(%_V;?VZUJP2u&!Zk)*+@tTzF-JMRWHwHQUIenGo2Te9+HA#vm0lskeGGGdO6tdv||2YD640aJ0D8*W)-kwTTPOFnA zBu?8kJRd|_?3RyAA>=-y4`*-c8}Dat5T79IK>E__O*b3j>nrzzy>^F(!?xsHsnDv3 z6X@1qd$C}9pG`~|L^IX;9z^=tVh~AJ4w%jIG_{ckE*P~=s&-a_1Ly8mGfnAgWf%2^ zs6_Tw7jv>@o=fGg{L-|p)r5-^Z9rhq&wD4niLv&iUlrS7TWlh(fz&`!+n+(dA;6wsB`J8#Pj057~>SAj}Ui*8-3*MPm%) zglN15W^9o`;ox?|euSmGPL|@I=^|Vt4Sg8dPq#(A0=k(qz4zG`_|?wA5?4a(Pn|7~ za;f;7{!W@AK4n9yxZ=c}T~j7hNQD%gyTyb9Y!i@bWI?#1Cee~cmE z+WNN}!DLIqsdwC0M^ zMS=UlwxiKP$F7}~8e5xlKXgvpw3p{Kv}_kZa>vxHjez7}EocOp{YczsAZ1)kIUAyw#_-5j;7C+8t4W!*6M zLnY_q*&Kxex5PtIDR3y<52Zqn#b&5wF>s#;;~=+lh=xgXi-Hejnw+pCqc_>$W zjZ!kJh zoS0@HU8K!!C7&bVW}gA1&f=l1{&u2y#a)Y4L`~IHhXQN{>>3 zW}4U}E2i849aeL7z|qxGA-DqA{POyOU4v{Dn<<@Jr;lEwfTOTkx-*`5<}&BTi@LWV zEMfT^e)251k2*qq$M}8sVoaop$YjEW>Q@FLEW3>Sw#Vmuq7Ip&hm#3ql_7Ad9PC`b zAy*_tx;WD1qf&+PkeG$fo_)9J&;nrrRMoF%1gC1YC=TXZ>I0#j(EfJeP~S4=iXM2% zTI3FM#lOWVX7*`9T5=|rpytxLItqN9CT?E7Dtux6^EGrsJY5z zgn-!^W2RSrz51Dq6+Uya;%J6NVlp$L7-u*JVbUm0cXxJfqj#2+g^GT#yR%9^upN0z z8J*+w&@sbu(%I6;7hg`gQ!@+ax>i09I1ZLpM-@bFVnPxl3304)oT98m^p=X|Fgd)k zxm&d%XoLln&Y7aAIYn*KSx|{4f{+B|Sy>+Jz1YwQSaY_fu5}xYGa`w&B1u8j*IWcT zT+^7iI>AstHpoz+YuaR+5hPsIfZWcIwhL2f9Dvk@lED8yM?q;4SiPuOjUqu_FpxX< zN%s9!lfY(AIGU08vh4D=a`uBwHW8L63*<@(Lzs*R$`hyi>M!O%f2sRd?y3Z%qnZ?M zZhKN|nU3qX^f#{P1pM$El5PY$v}{6iJx%q@Q{ zW?A2rE2T>|YnvvlOEYA&_#{xGO2m0x&XDU8DcGF4bDq%2!enGM?Yy5@yyQnTp(IV* zwq2NmtwpMXm+8*z9XQGMs@&a;@~+KurB#Fw@uqPoLhzw@siWwf*^O-*o1kt;onQJg z_^o05(g*&Ql1`Zs7^5Fe7NgORN79;lqB}H zzl%2^WnaT5?*^rFVAD7Bs%aV)q@7cin*?YfX)g)M^b9cVIlpIdZB8qdynXTivG=Ua zZQMxS=llv3t8QcMiHvM#H!0WUsuanwwCe{;lGk;&+PXj-ki=>T%mOodIP3p@h3>{{ zX84jA-Bn%m#U_boqtWPhQ+7*yo_;NWJ^%LGbr7Sk+AU?kDg+nZ^RQ(b9m6e^em#DW!#LE;e4z>x+>YWVhU%eeaVD;{WtxbW9*Lf2F)5YHexItqbStGjP5d{gTkS#HR zD!ytdAiFeRaNsBa!T-K@bVP70(iMh!toHk%F#eOiddKh1L@gBLUQU4P0Tq5^L^9oT zl97iV6z)Q~*Ba6S=p!&~8(BV$;Y8F~hG1DSPLF1fy}pDIJ`W_-+6D#;30|K4F3BE^ zXa?z{4={H$u)cft4DzJj@kQmN$6J4?m8ik9X`+=96aSD)$E}&BsztGvBq=RKEzCkH zm`+aSdDhd$hGepk2in`(t;hwlM323RXu7l4Zxhkf!T1=-c-SINXHOYy85*K();DEI z+x3h~3bQE_FdQJ0^}~6_g)YEDHUGPS2%yhK5CH_*9YoN2vIHXT$ESq|M8oS_HP*#3 zJ;_*2IR%H~X8f;=iY-=lC_6xxa>Jn${xU_KhRwr9~?sB#AWL?slDdNC}z8tO%*mE+%A zeD{PaHWy{pEfyBLq;jMD2TBR_jRnk62Q zlus}D26M~or#96+D!~F%_?>pf4E1Bfm?FuVyPp&>r8)+V@g<55dLqv(V~i%4-cmPz zGwj<>lwC*%Yy{{olxF^z*#u4t)KG8^>|WISSe}8~SQ^Er(96){=MnpKc=rDE{hL?p z_yc?Y;es8V9KL_^nq8cnp0o4U7Z<1R->^@o7booS;^Oe=8p52xyyhK`(b(cdJl_`GxeEch2~UG| ztv>wYTE*=|t4U*wP~Jb{OhD28u`up7cg+K#eR4F=gN;1}Sf<47DzJk9lRC4-C4n!Y z8@$fEphqyaNBr=pxkJ}w5op=v!0S0$D@Ef!)!+aHL+Z>p3IiU?=~Td0f#Mla-jh(~ z57GMCJ%uPaME{p=?_RIuA%&YLpfDoYg5p=XH+bMvPzW{|+It!laLZmx_prw$cdaJ} zsO9p!1`x~Kb$BXx-59}cK~He2&Gm@=*bv!6TbKI*)Z$K1kc*0=eC|HQJ<=T@Y={-1 z$hf$3s7_i-1L6fAk>oCfXI&@x*wW zPM7!pH4WzLPJ-q=6X@^?9#*-tTfC{*gr#j(uzEDIr8`BPC)nAmZ;iOR?@?HyS=~@Z zSYDRw*2r3g~^hr~)6Y{2Xi=_%)O($Dr|2uv>1?OH7Jl56ZHEI8Xb>D_BQ|qv)9i6+&;s>YNj` z-q4TjX<1@q?vMj2#=5YO*<{?5WmmX7M@EnugZcpTq`2nGB2_V ztevGiXDKZfSC5gIBX%_BYK96I2T$3V(y@k~zl8JPV6OnxUUwzlT(MJCgPb}6e^?n| zg^Hf?C@+SOV54cvjoq$m)nQasmXe%&lJ)$U7$WlTmhDg))Q%Bp_8n2?_gz`1@77sG zRM81S^p16Ne&fkSEa6C{5nV4hEx5CK8q&8sy;bqZ^=&G=4R3@JP!iI|ATBp1Qa^u< zLQj@;eT*eJlyOVYdKRV$&LN2UEv|ytB`0B(qTZ!wz9n2!moH=xHV17UMN90(uLNEX z%?MyH#diF1{;sc{7M79!2|&W~E#%F<0e=>q9H{V%cZ7cci-wdtar^~1qnMwuYN6m{ z%Bdl)b}XSX*q(d+nuvRgx(zMRgDF2yKckQ1^DEV%pq^JTq|Z;NZn|SuZ=F1uX8k9G zUnbh;x%mCMd-l3*)AKNINjpl5cqY`z63 z&xm>343@t27Qo!V7rM~5s?=ETmG^;?&$v|4*T>epZh#5qJ`oy1)is8Kn4TND9H1iQ za=^4#0{l&A*R59Lo34uD11>?=y!2s9 zApRVUKPCl8^T=;<>!Wz7P=>m#*aVYQID@}D$Dm_hA}q-ix5Q!n`HRcJ5(Ud&pw$6e z2(D}_Y70qGVBwGmiuA*_5)-%4zV+FtusG7n%0d`FPzZ*IwAqok-chj7tuJ3o8ek%q z`>4KB8B9-!mofVYrG#$RottO7H9uIGqN4 zNsK5w8h^B@XM`mcu_NfZnazV2+fqR61OMumh1QP5?=6k!)@31Vy<~G%x9xHgA zgzlm>x>8022*!q1Ts6phfJll`P26hZ&i^Tt|I30Nzis=hzF0$JOFaXpn3SHr)iV~q z%-v$#Z>9TV=8Bm8dOSUYV#HpIT0obN{(8d?E&WyGla=<8SEnM|}RcF|&S8KQ9Qx z7Au5|S}ZDUxRGU|H{)O(aX-}-)0>*$4;($wcdi*)wfpF7c7%T2P9(4sUJ8TJv&C0g z#y|2hqsA<@reNEGJ$rpu2_qNop)U7UCnUvySdj{2bCUmk#>Om}#>y2q`jS1k!(g>Y zWZ#JoJP~)5P*y?#{|RrbNRv_+LB7#}r^S9VQHN6hSN9vNX|QP$$o4sgQ^e}ItIklH zYR=W6cZ9}(37_^DeiH@qFeD*tlqGh(!?I%9kM3Wppm&{UV}z}=DuTH$cQUp$Pr$o3 ziYv{yH!=@u!-%78z3)AeJ7xxW3cypm>U;Fa|Ed$!AAiWf-A!)iGg5fxeg`V4hW-)& z{A$CeQ#l#2w}RhL_K>TRt7{+i+QLTPna1q79(hOzhGgL@O!9d2`f#MH5(m%|Ud@iQno6?(UFAN{MW8O7{kFT*JvEt;G4}r0J`+4A*2DL_ z9DIK{fVP)|mj{;vFO4r1(!M9rcnhc@_ZA`c{c33W2i;NLQzt|jRHnC+*#Tuv8?1=i zBH9dT@!h3BQPhz~Vq>Dda?UN@TyL$Geu@y1{pg#KCi&SK6sL=YC?u~ja|g%-9wy!@ z5up_tFM}KA?jX_PC_0QJTHze7Cj_4Wutc$f_W6)KWva~rgc0u~4&)6{ONNE@LOw*l zlhNA4%N-u1)bTm6$t3{kZsPFbM}49z-g{CM(7G0P=?q+RjK6GUR#)VtAU0bp$CdRhWM$U+$sQ3 zE{t4!mAGTM#sP$uVRIJSGwQIO4n#wI>|7hU2o5P7-fh9nb*eUq#v)ZP>E0VP0bdB~ z9(T{`f^b;t%6jWul2{WQ65W~>Ok?*yUtdz)p};w z&5=qGspC}?PY_3&#WIv+KcX5KsCLj~(3zqJ3JOnHkwbatFn$;@J_Oue4)6i+T3@%! zA*^Ei*X=Ak2-z&V{)F$CZ|Y zfy!U?F3E5}gV856I!#KJ9&`_JGk^+sYw4`b0K7a{jhSc}JkpEG2y3NQ??gS<30{)r zGHJm=lZ`FzHn2owiJT>tqbNSCFUxG(%XA1$AWccM0OV!vv6p~TjSZ9f^`kWwVFcAp9*f$?%MN&7Pfp*h?LHQ6KY2#gt90(>8gf=wyC6!tHk|LZAqUY9oz@P zewJM9{D7a037vmJbl%$)|D4|SlV&C=J`*jjVa>${1Zt~=d`?Wk7Q{o%o0(u0 z1$!nHdnbR~3x@H0mYo5m%`v|?_;!ufOi!m}R5G3gkt;E2V2idq=NZ;@R56$0XuAGH zq*1clIe6#>6`@jC=2V=x6b~86c$7ywORF6L8`@;W<%F@))2jLb@ zo@178unA(d5(IBh3hHqP)X6x!1(13romj{!$6KOXzkV z#97$#76-ek64)9Fkml!k9RB7BcRX{po=ENpfwS~MPldOAK(uiUOHRJM)HZs*e*h#~ zVTcev)+%ciN))o1&umt8iC+4G*F565eZ_oQ^ZhBe^-|xq-6Ce{`2GNl#2o6me>!*y z?-B$1->l3H|EW+g1DEPekpK0M88lh}qS0WBa%YATb`|VUsi1TYZKSJ7@raD+PBeqT zcQX>$9tULb4oWJ1XLF5K(APXDB9so;5%z9)AU;v$5t-L|4(ByBVdp1@!#_NKk(uJ! znCvxA%Z{Yqb)V|sZLYZEoptt2@6jDQq4+X6Yu@gI%Zv-=*2=PDoba5 zdR^*2?LT{PW)XLlHuZ37^kRrOcRASnSX1CR*ky2j+M{;cJBQs}+*f_K@VLGI2b^#J zQBrlcJj-#n5N)D7tTnq4#@|Z3OZ{U9)jCYlQ?O7gk!Ty@Bhxo3Un?d1!;h( zTdq(=68TM;K>W%Ry*Grxk}gtC;rYAeXqLt#wQj9G-PPtd3a}LMm<2?Bz)e~jV;=L+ zQAAnz0-+y1b}2(U=Dh76zro!chqtE34L-f0$)og;d-|Wcf#)k)xc_wi;l1}Bf)N+I z$^?q9f4Lkqro0@yx*T|07|l9_S%>6-JhCr`M$DvEBfj9Dwc@uH40W}WXR|JsgD+qI zy!WIx??=HV+=yvYdYu7@TW~K155dFMq5aAX=w>ca+lojrA6;;-fr)tOlU8>ep;n2r zsiK9Volwt0+En}yRWR!{E}J`d?M4?vs~&8wC=NZ+lap#@9R7*KC90ENq*cWHuF@8A zZP3H@mfs7*zGH7SFMRDw!-jtBK0}Fz!MQLuaw5>Sl=P<@nONxcPHA`X<}x-C`jZ^H9XBIy+(8D0bg}gwK1Hl7oI1vbKladSSi>q{hPj3 zd+eotP(nD;+oT?SlCg$e-Lt87*V6MT(*`2-;t=eYU)=7#eEGLuesQ0_d>JhvAj|iC zO5;J%<8+S}v1HW9QH^hE+~gTolesX%vC((;ebx-<# zJUBQwfX%*q@!cQ48!ZjbgoZIh6I#Q^2;3G%{@@q>&wsXm>7FT_UpzJzwv9W^a37Cv zd_LZUG_QY0D_t-vCJlI#f>lcj%btzT$96C2#wsIkWGS-5rx42HR zAqh`#7I0wN^DV?b0J_JP@oB>S8r87#Xx6j(aAzqNK;b%fY4K?@*WyL^-*bA~QtGz^Z+>tO z`tN?%et5i_K}WQWV0AMNy5cUpe{Kzf?Bk2%?$UHR(gr@doBmw>r z0~ox(u0r|{|NcJr&+qpK2L~gTMxuh_V4l}u#NI(YU(az;x&yF3XthgB!tYjoFOR7jV<4{I8es6VEet;`@VkoojbK|MJ`4Z-2l2 Z{r30U-~X?_{|Nv9|NprqfuaDu0sulUMjrqG literal 0 HcmV?d00001 diff --git a/assets/prometheus-federator/prometheus-federator-3.0.2+up0.3.3.tgz b/assets/prometheus-federator/prometheus-federator-3.0.2+up0.3.3.tgz new file mode 100644 index 0000000000000000000000000000000000000000..379a6c9c0d752b159732e0180fe94356814b1620 GIT binary patch literal 20601 zcmZtNQ*bU$^e*V|i*4Js%^lmeZQHi9vt!%dv6CI!ww-T&=Rc>;)YMGhbl-JX_03wV zo|h;V8Xe?+Eg%gDow2kEv#GQ^r<^wrhY6bovzaP~wT>zer=o@ir@W?}jj@B7x0D{(>pr>aq9r%-H{i7L`&YU!A(Y$FGbxrJT&@vi_HDh`Nl5me>o z1uo{5F9|Q1JcZ{!of4k?`l{>}}!vFm*S% zIPu|yp-{z|Hqi5F@YaU$hU=lon~{B4@5lxnB}GY|jU^VX%6-&?F#;wvN0CQXO8g&x zHru+l?xcG1!ZL9=94Be!k6QBWR^^4?RaaiaV9Ku%2|-e%K7W~n!^5%rg}o?0v4sC_ z%B@G@CE{>Yfs|j1+8Y@ZJ<@}lcw85*O|yudJq^Q6jypZx9KtIoKah0aQD^Sl(?Rg) z(T$-{%*LJ0a1lSD*Hsf54MztWMRRJRg=!5Qv&_Ws2;AULf^{OMrM}t1rKWm;OOsDw zt4}+KV%bQw>;-y|F)5Pjp#RJjzp)EOt7tUvn|uGw#+3k1iQ+%Tbzr9J7#``Nck0PH zc1vkwCxtNFa8?wLG7DF-aMU1UJ;uR9h`dJ*DvH=vq_5V_A0AVlR3r5cemvvX=MSc_ z(XUu=qCS`BKlCwBkQp%}b(wJ?pfcIt!m^nB6pK&6B`0x;hQO zB^sduYGVSL$`?*0pJx@Rurt`{vOlcgjo`impU(D&D8B>!9`4}2f#`(-4|(#|+14Ss z3T2{Ts7UndxX_Uo3xwVRr!h_v)bE$;ib9BGqKp+Q^pGW-T>O^5Bq_*bX*)^aW2{6d zDt}!W#%0n{tlFqqtFmK?t?JQ=C)kn|-)qta^8Z+6M8nL`r~boX+l8i43pRvq zEU^+VV^@V|B*kl>^Bb|yvla{S5%9y*d?U+mn4YJ|D0Rrxg=zjGe@%25bGJ+|rlO|C?Z8*D#(6rN0<{OeQ zpM_^fpu>#Dr6g6#a*-YZK1Bv1L^(F8luf0H&OPxQr4th z$R=9yCwZ}N^d$zI;I0~qHOp-^JgLCCsu*G~7>g6)^LK1g(-4*zrJ;By#T@1Qde!Cl z1Ma|a6n363nPy)19-=NLi5Mf-#BU5J7oUwvG9w{pbIp|FQ=>!pvpub?5W_m@uZcRaIv!2J*Z%U>9y&n06XrouSPcv=0bBlC4@7)~Ugj^w8WponoD;bJ8Ju zIJrIHVr?K5okKJl{ag9^r$P*txnjsh3w)?SFZ~~K2G4Ss1x~VDv`T%Yr=QK%_5##z(I|(aF9C=Og44(G_b2t?QI}d*mS67*7z>m0q!_o1DVn(<+qK^L#Mli_>ZltzlFyK9Ts>)4vC{1l$lxUD<-$5aw_!|Y-v zs2|EtWQJBPr5Uswb2>8Z6%Do+qjitdwejNHW}X>Kz!yGpH8TI|a6yNSyc~{)q=QvL zKZv7lyBqd+Z?s&g)?53xu>^e^(E%Nn8kK7hgKPiN00%|AQWAyG`=O20qS`F{&bknk zf+O5>@x}b;Wm>rwL+S#M|D#YUS2?F%KD+Yr*scT(S;` zVT{^7sKN<70|IXh|4zKo-|XCUe)A!{61(48qo|KAipVp^bRD$hPTu%$t>f5}oYZdV zllayGk)P2d9X?Vy*Xx$3GICdz=LQ?jN4t)oXSyki5q* zzi-cuE{h|6UY{3=>-*0F3lAP|2;ONWee072Cq7@z6fm-xGtdg?o=NE}z&wjdeh zwYLj|?{FI@6YARNq8HRYuMS?woz)t}qT#`1l~@%bqG0=Qa$m&Wl|$oOtj5_LjhW!u z6Vk<8q6U^BWs;puI(igbAzrlTa8hxI5f(WlLHnzq&~bxj(1uhI;f%2cY7=^QHJ$S` zj_wn799Fg25lyHz_K6s}^x1xtnBmC@zPtEiV@Oa^0;od8kok9UwzFz>OY+qMV}VB- z0pZTPa)dj@A&Zmfp?q9BOTCxBfAN}6eIgp&PNXnN{!Ucz9(}4ZQp5r*rt;vStRZ$> ze%XsWqPu|%#l`8E_k|u4zpN;zE0ljR$SU&_&By(Ur)}qN=^74M*xKS=NTJNZFAVXK zVojdQ`ge3sQzSddR;=5-s@dI;!K2~3uV6P`Meq}L0_|d3@{gY+F=3C+P;c<)m*aeO zFtXy`J7QioQs$Zs7d@3mC$-+1nEWPuC39tq94NoOe=->RJimrb{<>z6B44lXTDNMT zGdGi$h?srd1{8VW?Q~q%ZHPP~g^NNmQgn1*4i6@K12`%52ncaqLu#fXmBL@zC#ra* z$&1l4M)+>{_(|7WHW(*jrl;#(3kCjtmY&WAi8XHqRU!U}J=<88*iKClX(v!d!Km{O zo>vl-M{>PR8?{j5R>VsuBT`l;qX7BC>a+A7!rX-*R4fV_KFBowU3TpHuKL{W*oVfW zU~H11t6c%gJGocpWv# zQ?&0N;~5^1AHWfZgAIS-{*uy#Fjm;JZuV|Pp{qHgRB=!8UY6Ng7#LExKjL4*9ksnm zN?5^;xrpDB zm&VlX0T;i3%KvC)tWSk4=Jc79GU5=hmMUz;O5;2D^Y)AVn-XpCWdhZM@9`^zFCGGaEr&8gbjCVT9MQ|J`vhpG7?9MkpeRISupq#--#fpX{dV1#LWUx)&4hebyU+f>KuV z3~G#d^3rsJ#T$2>d2n02+HKj^Uv695H8KSGg}<*|Fq{}kPXusaP89MlKjvXwYfS<1 zJIaQXb|M-mJHfU`{WEGJrdhOV3!p`Tl${8%o#Wc-H%q>pI0p!MTqmd37QpKuo4;VrXdV2zh6Z zM_iK_w$2$I3IJ9IeP$SJ`T~3FT<*8~8=&kr$VS_YF(K|UqmmMnyS1yj5KnFlYiB1q zm+e^+st;-rrg|-<3J2#}(58bAhSp}znFTa31HZOm{m`qjrc5K0Gz$;c2D z<||$~Lqwb0VidfeMyW!Lp&*s)-)7diU}wmTa0clNmRDp+7>_#sv7Ngek0nDa_i3hb z;jViezLJ~w%Ux^1m`pMY4j}AxBQsJ0$L|E!L{_S%8aE+=nk+OgJzLe>J!guU3IpDW zb~rn2-kb=1AX;*P9G7YidgoT56Fga{W^M#+Cm{zz!R8QTRvsXorEm}sm-T6fS3BOE zjzI>`&0AbHZANPx<5a7~8UKeaN_=Cwcc8=I7Y+2d5e}eBqYL(cO-GkYCjaIqBrI}hgW_Uu<*~I4rreJn zvPhny>L&RY=ti>_O-4Y2{m5hQH|t6BXI?tBqM%_xCq)jU>;t;pZpC(2FuA1x5L0IW zRC4@(==f#!9b2%MS)Roob;Q9)!K*m0`>koES2VijvkC*n&;WUYeFPSQA2%ifAJRMsh_ zy2>`JKb$&HmoO2`A+GGIAxH4%F}>iJ9BpKAmpdfQ>^e?jfGl|H_HFWuGhhF9Fv-;b zTsPb#(VA<#1T>jntxnQHseL#|+=4PK1iJ8{xGn65X%ZBM9#$n6Fn!TB!c0SM>(0^aKGP==4B3? z?r(6Xp~5WQ1RxT6Os0Wy`0aCDus1CKUu(uXy8odrrMI1VCbqG{NN z`Aun5#1uGY<4Z-CU7BOi)#AV=oAgTY9Qob{p0zZN+ab!OuOVlW)X^Lh& z^QRH?T}d7EKRwi-L9Wa4G9{x~RSO=oQDn^~U14S7i!aAVELUEyYP{LXS~4}Zbm*^H zbDqi3B%upYXRxkdV(j#cKc<#$*wjfik1YDC7V13wdN^+RnnBY4OshWJB?bmLeKZ&1 zh5&3fsK=7@#vx|_F~HbTL{ygd9UOP!-H212m=?Jj*gnaZ2Nlu_axaTfG0Q8wPi)Kf z^%zE8PkeNry>FcIEYU`V6$t$nP0ZdLZ4?U=81l1LZm>)a7HCAqsPH7a;ofZT)EkJE zyu5jd@??nUj@tpeJX9g^B zkrV<&TC)}rgBARC>fi?A1NZu7nIo91P$R72D*{AqJbFb8Nft5ePdCM{rewvj|y zhMG&6^dPwc96j7gQmUF()kVIOq$#kTnm1u$il14)`j(5oFBTVa@S-ho;YBC-zSZA3EPwIKP;k$B1P<{{ zJq*;N_>ftecLc?Lf>$I&6V&{H4Gi!r*jloH;5h>V&ffqP5BX2gHAuqAGUHi-tQ^qG z;K2)?nKB3KeVL?tdu{Onz>E*A5H_s6)s)u@>;$}?3wUm9D+Rk9i9B^sfNAscmtF6G zfZK_rkRiy8wVc8C{*^Y?Fnqj6OX#ZcqJgh&rk;z(qLVa1O0$lV;Ti4M``^d!)={Q% zCvCv7n=_J%Ry3P&%t;y_5@Eh_0UddRBfcUW>L>B7l}zxubaLas>sm4?x&F-=@^u=x z&V@H3@OxEB!HRVY4uC%pU|K7-To)LVtkUd;m& z*%f|uP+5Z!4PQL=enb53ev~DLjYDqKe6}o|Hv)6ZeuotT<-tD4uO-~a zNXm)?+v?^t@q?x37 zPD9II)lpte@r@S!mHxO@+Q{K23091H;};B+)W3f zBou`4L{U9o32enx#g-tlJP+5+j)S(1eRnFPz4)(Phli6TPT%?YFAfx+9p~ABCrHvp zl-Vl@JoF-d!j)bSXhK_4w)S_ei^k&L!Y=IPq{uQgd~GUFA-aQf$I=QxFb!+`x*Xqa8WoNm z&u;b(_k+R7z#Ar^6;PP$N8Oi;(Xue8*=eWb0V8;59B&HN7bwIW3b0AW1`mR4HD$Bv zmkTi4yI}wph$V;#sjR^3@DOrvaOSA+taTOwXH;oKOQ=#ZtGsua(#lz+k%S||dbx$| z2SYu>x+Tr3f7+S>OAexu*mu3;%H?fK5foJ8Dk|5lHO21W0o`)d#J_>RpcgJ=t^~*U z#D5Y@9<7ZKswu?7)on(Ro%dn!wH_iA=;u>cVi|+Umpm=f@Z)7~sO?gv!S&l2Z#Aif zIJTDTngMX~3`*j11M`l{re3A;abDa1l3a`HHP0PVVD@FNaXWP##L(XB$j_o|Oax9qr9S@njv@zewU#->*xum=WCyC{m(cZvI zF36y`^e!$Nq|I6X-Zyw_MLt6Vcj1=k98e>ZC0A0Y!4!F+f=8gXaJe}caovmi;nt=Z z%l}(vm8Wu5DxYN5(1cL};t%cNcMO@qeI!kd!gocN!SXphMZP@@q_bbUQ$l zwpC~eWv@cPTeHjka*&iSrVp(63G@s=7ZaoFn`d~)q^x4Dn(25 zneo7WtJ$X;2|rDSTZAbLr#$oh+R)kCAXOG#`+YF6kNIovvfSZ-bHEu+T=+5M@O?6u z@gU%~uR@brs>Ffu##1g5Q0)^(mfsBG0li*1BSvLPS)i{JgkpQ7Kd9 zq}WP$_QFD=FkwN?!OPXZ3ZUS{Jc*30!nZtrl5gdD&6jh6AtQTsf)z1ZC zXP|nS>c`239-Y$Y-3s?fM6}UAWkP8^IT{&}zaiV%J@Fw`bGUSI^Bxy$gPO%AYNQSN zvR>VW^M_u=2s$Eqh?{U*7dqw-@vuNmo;d?Ee~{NQHrbk%ryF=IDxy_!7xAzQNydpU z8|RQL>UWmgiG!Zo{Vys>wMac_z4~1Lu_qnBxp8A;?}UIwF)S-RtP5|9i?g;qhWaCd zDo}9^r%HA}gefuxdSYW4e;h;VRd9&46|#n72{X|Bz*S(LCo12QROYxTgc!-p-7kfu zC1j%ftpLh9Z?KB~%Cm zQ7~_54_hs5tV$L%WX4*#L5~CkmznDGRpdBiqB1GgN3BL|BcRl9wOtgYZI5Gj^c|J1 zALk;4H69f#6gJrYpH1*l&z4}qgfJZF6i2oae=x{6RS}jni<848@!lM@jH%{7XgWnj zY&97-$4|fuKYjZ*1J2PAUg2(dlbAZi{^oruahV#VLWSJ4-MpGnw#ztJwFf&i*(W^v z)rC?GOwG&Q=1LC3e@$fxKL4LHB+z821mq#&dyl=&=1}Mqj1`l7&x`;uiu${7x=-Oh|m7=xKQ;Io)WpDXoS-MPU$-%3mlV~!iB!ISJmn^Hw#gE3=RxLg+XhKz6#ENtFBjA##G?5)Y!{PN@A98 z|J8DqAAeSWPucu0A|+@`Xl?t9Mr@$yE>l1~6v&-VfKdb~?JS&TF%sawUFO>87Jsh}yrsB| zfDWbeUy~er5s66f{(vj&)Y=9AJ@=(LYq?tP@<&HQOpiGyjCIi3(hDlQk|Vn~c3X8b z@toJ|DU{%+iBU@d$~z$Ge8uQrtw@pM*mckCJ>Zsi{Wb=?A{^aq0^GlTQpd3p4jB^8 zMEN+|Ljp;gu8SlJFFH7p(Waq3P^G_~LJFalpLt)oe?zWI{KN*TK)vo76k0j zTZzy_9Bb}<7@mx6@3fk`8HaN5kL5ZE3%+KV(Itv%cVZBQ1Kz5mbf)mN_uE^>*N8Pf zLoy)#SXy69*Ygox1FdfeJ%?2w1|<-j=fF`TsRQ8|Cw$#hq&+RIdN@zzAgq%(xn;DY zOw_^BK;f=hEqwmq;wCbC_$WqufSBkqu2{dF#FbVxqq$}1xyhI-=C_(s?VovVuu2Hd zJwig(z|}y1hPv$Pz{EyR!OTSj%S%QhA}ABF<~$w_cK0QE8pRDY?bd&H3->pFFvT>% z&4@t~eo5BHJGnYT&VWUSqAE-BcY@Q~QW(<9*JcKvC)nwOYq-*0fQq;aN8<;FtpwSa z+gj?`5-;{XdNo?~-*|wbXj&}t9Ix64vvlh0=3BfCRaWsX$qlTDWsKcVSo?h?!S7!V zy}iC{7QYqEgjx%ASC%4(ldbbXaZ#V);ux)JaN&u>Y&&=WEF;r_j14{C>6$B7@G@l* zEN@X9#O*IY-=~gs;O)Y~edb4E;X$+5x0KrV_FYXm1@HzWGx5Zb{cO*jwtCF^9DEP|jfq(k+L1g=102rQWhNjEWd z|H)YsXpX&S4yJgz(P4fDG zs1`oQOQ))3v4~ZWKq@DnMeeb&B(KS8zjZ!OxSiuUQ{t3$7fNyV zMJL2@XV5-mmmzGq45{OwnF}&dAzmZ}h9}{{)j?l#_i3)NG3g5}iI0sb^q0WZ?Ejg& zT${!kTNDtJlWNY3QA*Qsot<09RRn<^u4vflwv8^#g^&9qG|5xx#4<1aD*|m13TNfH z7Q;jr3!l~J1Z<*#8jc*eBfnj{>2gs>DcLKc3nRI0o`Q_xm&*L-m`+{&;?z-CK(Ref|F1vA%Cr{k}BtmM7iqiIQ8xpnY_~qOp?+ zkxK~mRHBa{F>`PHGZ(attr4>-=u5{nN zg`%+8+g}9LgX#{=3nGWH$gxsqp2OJMVWt=}y%BoY-sa?X%GSejlA@8o2PZ;n&|xPV zgx|M4S=9rUw&VSl<*vkMZDZ7*h0$J+Y$z~>e6}xbgctEQ6_9=z6>N?)Sr$8Q*47+C zU5YH_{V?uT;$5``0UUQ&L46n7GH>@{sx6v#^qtNhuXDPW;RCFe^dp96;fGD9O6RT z%;xFO1+Oo+Kl$EWFCMMe!7xrN7IkK*s+vhfO<$=P(uQ-Ac?I0u_={O8Z$o^dpri)^{p*3Pg3B!|-*G#jJ+QOX-^yc@D zfBWJZ2mScauzl()ScRyrZ@R{Z;)%Ebww}333x$Z&{n1xp5)-2`C}QsfCL_8#mo=E2 zHT0;H{>8=FrVWKD;pUv_lqmdjX z_P{i;OC@^t>-P;@qhstgILkO?JYb^DcA*pOOfeVkDN^JN zSh4C-)@DvhtIdEA>9@peLXd(x9cn*pNJoxqyopu|Z-IfKozucxwuwI1NIzof85#d_C-W>&0*g+T?SpjvGF`Q zRP*}sxv_h}xs1}`!s{#Fu?dcHL5Xx@>LBzMQ9pYHRJ8f?i=F%*M3UsUAf%)ac%GgP zbQ#OS7+`yOYciGF1V1)(xg?K17dUs0KNHqGyPTR4seVJ1ayaufqYFc1_44rJq2cPC z!1nyLU^UinPXPOyG;#V^7kYtp zPP^P|G(CZn5uaKIVz|ZpvAUG*zw?!!yCp5dVD-~5&}E5w7N>fTtxg+N*M_grpP`$4j^4hP%hai1mHeCvN2{VOO3%xHLw89kgg?!P34$v{lb-gGX+MLPRxAS4d*_Ne_DT_hTNfn_ovN#~j(V=&?jS?H*Y+ zWIsy;wezweHZ&x1KJh?xIb^c0?Oy;!7%IfldrmI5Hhxo5#E)4u?QwrE10JH%qTUgr ze6h?ULiJn4H6B`%0W}6KE%Jt_w?mQg4E2dU4`5Nxq~!2kz06D>B7e_fqw0?_1LS{I zrO9}zW7f9KCex<{quqI5CyFwfCm~pcJ0X-W@m~sceUjdQjo+glB+%=iohRPBe^m%F zu#5i0-x4gJpa)}>sEEb?zLlys%%~an5?zkMmqJl;zk1b$9%)UgMp@T75+W5}T#uq( zC>I^T4~Fpke>_z2gJJkQvD9cop$C0};#M{4ZQ6{WQFT0zSNF+=JIiKZ6pZNR(4b$J zk#qk2A0yNruNO|Ji2B3g!IC3w9~U&s$z{;mkXg2 z1wy6lI7oYK+b$%0!@Txw#eGhJ^wtrYnHnMyf%mAn3fh09N?t|vw39$Tv|>Bb$zJTH zB=~oR+l=v&rKF^UoP8N+CQ*03v+mH_#ibNM8iY9a3xiZG5;>&?oX{x@*Pk?;Rdu-e--RG_=t!?W@b5P0|W^b`vO zs`~Bpk{OO*FIm#{s9U>msucbwzyK6foul*>sH9K!W}PAcO?poy^&X1b9d9b3b9X89 zx7`2amtz5oJgnF!)PM4e>`FDuxENOdb~RC}KY8x^9+=591xu-36U$H}ta}ICU-*Rv zB+Ae8s|4Qt67-$LM(OqCYG9E0g@kj*q!9=l@vim#-|WjwbumtIJnsJ=d;x4zs+*|8 zmXx1vw(p2tG^DlR!yF#^}DZcf$LQ7M0ZX7-?$4|#@#Q*x@|s9 z*81TsZJy#7C#(7DvRYqj4!?TFNs9Am88KKC52KSq#kC+wO6o41sm|!7gzIeCC`!uM z3pxKigOy>-v%8Ood?_RQ1hlxG-qA=(e%J(z!7({j+*7MA9deo-#yTlGV|`{Pg3F=1 z+GFWUtVk``7Ut%L*VW0YhK6wFua*~=qJOvgYRb|_)ji|pnS+$l09%TDHA3^mO3DA0 zTnfQMEt^3N|0B7|OKcY>h-%rQnYEYUpmmur&;7>L z>CS6-ww(KsL~eV1+Yxvr_3RRrlq#W2csJA)Bv3d)de_g3Y_Rr4V)gE<#0q*Tfb`)J z>9g5KIkP$KJZq4iobzNTvEkz+SE7UU;<5ezBv!n^D~Y1-q5>VfkETu+18$grx}ST9 zl_0tYz{I|Qw+;qi-~aidj4<@0fKk4aRE6{`eF6keRqt=*SPXG0-B^V+9S zboH33l`9_z%siblC=>!7e?AfgejHyt9ApX=_IGu8y&qUNocLf1dIS3eM@Kw075jbe z+P_4;n-+>}zRUVIJYN=Z;HvdgMB0XUwO}8qLPZz>u9B*h7&uYoYc38+Q8ju;7w56f zg|kwP#W+C}-&Eunt(E&nF#zegho{dE8QoL|%3fO5@d?e_FP)uTQ=dq{lVv6r=3O!N z2kesZPAf!{Kd55=Mr85Ra(8WeA65GiLps}XVAHdJCq5?)2TlpnI5=w-p(nf$Fo`EJ zw9pNHw)|)Y?_s&y`)~qypXT@MaD5)A`19jQeU#r0a9i_spNR$Iy;t_O|3OF`Y!LW- zjcUNe0ynTFQ|I;YUt)3G#CKiX^0lY_&6ajbVGHCuzP7!YHr4;WF7LDL^rVzGW4?F0Q7b7~?$%CsoYu3*NevBlYRZlfro+{>Y;vSi`;soF$ZumWA zuGR+x++STizdy#{I{a2IfJH>W;I~l{A=f;$n9}ErN+-4|h!Bv`_khj=XTJ(~u7gQ? zb;jp{o}S6GoK{*l$k*0*I$*Y%L6l z46kdPJ-0sNXpJ3E?ENDaD1KaNpS;Pr#8DYX1V++oqu6(es$xju1yz{a5xV2nSlhO9 zceFM7SCo4`d3CuCTuMJuw{XV#idqz^IWpJM|E9k9CYx2d-0mgU)PI|J_!O;JW!MfN zBO3I3&#>Ae)%dI0`?n(3ay-`aJl?d0%};eq!T$YhXE5shJ{7wP+*3~{HPy8X+PnxP zB*v6^$ZOGe<)WNka1qdxEtPvc{YrW+B!9WCEt?$2xtsEIRIeZ=fpe#3NWZ<;*{D!Rm z;|Ya8QiOQ`-QT{Ha#KA-GiN;t3=fy=av0F6k4mCQMdN!54TV}TVt&ihCD4%nLbzt9 z)Cow4zu6kY34Oo=e;_`b^baq?Ec6c@noR63LKt0I-yH_rlH>Wkf4u_zg1$~w4Crb0+sAZah%ZrwJgej1{77(~x|0L1mGMFY( zWa#y4?d20BeaaA;&_u=#8W6}wIs{jnAwC=7qIqIlCn@HS+0fcp5^KrKb~TS9aThdL z{Q`TL%PjQ;1ztk8x0(Od4{ctr?8k6}UNo==#~fUr0^Odz>07}*`w#_ zunnIIz8ebme-eKf8b5IY8Yq5FpPrQIet?%!h6g~Q2`cia7HTz$d7;{Pf;xb~1PUk<|F6m!HNrN8@9Ik)ZTsMRX$x=~J7 z@uz8ic5Fks`6y_3a{df=$OS20?QiCh{ygrS|5FL>86|{gs|*IM%Ie3i?Vq{sthdkToKrUS7>u77vdPOJf#t!sfbcpC_3QKOY9e+% zuB%9S#F+e6NZ45iK`R%Bwdb*R-$??G^j3VN+v9gTBf67o%Gb9s+65@5UXH$6NR3|0 zp`Idu(irlI%PI%`4}oZF6={n;@0mkDM*w79FF)sZCHWjeY$P9$giUWZ196GUB__3bwmQVdX({eF2f@tC6YG zQ)FRvxM{GL`+W3RWwwvJW_tgTvYE@5OxdbaGI;1&30Oa-j)mMN!@2C4-Bqr?>)}PT ztFDrrr~u&FWjnY^py`10#XR~I_5Q0amTn6qa7PSlsY4DK}9xq^i$ zZt07VigtWz@J?>w!iNd|Uq4sNkB?YbOh0c^Q&X#0Oon>jF3;cVO#cP!fDhm6%dgin zwK$?TCT1G0J`ZP&v{*A_iH~oO>xn{zhTjhlbEowN`usxuQ}ipkrM97caw{+IEuFlKN^CthMHAWnvsnl6wXE6Z{TLvy-&K5G+^X!h&3-6` zOmySS4HzCNfS@N8>XVsuR`Adic=Pl1B5QZ=c;*`TPBiZ{8sAI*BmX|pe{g{;YRXu;3U|XNoHN; zZ~j49qQYViKajKC!)9k<+qFzN;oT5Zst`w~r>N|hm`TCXqtWEspg&hMG2!diU>G@= zqoCa`1a6+?R_QGp=}XRI{s7O;ltY1qqB8SXKR_5kd8x@RNnhKP9v}h_d#&s9r4fO{ zuiEnwpb`MJ9ej`RR?z)Ot3JRJV_~J*TgVfq85UXSD`~!wz_P`=Ww!I`DaPQo0i*bl z1ZkJ=Piy%~7%nNkV1OOPTMMv6{AX7n;dvE&-B-NcA^Xdr*nQTrT(jrhcab8NH%Q>( z)0iUZc?-mi-1ES*XPZX&n-XnZ;yc_3D+YBOi^Q#k6W%LaO@ z74RqfgmgIO11z`xf!ae2`OZ6u3b?*CF$f`<#0}s6ldriZ6 z`(=;eV|U@O*y88EepxZ0?|;Rynl0adlmMjVgy!XnRxwV!;R?f|s8-}2`0OyR)=MfIOWGCjj&aRjEUA7)93ft zp<76ZTT}L^jDSo{$e7nL&dL=S?cKYlO=sX9G35dby%Y;T-Oh^DQT}ngdI&)LJKv~s zBt-Nmz+8>1y>l2I0uB{6xYe3J&cTI}h?>>dHAi0mIL8S`+NZ6?2*ioyk{GS(w`SWm zLKc_z3oJYcdyNwumaXd8%)S3qPBm5V^`_zy*DMk0l3H~on3AID!v&J%To*BiaTx7? zu@$%D`ry*utARC#*%ZuWQ3UV@*InMEne424im83A=t_0S0%8%*x)2*bwm`ec49-`_ z8J1!;n_-fU=lILb*HA}6KrFVLwTVKGGU?LpD;GQ3f6|nE3XQWwY0S&W%jG|fiuJY^ z={p#dbX4|XX2_BW4$kliA9(PtHV!$Pp&(uvJ@uEFjRo#{3~>88Zdl_P(pCDSiLQTi ziM+KNo+u2D`&M2hiMe;vFhRd%h3hwYU!%;*`LX}N4PXn8bs-XB5z^Ts*)g8*KdUVd z$1^{U*%v7uWif++ME~Oi$on6d3aOceU^*<0N6hSHx3d;exm>p{_~uNCIn)O}%B$oP z_wk=Awk^)?T=p=SMYEFrHPJWZ;YR-~mU4hEePXA>3pSUr(}EPq=bW? zvFSZs-L$71CJ)hU7e0{BTsnvlg)#JC>pRIiF(Fz)&sLpZBMp}9z6S>(TJH!k2y5n) ziHl?Lh}MPyk5ZXrGEr1G{_~Ec8F`c_lt+}p_85Ooev{af1sKBTt-y}T;wdVDSyXqE z@WDEYILVp6!_ct1N{-f-&Mud;7`RW7ORFpl(}u$oRoxnoZcCm_!E5LymM>7GP}j+T zIWom*qAybg*a~q6)jiQuB)P$o<1zp24Nzo3GS$CKq6m>I=xvzZg;DqI;iOym^0HuA z+c>c&lW^>VB42Sk_YN`sejDT)Sb*DKAp11%Ivb)M*dHqLVVP zt&{CKwbn~;D$9^{y}saY(LCKK}+%LNd&Izf<>k{XO|knRp)n*ySg(g#OKKInO#_qyKe z{xj~+_op~o>fOZG3QQZnC9V6U;ii8z@d&79wO(qu$SRHJOz>U*s`l&2b@la&baT^< zh7{Xv@hf*?@6HXr)O07p>6RIcvOdv{w1mNIKJj5QTIT5}L$t-|Tr+}0qH<#mC_7ku z_cTdDg(|6RkOkW$L;Lm9HeXzVbi#hO$&eW0rdgIOF@>gSqV}y`Q4o4J!QzEARs{U* zhulw@0F7xMVTf07%O~B((bo~;38kyVQz2H@=-@oopJMTfJn5baab&BL(eBMi`ZM5= zFKYdP_-mqtQjm1zEUUsmKuXOV3J-TqnjpGdXvgHN(A>>Pr@r-Oqqmao%=XHDf`>|; zvQ4^I-*ApNd(p!kz6d{^MxOKcHQa@p7S!5oI6>>;AmbyXXkQ4GV$}~_7ajzb=kq&W zRQZtR_n<)|@sH2XCzAzqTxxFF(92ZT)xNeCWQ|1On?&-_GarnF9~aO>Z|p8t;<1#I zW#)2{k7aLERjJ02Y;@xHeslDzuv0 z%zt&P!Jy7G96&HP5G60iORpNf6yf!yQDG8bA%MfUT^JYh>-+kx6v}5P2;Kzo3QA1E zgstPVi>Tf~3f8AgI920am(21owv#Uay`pzH9p!PbfoMb~$_QS^W-?vp1{gDCfJLJR4b3_ik6~~(v(l3H8zIKK z6;7pPsOQdL@XJcgRc;&6M>65942{mf_L-yiVTA%wn9+L8I! z%bFD!S-GSAam=%yQBkV7V3doig3j+mmY1GC(=?vzj3I4mM*oXNdmt!#g!3-MERxrn zwfK0Pj(e_tTp2>`qxZODqaEGAnk^M zr3Q?jnx)Z3&Hm$j*e9m=#l>AgRYg-gaK>aeuas86_A8MInZ%zuXxeVsIa7o<^p1{^AcnZH|kX zzj&WW(N^O_az=SHJf6)h=bXEGYCN9>fe0=VcReK5C+pH^wFU+ON;7U-)v~Vzp&R4) z&y){i{>Wsf`9@XaJabgy^X9p78$UG}FPG&761pkhWz~*@&C8CM>(X}lxUlr=wu}3a z&3(WV>DQp+S5j(OK5az3!re0v18yD%CU#?jo6<4Ap*0d_cJsCk^ z=r>lhb0;FGRx$1L>M8uU)q|)o=-WPf9&(OWGPvkZ{kgE*Eoq$6cJA4 zGb9C3=M87`VkeXby19h$0MgkhuB0U$VkbkGD%tz)*^&E~kz@Ud+;uUQ^cix4pmhoE zI)H?7I!7R{{t;i)n_iv zO-w+;=E4y2G}=`dmkHakW?kENjywyrNO22ot%4@=XuV`E+79t zSvDfXXfZA}@7D62S%3~RJvw~Y&P=&Jn5|u8?fa4kljce{ka+`G^#-aQca@=gRxl{3 z(LD$cJiOY0gZ2`qFuP^}{@W#F>H(%NdcYtItSU?bCv|T!Mc>N8RZB79TJpu6zzl{o zL`*s!i|ow`Clj%L4nLCDDdA7e2-COseHj-c)_yelA4<1!P?N1+Z7DZR?=)K~JPUzN zxPE_R`C)9|;ubl=rSi;Lmv;7Gw|O(T-%sLwzywnE1V~x3k=gh+3BGDOY`%7yZegYo zN>JD?^>dhJ%5`nL}ZO0^W^_)tss=OgUo$YQlVL~@?Aocxszm0Mzq%Kzh>_2zH9P(l* ziR~LIzWbA{9wvc#Sr-lMJJW(RE~ypzpXnA~;P$btB9x36>sMSIi~m^m`#$cgMvsGP z;Rp1N{aRn*a_XgVuaqU{BnP@Qj+8a7R4IWFlpL7*gIn``49hIVz=}|#=UxoyF(1&d zuT-KfkfzYKy6OG^gZuY)hvEK{xcMhaJlAglMrtPE{hTf}|0!)*ch^{n4&E@YWUu(c z+Yr?rW>&kbK^`5OoluL16Poi!GRtmZbz?iyZ)%wQ0gd7*zwWHv6N`yHiNxL`vzwZF-ASRp&i)H1R2QH2nWd@fCXXyfNW^`_Kb}(ZF zo9ryb7DmK8RkAS@u618s39srlqR{?&DPc2wq3C%W%P#4c%6KCxw?)f$={naQx!u$2 z{_mx$j&L!f=#{E)I3_=m*W!?mVX-FGMb**2?Jp~1^0*;7uF@b_ITG*Z-%z&4Z&|^j zg=Ne2+llA`PH~Z>SDThE*k={Wd4B~5NA)<(Vz={`4 zcy%jYebrju7g`liiErHoN$gq7hn(Rh_`^QuwylTF6oAuvYpvThknY1*%1TPk^H(ZL zK_~mr%Z-wgit(!ui!~WIIncQi7RKWtpWF6~!Z(-V4sE5T0EZgMTJ$+}2d&{-O&#yw zBD(M@p#Y-C+q@pQbovX`-ohy06|}fI0~=U!^b$jeL+V*ugFpS=ZaN<9vd@Lj{u5cV zaBN3XRYvy;htfjr}e}BQ9Tz0S*|lMnZdA$xKscevUrb~pB;B9jGZsN z=UAw%yfGrIj=&S;SX8d1#xo7(Xbs!A^@OIzx%86@F36VL_-drnxLRO>ANf*hH$31) z2)o2ykr@c$Cb$;X3ewa)?7B2QQ&Giq7h9PUWPYpQW-&hRb%^&h3x1iogZV0#F;hgm zQSVK*OyYGiU3`cTxJVi{=w4h!g8y2YnBCL`2ELUtp;Mi-Ui_(jgr}b~{7J&ez47Av z`M)d}tv+Bb^`h7%B{^1-p3~?PMG&!u-%nFJM4?nk5m3wG*c7eHCPsqtv4DjRH~yrb z+2cYF_v*?AU=y4Qd^60?bi~u~+ybuj@k2DVU5q(N4pXzs07qN_30VRhUeogg?M3k6 z?ot3UB4%UfZ&TNLIil>2fPU}TA=yl6rdQlZH@0NelMuc!YeKB6*^nGO`Gf>@%6H5VCkr_2(Q;(M11=7grRmhy8mgK_*02Yjt_#r`69R^%{Dd3tbfgP#?} zo4SDNr+W+)z4J{6|B`_8JX%iH-{Y(4i5Nq`h~t|zivTX%w=)$*q{dMV1vXG(JbUOuZ7UU*5E@Y>|?kqEdZ z5L&jkBW*P4x|7N8%#`yp7zfRj6E}PoHT*EwL1>4`y${Z#yslN!$16bcfu9bsHrhWk zTRnQ)O-3KlsyynfUpL7dMMEy&X^}D>K^Y$eLZ(5mty z0<)#4Njwc*q|g=+$Zq7Y5{2Ad z)lDc zVQyr3R8em|NM&qo0PKBxbKJJFr~l@s=tHeblCQ{l=;bqYeQI@N#nE0{R!K@yr+iZi zhQ!Pu;t&i0N}eNYYyaPGxu0YUFB0T%o>E43JV;6&60gRq(P(u4n4&ls;M7AAPC1GD zUYL?woU+F80Hxd?p<(pmv7y;)HV^jp*B+fq9#wIu&63)Cl)y^G>LwZ+FhFJk6O4|ORCZz38p(Di+Mhxp#Pg+a0S@|fz z9SEX~ah!Ub(x^6BmN1@TG%Uz#HJg6ZZ#gI-Kbv>AZ3iVu{;2xV32}ns5XS+*tmAkP z&^RPqrbtpcY;%l;%*Qe6MmTf;R4a-o2Oz}>WrWjo)PWv}aHrAm8-ZXAtWipslh^dy zezRN|L%{&^v5M5|8I|OTiuTLahnNK^NmK<_0}MkHqdrcdM^oSfl7?P_Qa*w^+#OJQ z11S!$91X}A4uQmQp3))a1Dr86Uv&u%2}GF#&J)&YXgYj7wqi&SW0MEjIAs$=c5 z+rD#yM|U(0Wv35mESg5*zT*MX^@wUbLz**V_E;m|_FI12dCK)ak^kSKD8uYYHh@|D ze|NW4<^Q|;tu_B&Mc9D1xQDWcgBozTekM!`-`Rkx0bxKGAo$nOhxcBOrbEOz4&}lX z3Etu$LMet@loGMt8E^_+42f7aV%>2HBbuea@i2)H$IN$}enh({(i;$uu=oX#utfa; z(7dAqJrpr4Uv2;&U`TPFFrJR!Zh%t^D2jj#Q6Dq7BT*zP&Z0=rm(rok$3~3fp>Fks z&fWpnbpXVc6!1vZYO^KXBMqHE#&gu~z-`NKH~qFF=wX-(*27d+0_ll{!X1jaB$Z`d zPjM67B_w7XN#aDW$Rd#BjVwU{h7|Du7R(IAx^=Q}S%esZJiF$oGS|mjoTenivO!M4 zqD7M`o#-guT8iOmh8<^P1I}oOAwpdosVR}r5D;Ue9}jWNMT!gDE4ZZSmo&y=G=(rg z1J=kGPQ89cLfnXHh&_fQ90-2ha5gp^@#<2(p=k&1A4G}o3FAO}AmV@{604r#G{Pz3 z7`9UUM@CZ2;J@yt9czi^ny|MTV9IQw{Ya_RHWeAzl?@c(AJ zby(s5hX<|0HUD2lm^A)^!Nl~=;!!k~Tue)>e{;^I=nE)x3q(LSag%mgEpypA$#m*~7|D4L4il(C)^eXP<|XL@jhh0JMdG=%&q z1;>ugD_-DU4+p#h?isz*1c=u(5#48+((WJtJ~e}utScO-k_?b)SBOX5DT1P9x`bJ_!~nNo!WG#2Z?oE_vl z!t)c7REQvPrW3?#YA1HANK@`yi!GSQygzl4iShIFx0pZr_y3TzcizlDpaqS2FQ`KN z3)Mv`uc5Q_azWw>KIgq;Zlx?$>HO2H{%2{@S)(adh}IK{k(48P}N@rD_`*TwqQOwmm|%2lc&4kby2&E<-@`}>+KU*M07a$IThcqvNl7<%22 zD#*;5OPt=4pqgKGnK?Pn?6hhty;>5K02{(Z zpjpfXd&?M})+Cw!NF_g;#EHtu{6JM+*$%eG8eDKUe|u6{vE4#z{hLx<{llHBfC+2d zXjb}!53;Tw&|yQ&di94emSNV|+uz;akiP>u6vr)Z@9^;O@b&KBTh31m^(%uKpeXWU z%s38R*op|_;DdYl(?5@YJi55LK0Eqw;(~j1^5Ojb(bb8&Er`K>-=Cw9eu{c1MsEJA z=tx=2*l3oDG0Z)rT<~@hhxjx2(99!P)>UlGifzFcU`a%{R2#W|QH(jd*s0rjtk!dO z-pKm&2$|Rv#eHnmSIK0`GtYj_D4L!s$k+-=z`;FM97#W^iMtgc#*6y4x4~5exp^I~ zURbW8S3iMk%V#>yezjelRcczj!%7($X^ZzP>UQ0rL+=?lmu!gtGk>e|FJLdyEL?UpHI0!byfyUTq{e~Iig2R|a z%Bj&VS_e8P$%yh4XkT=vT@?5z;{%$Kf6EiKfAd-iV=d=~#9;@H)pWg}5q5@{qY!b_ z$uFJjgdA(nhld9bg+mg1Rgp&B$viwbpoOmbvZ7p^`~tDqEMg9ODjQ6p9`*Yv?jw#p zP89(iaCw?x*GaPo%j+8tP(t3NG)tJd7W34#6+ns^&C&o{F*%=Rr4+I)6GY}RXSbVh zV484?)2?0pE7$QnZ{=fzL&BePh(!JC5t=qaw*=ok=Lm`Vzy1;OV6%_;rui-8lKkNg z@nEp|C(9Xg?EeFlhB(II(zYNo_W#4(>iu84d9b^-|6570*+N%LxPk@zFWOyh=_jHb zk0IMGVO&S>cwSJkT?BBAUT8q+joX2QGVW;^V@Ew@Nk2s)9#8ItIKmuz35`fFayuYj zvMdNN4so~xUEHH7_T+1+O&~OOSb_slcNYaWwAXu2hGJi6Igp#TD9TJa`Fl)(v1qTV zQi3ccd~{4>jz4o#I6#`kN9>Ho7nE{s&{2x|iQ!aQ5jOSv6pJOm&t=0qaE5QCji(3| zD-Dat+G_=Sj9%BA)a>H4UXIg$p#(YzmCa4IjbvYTywn{T<_QW0@eFX$7N=PcQm~zzdSuZF26WGeOtUJgj2;vvIuuz^Fzk?g(cC7(=o&Eg@jj1C(|6oBN?A1 zNra{OVPv26t3|4lw!HGO7?sq$wQ^Zzh(34VV1Ivi-vQ_`wSJzcxD=!CV;qe{!+szU zR^nXICAT!nawh7*Cc{CB`KB|4JI1QxN!%afih4{QGk3N0<)J8|cjE7y!cIP2#1gWM zz5z(wPch4zDBt1nS88@%jgq1_PE}6*bfO-f`#Am0g*o&;w33z}vB zeb8)H^gp|;=I&bmvx=~xx5z$+tq8}k6;XPVCE!jIvs`d3@xTRF{J*^oO?hFnvB<7# z633LoR@vt8==@Z(HRk__7`WwKxiPjMtJ~U*;uh*^fPtg5k9lq&8dA(;Wdn4pY*EIF zfur-&T#EHU+qs8?38LZr@|*#QZC@OlPiM`ShgU&prMV*Z;-I(c2Fv z{xE!e9kcxZ+WS@izx}Wu1$#$ECU$mp` zg?6-s)0pGFR+s&{W$c-f-0L>?6jRpBBoB|WeQ`Rr-Vq(fkj|9qBWr%|T ziU}Js-~+^^ZLZ)bG2-MFMxYl(T>eAbP6UJP>%{}%If)!utw5>IwF#=Yb%066e@~WbGOB&4h{kA0>$=&L>0s=sjQt|KHo)-L3k6@9#F( z{C^c;&HvZ@|C`|d(kuNZsl^htb85TF+_wDo{(m^mB}ZxwlXxt82R@zC5R9x3=USUx zbU}?1dqCp=C!95m4!n`j&@&HOY1Y_Qr-?4cF{F4%Z#CY4Y`I0p(fO$lj&sBSi33I! z@f~>eN*^DxX>EJ;3btq}1`a_l#cYt9&JpoytSiiUXL|=SCRT%(r1*1!1MNYS;Iv26 zp;!_m_LKvM))6Yh{Cuw75k7$5CLZE`Q?LgVev?|O-^5{#h~pJg{gI^jme7oeb)ggT zG{e97&j0&A|JV5kyn1y;IqtlA<@^IqOUEkh!n6Y@9_7U-hbXo1?hpgS7!3##t&)U_ zwC9)S#T`b?(2y8<=vH)R4-haC_ah9dfvWh6%^2!0#jXlRQ14Yu$tjt!o~5XBjaq7eiUMlobb(bjD-os45wgv|-}M^Tt@ z2$S=ro6;K`1LKHe*>9xe+zl)X2C9#HSt?N|WEq-K(^#Wx2(EWW(;L{*1jl+-{v%sIgMb|sdlSPuhwC%QN93|Vfe z#Y$@`ILJ>4jm0dKEMnS}=2YomcPtMo*6&f=iC#5T)Vz#4uNw zDF6SFVWxK8(PI&yX6b)gyX~s{f4J9P%m1qg0phq%Q)0{#EVwf3Vn^{vuZZ;2Rh5&^ z5|!J4K1~3a(>I=MJ>_x#&pgEGv%&p9k^u1vi8~+8>hWb69#D( z$71iW?mO&V;r(1RF0Q9G?m%keAn(TS9e$qb?%HFEZDEw%9#7?n(cL%Crsr3 zV<3;qBf`$Q|J!fZ_>DL66iK zKrvG(GK-SrHd81;N{Wd&RVmIGN@@lA^tnfLv9Q4UUm6i~#`@oF@9kFC|KaXh{#!|? z#s9Q~x33EFa~gE#1O!YQ3#L1%wAOCkv0cwnEQM-i=M*HF^wp5ir9(WFmABfjKM+UZ0a?k~m1ON^ z^8;}lZAUE9DfvyCZk(P=yQ-9T;C1u$->YDu1IoByY4X2vO6lDYYb!23z+V37Wocwb zGi~Lyp#;TirBT1`j*NC*UMKWqiY-?entcJ+;^gw#0FB58hHtSDHPr)0JY(u-HXNc< zz3!2?k5dU^BcIbO?oxVlou!ex?Ld{9jwB$W{RrRUs7xDdy*>HoPw(IhoSyyg5x&4L zM;B+OXX3@l#l^=9_yWflr&p)PNAE?+rjU+IZutQvhz~l@pb58m>HtSWr5tIf_4a#P z^0i__#`F{0RGHVbQ0gn-gY($Wr%hT5(_#J{2eo=#&2jNcgdEkyjMQTR0|t} zixmar#+yd9m5JY{Zxs>|ry_wmxfs6~r{a#(D$0>ImZqR2C1P0BkX1`WQKZan7?X&< zQjv+`QBjRSnMr79YSbpsrpjKqzPz5QX1PjzuUb&OHD_&~PDrZse;9s5|He^(7|*@D zWX-DcCmD@CA9U}NWA{3!*CR3EBU$`&Z;^lLuz>uxG$Qb<^Iv;^-2QWY{##A3jz#&E zU5@$d!|?zIH}XP5?X=Jx8I4n?z)J_uD`3kf8G*}No`i-Bya?RMXku?_z{J+tq(3Vx zwf+}G1fFI8)2`b8wD+0^Yy9t(g!}u(E4U@Yj=Y^0CpvLRcrz5;5)5z$#({0uFu`k> z;BOu#*iv!)2}r^9Bz`|w!F5o2bHN3BkHHJ}Y)npXX~bY}PRL+yVa(uK0w0MQJdOE{ zPr;)|Ff;YxoQm~@R@zb34Ix&>dBZH?#FFj`vq|X%6VPFo#5mNJJTupsEW{9949U2> zSmFO;+PyK9xOt$2En=bt0ddFpg@pbb$07a`fs+>n;MF5M8J+hlMmqMs0@tYTA}l5U z=OAbET?WmN|J#R$WAgvr+W&Jk!Jr$HQ0$Qs7uS^vZx`ItvmZXX;A(}p3$9dnyWkpy zcWFQF*+)w-&xiWYhrD+(go2bs>T*~m0XX-(<*b%U(`zwl^kA_nfU`Rz`k zq-29$vSzZ%2~fPL9Z6Af=OK605b`?UZXnH|Yse)`oVvyv+js>QSbDzNR7v7hZip+2 zkcPI#9Nfsj#&9Uln>)Tr4+tOf&Cv{(kg!>f2R>jRLF1 z)HH@jP=E@|q(H)AOx?80Kn4@AH)`aArQkPGY@u(YcnbK&9VYz&cRS$jy5(qdC0GD+ z++dtIrh|(6pALNSWY$kVUR|9_j1`88>ic=QN#|E}?0j?o-8J zPt_tOk-Q<9x-@kW$HmQ|sL#Bv;HOzo>^FF{BiV-FPt(_Ro>}IChX+}Q%v)d0Cw229 zoqu#=eo*%R^?gyj#5Bdcr+tYX2_|0doleGxq<3)Zl!C%Nj{doWhh(>$#mMxsM$AiaLG&Mu~U#*jgi$DaLy{KXAPsTKq-L*#;G>a z5_F{&NIKTbMdv-th5)ry1J;=2K%dLte9Eko;XPHfLiL;m>N%FBuv{wVluoSWeljf_ zaGsPRtPVN1ZHerVq?FTuMjg01KDXl3=zWkti1pQt4WkGW6!d(S;f<9E$G3LzrtqWl z^Xub}XFr_2yZ+(y{Yj}7t&6YUw^a?cf@Hk#^NWuk+E*t>A1<%YKVDqTU$~Ua<}Lci zkCzJ+Err8*i$4B%c6M@n_3>h%eu)jn?|hZKyZHFi`SsEJlZ&g%>wo@h{;H}n+q@ON z|M>3x$Fb1%4T?iFl!9C>Zx-J$=0nS%hl3uhASz?xFM8LKF5)c3&+NS zxwhJN0b`%rDw&m)!A4D(h*=F#4<%?*I~^*o2ojs_+!S`g0u<>}{CL{Hy1eRCm`9k> zr25W-qxbKfiQW2Gxa``Vt%7D3iKb&`Pbj%^$|`NZ#_Y_vXmiD`&I6LiXI}opmUOrOy8en^W_z!Yu!f=I)sNU%S1(KL4*G zJS#%-#1oW+8eN&fpic6bEk&$hj{hu}th+i%MRLn$o*f{*e!>&KM`VodfS!;*q^Mo`igLA*I_kb zDfv(9eLekcl3B9gEdP()R@MK1f4{Ys|5g$n|2d?u<@)Ejd-B^1k6Hioo&jgA|HJnF znE%J&+Wu=LVcPnCo`vtsvGD6oamu;9q?J;$R79v6tm`UIJG9rNw=G^k8&ZwkSk>KB zl`g)@*UOK)t6lV-c7gF;Jax!^b!pNz{czkpu~r{#e*|{I{3amz<{9*#K?i|MT%Bgd}B--zU5w z|0?nwzx(gBA^(~egMLH&F^}vvZWIM|QE-zb()W!yuY7iwIIsLJx^Z6le3&?|{%VMSwm==T^uN2!s{MDn z-CBSDc_rcgr3Vaiv40Nm5MKv0qA7zna4&bB%@i{l-QsW)I_=+-WpeFu+6P)3o2fpE-`@r-V#;+1z8%T9M=a zrF3$XKJIVcSgn5fAa6dczVS^YJUqbtOLa>(p6t_*^{?e^m;tQ$)ntf6(u0=|h7to@ zGo6gB3;nQdmvMg&MLnt6rZJL4& zVQvRL-RH_-7X_T$5z^i6Gs%+f;vSH(!m^VHHm?&ZUM>7rgC33b4gmc3RnGE!Pd zRBMe5(11zyuw!>~xl$v2O|JMfmYV;$s?JNBw|X<7CRcxHFp}n}KQC_H`Ym{-$t1kB zFr4P0!Y*_&ArqRbP@v{&teUi?8Y<0`MmX0RKu0#dM^m`LBOvj_R67$$yT1o*u`>^K zcA=kcuw)r!LU9}lYF<8Ay;2yY)HTF%SD4kBa?+=4l2!8j?fTU&-yNd#CZu<j$oUo3V0YaZ;c^*<{KUpfA(nto-^v33djwdwE9#C}!Zx|tvM zb^HT38ue8_C{+_mq)UPP9&+v=-SV4$>l#dz!62EGR~Qb_xd{-4oNs-H1={ITQ8Nlry;#+MA+M=|JrXl zjv}d};UjxE#Zsp`7fWh%Bid~Y5#u;*;Lm87M7e9eT&U8riQbYBYaCYDCiV3+jkCBU z*sVp3;n1_&tQfU3?#d$wNoaUv?L?B=q_8H-t!(SyOG_}K&8Abe!PU}Y8T+%@oC_KE z@2?LJ9A((8t^jtM&1TC6>@@(lzhKCi`Y=<913DbyIFydnoJyZ5(i`P?Cxvyrbl0Z& zx0=LCW6GJF1%zfy799*I32=@Mnz8&lLMUPsNF1bM>y(f^ae$fmtWuiAVzT)we?jce7xUnM9#rkWcH4Vv{LhtyHL&Lz*z@ZK_FOq+p+5co zX>f(hLY7)|JuzHq9dX|eSgG#U#ehm5gHvjwfG&+v`WFT%oq^)1$IC|WTqzD`37A9C zVmRyj+(z;1huF-!JNFr%(c#rb&I_Gd;#>YQNTxQ)>r=s#hK1$-LfdJ<+)ck2n5F-3 z?HyL+|AS_8ZU42BAUJ?jCH6>!rKUegBC)+Mi8c8|P?O^_SQ%z=Y|nl&h{?K|C%{XN z6EWF?rLt*#<*<@sIp?^N7WnFx7{1(IUNtGdW7w{Z&TkyHjWzp*`KdSA7ibcwL9-xk3!Hr$I4WE!5bPfQ>yZ^x8bdKOwYgO?9(yU?*{iNo%BH_AwcCtp>TU zCi{GiSQf5z4w~3Rg36eR-w*bywyUwjB<$CS+~OEBcAnBMF4rgOd53uwrBQ!s*=vZb z)i_ywg}@Y}kStk;$UB;+l4%&WHc&0*fswi@#o9=>o+{2tF}9EjE2F2x{j5b3Eu-*i zJ2+`6Y{J*FO3G$_lb1^+-9qc8l6TSqLTP_&BIsfDLXy5|+zI)_t3fvQz`Mg;dD0(Z zKA_>7^D4Zrl|;F`cmWOZo3^x%t4)@r(HreJ*J>Zi@4mM>lilOx)Ys;Yi%#*04VyN{ zXCQcwFNnE;yr*qXD(GqTfrhvP&BMdPzs!Ml;q%{u*pDyfvi}~l|JqyIf3GGyEB<5c z!W#pp`%?~#Ys|;*8}o7E)LW6_;wf<+AGwUaYJA6MAEzhAbe!CtrSTl=#g{pi?+fq$ z3u6Dhn8*J6uX);&r;*f1pR~J79qFFbkDT&pgV;~0Ncs1hCUNHL4nO`Z*1nc16s4u?>o%N z|Ch#od@&FH$L^T_fAgTZ=Kre*Yy8KV_>VqKN&l}313BmVH`tF#3KR`pKmj~!{kO*O xf7*g2tk?ev!c*Z+a2(-&iiUD(dDc zVQyr3R8em|NM&qo0PMU8Je6J7KQ5Ha8YG0PLm7{G9x@B1%$2d@oQrcf&N-cd%ra*R zQIRoHW*JJxG>9^#BofLHO6CgT|2YSx`|ci|_xHTd^S=N0*`Gegb?s~Kwbx#2ueH8w zU;7FV!@!Xs9_ot4pomyJ3gZNY;}H_tNEn_d?g4W}Q~kdeNl8gbd0AP?Ur9;H-~N@9 zlUAUTl9iWMke8H^k(Q#8l#-K^lcfSA{|RINWg!uWFg%sy|MOVCo!tMD0>faiL>Li; z#SoMk000gn!q8YJaX5@fM1$fetOOjy6H$&R5TOjLua#dmL!uA}5Tgu0h|MF=Sc@Wxx@JCV-U6HHWdzyea5}A`tN&$^Znyz<_dvIZv z5A6TDb|mbPAQ}ha3F1WDf7KQIh5bv*Nz4Ci|I(6Da+3eC|9?f{2Ta$aI)j%N6hMHE zC=3Wd)~w4Ov1VTO_;ve&08k$vN<)+*Aa1!9B-X5VBVhm1F28rf;ZYc(BLF!-kbwM` z2!I&G@2(m6ffFbU0_#oy%t$yK7EdHkMnZ&PqZW=P5r`lja7Urh01SZu*5?HfVJHld z;_}ca4CyRDb^`=4267Vz5RG+TuioSf$nMM?i+4d|VF&^M$CA+GrT_ts1Q8@Oh#)^F zBC!N;Gergw2{N0T1x$GgB|n93qP`|yDBuE;UD*!^ge8z66~#j!5{NJdG>U)(5kJifgCzn&c+d$vYXl?0kpRR_$QlOq z)P!2uidu`?Zd{89iwZyhapCo@03JL;Lg7IKAoOD)lfxmI9E_|V6oSkYr3(ZCY;e3W zMG%g}0uTq#35CI+FiwCw47IlUyu1K$z{h7jR&HWiTwENY!hj*U+7t08kf0`|!a%|h zQD}fHEjSSkIDtd}jz@f_0L7{~6R;Q@jEGbd^uoh1P9PvoAi{_wf;b$DL7*t>%v!|O z$4i_HXnoYY)*t%-ULXeHBM5i{PIwRpK+os`g1SFgQ8UMrKtUA%fn@;5A`}t=zVijB zsR1&QfUvL%8DnE80eWq+M3gIt0I(z?WwPW+Z<;SeWv%5|6cLajUpu16zdxoc@Y5io zfE0NMh7H;qb6!`#X0ZWgXb{A03?~f`K=R}X01SfwhL)OuO-LiwZFDIbPVPS0zN)a zI39!%!Sx2~2|l@%=~~`SX-BCh`$GUg!yG^~VGY^CaJXMkWB`EQY)b$D1PP*D#R*6W zBo^xeQ3h~W0uhQ@^ZjCLcO;w>9)29ur|s8}i#UqA>}LHm2xb z88C1>h9w%4`Na@7wS!|_U11o6@}?>%AmM<*NDz=e)j{Du-5%O>2LR-tuoi^{$T3+A z5U>Z7)c`<9Tn~i?3F3#*SOxDw^UJ%fVDLM#Q+cipn+97JN#2oO&uPlj~lYOI4Z2q!{( ze0*$wDM3fn@2b`?k0uU>robc0w8?<LfCr!d3~Z*Ux=2ScGr_zfDr+1mE{G3{S&{)lgy_a>kEn?1kHM1O8X z#FAC;>xO?w`6p5SwDbN4{Qk6O_V-xcyovW`9B*#-_ZVK&`*)T9sj}BJX9UA-ULb#z zD17e?g+dwr37Ohj;`6W0)bQYsf&ZtaVw)KI|79wM2VJplAk-0sfuT{JApZXna`|5n zh5iO=7WDc-hK&DzsFIvsP|*Fa;PXEg-(-Kw*k1x=;NKVT{tTn9`;-5Z7+r@v_y1`a zeN7<%6pC`fkUcvFON2UNNf?A0S?#~U=KuDk^&cqp|9dF)@7{3#A6EYI{m*yn|F6FT z`m6I_GSX7NJ^vvisqmloKmUsIn{$(E`God{W&N$JVv}M1oxEa$+^<3lxt{D3Ho+>0 zBl+@ofd~S40}k@CAO;S21M7~3f@1#N0{vfB{?h*O4lwwC>-%4_`TzF(w}PzHf9(HX zQa0N^Mb$785sAg4JlD21T@)#&aejD)wUbn)STy)|eZzXz``4QZ-y7peXfgzV0vj8T zlx~z;@GKF;P+qRAvqm}PNx+iuaB#g9oJ1gET{kKzhk@1(y)XcP8;ExxC`>`!=P0D#{w#lPl=K%%@6qb#en{$@XI+x)Fb{hjdkTVa0}>2JpS$He*1 z+OYdhlFa5mvYt0jX>Bn}xJfyHMA*jkjIju_^_9GqKK;9AxUoft!=v0#XwYf%@o^X$ zjdeHPc%Vyw!`5Eq{Ahv15{Sn@qB|Dv@^htuvG&gu#s)e+Dt>IyQCK8Zr zv&w+HoScjt0{}P@)@pHev}Aox996WpjFfN(rW6x{5p z{@vm5|6KV?{6{&|3L}D0909j+;OZa5gMXF($xBN9CjXO`lljm2zkf;LUk?IKL_i1) zVgMmD7VAR70gyif2@n9XDf&VH2>G9|Fd#`e2h0DD#1;O{2eN-YRjz4lupV*<;`%TG z0QouYNc=A1tnJ!UC_2JWXaHi41OXxpPdOU2{s{mA3$C3OM#9`c$_IhJBLip}8+^xF z;(#$F5CI4jfy^{uY-UUVP#A!SL=gZuIr40r4#45Da1bP8C2`0PL99ErKf3;R>;Dhz z-&9vq$4FP)74gp)<1g%AR{Hnf|4RL5|L0#*`2Xl@SIvzNgVq;05eXx19DCXHB`t{n zA^-=EUw%!?z|RlpW8K$mZj-%`L6tfKgZYmdR~Vjvq!5Oo(E!$QgFfZ6LIOZJ=7U(X zIuJO$c`xWRfI#6XTjCyT9~Z)Y>|U+&OxR!>L;%>eCL7)WhQ?x?2q**y;6XI#2Ez~m zQ!PzxK`aKK(Z3T4Gj)4R}<XxCN=h>F z!eRh{1V^qbfO60RZ~%WS4Gb3V3PYnkD6gD>)4$pt*$9@W$=dj%7N^A-pnxd}LtaUz zH(uPqa5%Kbcb%>sizAy8d7=JtF>GE7n{5!g!SEUp)K80qY##s&1CZsrewP2&E1Ur0 zV0ai2TwBcRZx#PZOZo3DT^MvCOe>p`mG+v`iS^flx&4Cn)L3>a(BCvSk zTCCrE-jz&`Je(NdKqCGC)a&gj%{MHbEFg+PVdx*dNEkHA1q9rYl-2irp*h0f6y#6& zKLP&Tu;~9x`2+m_>z8i-#WDUA|I5kCOUwQ4`+rF}ng8Jbe?{So*uHkL`mx4~uumZl zIOMlvHXwMMwu#&AVw}B8-$!M|IP`2jop#p5>nTrYF~gfE7`rh|deXSkn+Bh#CC@Jg zzUL0tROT^glnP(89MOpsE)fx(exe|KaqAvK-UE;Xze-WXLx&Enj7Lb!`SJP9KRUBi z$Np_>`n~<3h4H6@ZHK(@-);;(PV-yo9#qee_!N0^}0$ti%1 z@yxLu*{cU`bge>e#PdP<{P0GnJOffMnNELWj;-EJ@1f@yZ=$QTBp}Fej zo(Y~Wy1pW)Jk2}67!u~O95Aby%y8^A=-K7Z1-1bjR68lA$JGBGXaZyB}h@LTM&Xca(s zGEO__R5-p`D`2!JJ+6Ez@$k^@nxO#TiQ~7d5hFrwZeZ$^ssyJ+9zVfx%p#{E=CJUo zxR3fB3$zgkO?Crw;l>@}qYqj;K+IF`q!=hjg6K_~#^?`L{dsM#vA%@xixpS4PnF6L`aIvX4yop+;+S^ZG zxmC^P?j)#P)nnknsMdzuot1tkX1(8LH|K$oWKKPiXOWD=()6bLr@gF8Q||~pI3ZxI z9=J2sFJ3W^#d-NG-+c76g$k*szOya2S>aWHoymN5v;6Aq+q71FqLcjnq*zv`7Kcc3-&mz}CA?xWUyF3tm zS!iEG_a_sl{(U~<`rnM6cWFPL(z23Vy1sh+Ltc(Ta+z7oWz&qUc01J!+@7Ao=qH}q zS0zhxGB%c(AwtQ-&8}I#YqEUA8VmC_VoPi(i|4v0S5b*kTU=-!;Xk>(dfewe%kcF+ zPtE6u)ECv&N|jbJ1JuJdFH{~hr@RZR&g70O+p=1A<)S$%Agq|PYo=AKV=7B*ALnNc zB_qh3>Zz$_ul~pv@15(h&6>~Z^pa0mea;$2;zuJ^3effGH7_ERXN>uRF`gX-{lc&$R*CZ(*v8~)?WG(`sv z+&-H@vjnrx0mXN%lG7ho0T!bcQR(yVw@Srj`0Qp9z3v?R*-7!@kVYx9W~8g!{>idh zA%*e<(yJ5K1nA7>^84(!AAyFxUaF`u!|u^xl5t_&n-CZFL{I3ztx!EaS)D|4+8cw< zU7e=GXCnu0?bxfdvu4P0F$n72?&+*%l^6IR{$yS!zo&fJ%a+3-x)<1Et;2ZvGevh0 zbldZr9c7>DTQxa}28W;&uqdO0JQpRe#v#l{{DZd8o2gUpJeX=8BMHFdYw?WqGU2Dc zT0l9(QO8=ZKpf7O8Y|p;TK*-O+wnTo6E4iXzy64>h&_U}rQD{cqn!^3P!_!j1k9Lo zDH#wPM#odE8!RR?@ph5nWzhU7se>=}ud=tgI7RP|sXP;j@sCZp*}zc}8r0fh?PoP& zS+RVBCv3h|xx3D0$0^os+&-gK=&(oliJAG22b`RZTgLZ)IyGzQzBBc0!qPH#^T?o0 zn+*4Ink!r(%`vP*K=gp^PJw3ZLUZ8r=*k|0yPS{M6U`dtP!FH*7i-;kNqx#apDuD% z>4@m~%WWL~!jC7Bb|ZqLGX0VXunk z@u6qWgYyz%<&bGd-y_>#uT>=LBnS{>fXri-kNg+DcxTgehzNY*!Mzt1l(?n8$14;$ zIo;&s6>697b>R*7A!>&F-8(+gF27${QJ5Q;{7jg18Zx}XhmRUo6Q^zIdw+I*W%2<3 zN)?A1!dWgig}pMT`t%OjdDeQ``@yLw=A&gis@Zfe_#8)@-ZP$av>D=O z!86A0Ja(}nD!xkADmnjtb%JHrL1OPK+>6^R^(zvOFZeAFpPiX6qMQ7Z_6@JP@Ubf} z@S=URpnXOE%EI!B-=T$XLtkH2Ju+^V*G{N@?JO>ytB}`S%-EZHfhvBqzNI@F9Ae(z zWpyWFdRN7XN~iAS&*ISp?)l>U7XrbChHB%NvWA#!7|ozfQeuXB_89m`C2 zm8z`Ts`tN1T0G%?OTwl4!PO39zH0pO75-)WwB^ZV?6G4XkMo@^@snVU1?V%5blhpK zNzGVXaII0s?hCL|Lx%RgF645#b5hV73*5UZ5erZ1evv5YZXnn2e(@2eqwTZ+q`_dO zdjHLaV?#-P&IY%9rf%P2waI;S%J9lHXZB1HXPK8ZA-fg&-XxA>U$+|Vw^DW=FfZSo z3Kzy!Or7PC7m63yp)+AXG_sVxmw?Nb1X~qGix-bg4NO__X-3AFS88z2FT9SD&l)zl z*d9D=taZZwAWQ~_i(?ZWv(PK6e*UWU!$>_TYY(O_2P|6ZNCaJ+!u0Jv z9ET7vIjDpWZ-0sxIG24+G(ghoU1T>DKUqFf)EC=wOGt8}l(uTh(;z54$Kbhr z=Uzt5OYxVbt@j(4Uw?Vqbw8_B5(j5fo{QO_&ch`RUm!27nT=cG^NQRlo}QsNnye=f z?xSIOK!XI2fW7?rd>JOKQBA4DV|%WtKG`y+td?0kyH6#0`~CYBrrYg1+OBB&o;*Aq zfZ)7<;Aa%Kcr=i&rfzL`tQ4KD0<*p@T{nLf}Z^os(p58G(xwktndvt+#NcU6iH`%ufO&8BB3E@OV zX2jhPiwUe}49yP~OtWkEM?@4}ARZ2}41jk!9C(s-+a>Qv=<(wwT0!_lTYrrl46fVU zx5#pr;F~;^EQvEaW~O?!r&&(krb~1PesK6s=XR4hA(M)O@Qf~pLf0Y5ohJFI4*Q$? zCJwxC`LZojm#0H)^J9TYed`)fW8#(4Jf?lV$c;T=1>;Z&Esl*f(>JHf-Sez zYP*T^>v-Bn0%FxyWxd?u1|8m&oq9w_qCFk6g{6g|@~Kk_?{c$o>D_zXer(OSdp>(_ z9kGsXaD5ZTPIYywyb8lxmZxFQs4u*Ud#vYuO!TRWNwsSWcb<0}5_>&@MAccvp)vny z|8B&6HdUKMs$7MeeTS$zxnGGDcS})qs`$q(-Sjj&Xkf~KKVuY;7M5Q6>S(5uc@p)3 zLgTBJHLl+NCw+IZ32DT-kc=$%FCe+IFi$RAYDrhdS^_O)*7p}(O+hF zhY0#|aGs&N>ma0P&-AFsB>8CAQy#_>xwqrCp1}h)+k&Q7h5G!sYq1wa>>fFpt_T_A zDn06$@+(y(Z96Pm6j7Zxl)}M&)!X603-RCZ|~k2lRunt(5_Jl7LO+E8Yk_qdp_A$58h-THTv8f z@4m%MsOqTh6j;6_TmE$km>N%L`FLJY_P9qXT)^O5aLzLe+q8!k(e2ctW|fVE&*ino zU3qz5LpYZ(7&QAFU7F)=udNwYHf@c^+S(#N(YUjzQn|AYuu%7Ee#N=7F)R*V<%kuV zeykwjTyXQ!t)vI#URoamV0Y^F9eY5=Dz3UBT_ zng0^I|74Tt_S&qR3%YH+6Wfl29zikl9q2uHa|>n{)BTznBR1Qe1LeEeE&Y|)TVJAt zQ@Y2;13kAnl-@E+g^!D<@jqV`l2M+FKAC#y^RuSretWyru0+NZODPr8H@KWVHtl{6 z>Y{zXr?O=Xi9YZ3#e-r~FDtCdxr~f*E>)W!Xf0ufOy)9(WVg=u7iXPc!Bg3xUqie5 zB+D0I;aVp0cd|d}d!-zb91dG8nLWy;*=%tomO9rqOxBS)1qqerdepgn1)ybP2sRc9 z(t1p#m9Zt>`yO%Euxq_>y;X6$T;Zrqv;3>up?`1OKR&>ZN5;@16!}B7b^79ww^I!en76@Y5U@8-9?3MpiR zJ$K7x!H(AAV(w{;impPPmXKYhB)=#&+1sC<<4)y{I@8R5)tCt!qH9v81{P?97X0Jt zm_*u4>*?M%k~sGcG~SCn$G@~iapu8Dqtv$ZQkTUS z&Epuhx&8XgF7WN;?rupLUn@u?zr!**u!}oa`@Y#d7b%V2J6^lkv+=% zBJ|-&N;z&_chZ=zJKdQ(KP~d^ihH3}sIol=!u1{Ul~lMw5k$f(^9p1C4hClnGug#d z{vPKSLNAMTRH~_>Z_6AoE)%ZMoC}kU)(ZD6I$AH^d;V0|ZCU1LpIx}HkZV+ zlYOqB)HP(@4&77HNbV0iChpw`g-FC?-aIlSg~<X%%W#!6cbnDyxhW=3h zZkkf;K=hU6NAgR16_=HJQ=cF9ed+uqkI_gTF!8Ug=aIAG0}5aDMk>$9wzjE12P$@~~}_TN1y_k59~f!k*q0 zz+58FwIg5f{->S{-uCp9jM`a+fej^mA|OHzqU;Wsj`%WUsvfUUx;lK`~m!;_sY0QmalF>s=C%Nysp|OhasaGw*43s_XH| zTfpeI>HW%n_^?)W6`l8Kd`T{g@9jS#P6spwl*Wf$vFfszjT!;JWb#}-B$Tp;vr63Z z%|!Eb_O)vkRh8Mzeo2b=^M&)G>}2@;`a9+(_Iu{Bu2{R$Ru_y5L^551nwUUpPQ0#8-$XfBY#!>hFi2J+5ar#Gby9b)aF z(>5*Ve{N6L)(& z^OGE_dR(Q-c6sDX;R17c$3rR)qGY#=2G5VpsGz$-OSmU;#Z2?VOhz0U%v~8D@q`!# z%6b}nFlkHg-Fa7JDIX=HWLM-L_ow(QWRx=H1%`3-1vi9i|1hQCV(<_P&<6ojgR- zyfbuVrre27xLW2e)w~!#-(w_GF7QBlgvyqSgZI*mzLK0aR=1?rVwURzZ{%B3i$}Ze zyEThLh0jcchA1tz-u1XY8&F!&R4EI_1&b;U_7x>|qxE*SpoR=Z-aVf_dRIy0S;9aU zLV6wWP0>9~a|!90LUyi0-V@`sGLCp35M zS;-Y5IYg=}J~a0}X2d5 zc}n{7;<5scBR}=s6sf<7%w57cT3I8EKMdtbGYdd#{~Jn+DN# zuvem+CZqwdIE_>VwlRk2DN>cL>L;r0%UNB>D?#uHCOmt z8BN^&Z59zaZKYB>*_|{EA6UUf%#K@Q--IWkRCOftN8_$sve3L*(!tRz@Ui+pi&o63 zxmV-gQu+F;zfKA%l=nFA-0?Ep1Ya;;oxa5}-geU4##G?hgGx37TYPWiP&m>;qFNy& zv(%!~>&}CoGZrIBgVpID7v-5A$2a<2`IgP;`2b_1pyW2ub-NN;n=ay?&Z36L~5Y@tn)pf}r@lA4w=d@)ei^H_?5 zkx%HuSKQgyl)NXOX-?Hlym*%5`MkX0T|K91uv2#Qp*W*4}47f0_swztyCKA^`X)h(f?s?Q&Bck5R!2*cfi^E+4_*rc~< zH2TM!3RRDZIThFF*2nQFL1pZM<)Zi)vu*41j*`l>n~CErH@kKDpO}izb@%47UibPU z-ec_6$TgvLN?k=h;|^MDthV?nZ~Tz0rtZ=H-pK{#CyG}y*(ItbdTb||TX%yisi0mo z<2BmyJ1(pyP12C7)P36pyCVA$0x1{S>UsNyZrwB7QGcx6@6+TeUUgxyt8QODOKpx{ zEowRWwy#~8Z>zQX)6bK`^$LD{kDBams*WH}IHR4^az9s^Uz?P$uWeVYYLuLRbo*Q3 zHRBZ~qu1NEB;D40F0ZbUOPA#tv?a?^BbQFrbyl6p=yf`ZTh^7Pi8YQ-VD<39g$&ed zWvav5qM7z_D&j+TB6~*mbCdhtKI5P5YUiKr+Fs_$la8|P4a*>RWTX<>!cJ?>Aci8LbF|IsMhS2?!F?AVLHm1roD_g={AYN*X>(} zGf-DK(owGir)T~7dAvngK3)FP$QzpN%1;%tZ7&z#CI8m9iC0o~TMo7M;1;QqlCt-= zTvm`z|W0lEu=9Lgn{*veIG3}d7-(t9Ir{W&9-<*ex%u%;G&)-Xr)xV@$ zA1uu-sQMvgr&)4i?uCPGCRiO!Gtl03+G?BbbAdn#Ig8DWwe#}j7_#LsY&&$yH8^C!;jg*yN)q+nwTnm zI9mGPv@YkKqy%Y@p~>sEczqM(T7s#!3|6~8XMa(R&0d2iBb;OHlj@G1XRW+$>UC-R zIh{JAdBZUyo4#$LMA^sLq3W0msORgfzIt`o-%h0L74z477CG^D7h|?*dK_TU;oV&v zao#TR6o*IH^Rol#O4n-TY!5NkCtPZ=8m4DVy>d`TIQF1soVY=a^<9mvd%_QbA-cD< zgcBC;C76ehXu4%X+}( zU4HL;crfNudX$=!{b$|ucaRIUDvK5AeWHE0#Oe8?)E-pqKf%~q)G|LM^uD6akMDgL zu5cTxDNZM!MiFvv$1N5=w!^eq^sObL$u>*5`{~4uO72C*zQVlv1g)WMfLoIqJ}I(# zmpE{=l$KjuI&)xvTC=CiPz!km{@sws%d@(T6AUe3H^ro@ zA;JzrDLY~>xmS^npU)zMup5-WJUz9vwSN!qRgGPq+_OpzM0pyf6Mot!#9tAsj}Z2s zEU#06K*YAF!TWAB#~jLsN)Fn@zLE4q79HB&ZpYJ;x3onZymviP(*Fo2PMD@AZ;3DD z^El-RIpbrETjjs_t=-Ot$Df~u;@R$M)0dt262m;vnNhQ-sNqGD(C{K5i)vZg?bI+2 zDIMvkcDt}p-|X+EdNCQ~(88J(BuK6hdfB_>^k-(O(olyM)^pFbz2>+0OR@x=A;Xbm zmILi}HCvEHEEAnoR215;GTZImFt*z**MIf67t2!1PWR>M9qQ;g&D?vjEB<@M6gwWQ zJFD2fT1tM(Sr~ zri`neY)bN^bq(j~Bo=sj&`IJ?W(=cMxyZuqZpi(z`No0vjji_pX*xb<2 zJLMXfQ>tN(W{z=v9^a)!^X=~@<9B{-34KxKT_3zCpZ?K9=zfaPHLLiA&cfSgGz`Y} z9+eP!ZB)NdEN!US==N+Prm4V_r`GDo3ocWyYg|0FmD^E-vx~FAzH=xERxiqU^WUIv^XZsHJAOSvx}mOC z#&F`pk?az}{L=_0^()tsc=HPbo*wJbK7F0GMkL|*NA$(7`@|Gp-hp>URO;F`o~}uv z-7U{YD}-hVZds=K#D)tE+ESr?kB2(AWtmkQMxZ>kM2v`j0`)M%k**CVX)B*j;8{>T{9T~+7+x;`Pglua}HL9>Lp<{~D z%)QrsR)c8|Q`9 zvod$VPFTcNry+|(`yD$phwu6_O3iXzD@}3`F)@^`6bQ&d8r*WaFJWf~JZYaD89x#?tlamzc`ZcQeC3un6l5T zvmjw1#xUXe!p!V;t~7a4=5fE;R~JVES~)GR1C66m7NpuXwLJc@-kVmz1N5;6`HWX{ zpPZYG6u0HB|Wib3yo>acD9Z%ZGN(JskI~J*ozdYwUj(wk+7FTC2arJ!@+1cv2P`yW?hm z&zGd^Ejdu)>m0dQF-h>;JHT;BmH!6BbM$k8S53BM)j!v%+J^s2CM}i05Ej(9f zld$%zf6ewKh4y+npN8-hwb0h|-RS$D4K)Xx5UA9!Vl|16+(Ma=qX){^7Z+_qnz3?b8^gvzKl2MFh37uF?&EGh}RpJaBH%wYwO6tiRgX!$LLF>dReoq{?ziC5An% zqL)~C_Kiqijs$Xsd(Tn6irI$uWmmW2cvG&Z&5z%&&3?bw?iP4IAZ38t1CpqkQ>B^W zy)a+rJ^l5ApuOMp{9w^ByTXGFK0}-ZZNZ%~S9v3CIV@eM-6tOPXug-xm6qmE?9X2A zzV78-K-bfL&mk$@)~%6iGRRWR%e|mI*8soVzO9ZBI?+CU_tV7r5Ju&Kuw=Z{1#FOI zqBy(fa{~NwN=spd^j2qhgKgBple1Pu^xQ(UAK;ZVpu^*cdyC1JJ{)kkBTB_x_>oR{ z(f#HSHfP|lP&6#uC)srMZRfm-ap!{c{q+ zOi_D%DdEGNXWgaKG6;TG!7%Ch;j{M|I*ukn2BVftuTszDLiPDWjGfpl9*pV_rO;NV zOAP@UaD61x7@fXgAhAT!pD3*_7}yZA5}F|;AR7|Y5c4Bpl?}1koUlgiS3_u%`zo=- zY#-gKPaC9jkL2GFbA~^pA*Ow=G*x=D4V%Vr#6flu<)w?ueV@#4S;*RS8M3;R)^c@OZguL?dsEJNrNuKfHO3d6AVg)Ag3sOpuS`#Z`yuGXg4NEor%275N6j>}_>& zRbm0@&y^?pU^wp1QH!@xwSm%gDts&F@_nV3gFYUN*`B%VQleaD8^RJJ647iOB-dcD zV>I=oV2Ojc=3M&M=1C#Od(B33LLXHhNAu^}qOQ<$I;bA|2>5VmCkqc%$!c7=<~$$y z1oA+%ZtGp%!8?o5Vzc`qJRj^Gj>_`9U|G`Q)J{I^)z z^&pssl#aAiHioR!6*pc{^_m488DkyFg-|Nu`_{q8kufqD;XeN}7}2}71d4*i6WI?N ziZWtK*yg2l_ygRe*y@kycB>Yat>(Pt?euBe$5CKmvJaNEy^q=B4jKi^GxrN?-P2e& z@N_g=m~?I+vQ!B&8O|hg_0jW!-5md-L4e(+1zmn`FLG zYCq{0tpw9)r=^c$s}!JaDiNxH&gEUsmrV%6B&TNdX=Ezcl@n3=C< zxcGOppEeHARH7yS(qGS5X4OWmJ=%mlm}@p;4?}IQ#*mJ7V=m8q>9S=U>9Vr%A$D{^ z%sdJXl#Vo1;z?V#^r|7LKJnhuEv3URQz~p^KdfU9-BL34ke&M(doaJDiVq3tXt(vJ z(Eey5#5}Q8h`9ZH)KZr}y7KWx4-h$n1eriE%`4X(yYZlPSQH7fyp7 z6-r>z;x6JoNxyhuX}cYBTPzNX`y?7`a~gX_ey;qm@FC)<2+z@?FS~1?x{isx)@@ov zCr)Tvtz7xif9#oyqkugl_xyL#4CaB) z#~0i#a@iI@{1f%eb5G`Jzt!Alc^zW2z-;O{SuFkNk|8Fjm)^{IFut3$=vtqq&w0rN zS%>Tg7VQ;xXnI2KlCvhIF!08yEQdxGUDE@AVyCPt=GqH9JfhHm=_b~elP|X@O5+gZAg2%a`W5(1iC%OC`#o8mUGp^BAV`w>9P1SOeH!_WG z-`Ifl8C;#W=zu<77Q?;hwyygfU_ywBMizeFyS-fN{7qd?PhZOew5LQF zpE`$HcJ*mToS4i?&*iSaeka5lsHJ;ht2<2)i$DBs8;kYyk&gzh^bK>F0uFRHF}9|D zkp)JPS)UdjjGhRs6Fi&aDL)Vr{k*X5w0+5q(3d%yg*!~5ElNhyc=P!5?ZSh;)veC@ zfo62~ch`&3#s=kR>m)$=Z7ArlDoPeQPOD zEtGdAT`Iv36uY_5_3ZK&I%4#ONxldab#zP3Rq1AbdCtew=wRG%!i(kzVSzD=Y`;fm z4TpB8+-7tzPS!O{Bk>_ zaf-d^s8yZ^hiyUjXz+5%GYQ$!%8u7L0F8>)1K*}mb;AH@kP1N z1{Z8%lx}A>oC;yU968T4P!`X#50ZOF%)TQ&E3~{L^G1s~`XFahO9I;Zi;h{lx_f5g z4Z9GTR)gl=6e-<@czv~xxikH<-0^2zOU@7jb9g(BewenZ95o8%6p1izJA31DLyV1N zLuB{EPg8AAnHY8Y#!3sz$L_4&ty_IMJ@mnT6<@6UHG5?td!Ep<=<#e(3A-%zy6!Sv z|2L1D4_?l?D@sV?19*L6)yGdfQ8UA2&p92X6C11;dvVd>tWYcOqPMS`*qy7!GW#Ig zqBYzPiGDJ!fk*HqI^0gm(+}!qh;=v5 zg-st{F?ko6n;?8dpfR~xRK$75o%{h;=;uD=V+g~&*T-UN!mW#v)9WfL56>5^Ot{Vm zqgJUS@mI=ygD;up?LAU9py2tThv#w1$DX{Av&hRKU+tL;(`Kp+u3GQnfB7Nq>+PF` z2PZ&{1K<3XPFQdA^F)u-)OGc7*Wi)r2NqeA-ta`e%aVDhcr~WlI1|Hl|A{yH-mbDQ ztIHp*6rB!OA$==a9j;%ohrbGZVV!p4rf6%+O5yGHS1)M$l($2kC0Cv165VyP=Gg~$ zwfciZ0lG%qmItna%m6Du)W3Il-t1bkdaQiqn$tXQN0Lw)$<5hIp-(JKFd)zfY%z2f zJ8&yhi;rI?(VX^zJRehO+a*Qr4!qaFHt}K)L1ZG$%eMEOoU-SP!=~j5C5Nv=F3Sog zwC)y2usfrBEXKmwW+|2iZfD!#Q=U50^)d55|ZU8_x*s$XTeP$9S|45rb!&0|Bb&3l}cbEsirT_dmAxIn%o`IX&1` zr|diPbvljLZ)NymQLXCA$5O$^e&o8ZUz^g}rzg}WLQO@)R*2r-{6bZ&jRS<*IzH>0 zt4kUK&#jP*-ZJF@)_Irdox&wS#M$SQL{rWOHFeg!$Lqd&E`AMr1@HgTEU;rNR#fVE zw|5dwv*LyQf-B*@k96YELnwzr!R;LJ)n-8k&-Z^`v_E?C?84WJE>Ajk&6UN6zsDxK zES$=89M*wB>*f>hN1hS&=aFcqVOM<`cBPsAssm4-7nor0HU%)w=Vl-Rifc5)g=J>0FLelhx?jiM+aQy{eo*1Uw#Qj! zqUW4F7UG@2)^Q2+L!RCbZ!p(X8G66 z9?UnoiXbCf<4t(VM&tvuiX74|k_X&Fy|K;1z!0?(9~{Ju^DT(XV!v9q3qmz@6l902=c0spBVe=HBES z^D{-FCHIaz(Vbt4wan<0;=Pj38&R8hCoj^TXZU1Y-Ne3e*fGf?f~GKw!j{2enY#y$ z=wrX7KYdntoIe@-rU!X4mwdH-Pu#=3JRf4-af%r)kG*3BAqQ8jATW5o1P z>~f#~S3G$s9)P_^$9=kpRgZjpBSPN0;yr{?FMivfImrL~I7wUO)Vb%(%eLmDOx2fn z3uds%5U@^!=Ru|s&P0VUMIo}n82Y-G>`aD$W@XM#$h_P$c$5Luk|>+%Qklg= zEFx1HQ{Ih}aO-Lt7bzWPo_RjKiOOq4?G<^FDnE(!%dthk~OwqY7215Z5MZ} z>sk0IyA)U8rh3}Yw!i$WdL0~Rk%A7CM?qHkb6=$Ht?>|9ubE{PIVMH!$#=1zhwj0pb3uYhu;pCVqDZI6(Wmaz`CzFzka0$=s<)$lyV-M(IL@OO}pF0(IaCrZTTZN z@&na`#!FOTg5J1X`@}(r@32>XDI*)=9G{n5%>z&Yr0e55;H-x0k77re!Rh(^!YRJc zj&G2U)o6(iZ6aR#caiV#U-HGc|Ohsch#v4j!<1Ao#}bgpZ8# zw)kq>;yQkX2VOE-*JPLvM4+nhn6~eymL23?_*`PZ=ogK}enix~7nfzIvm%tqttU~m z&_oKckIN@s-!b>?^%XB$E6u+z-n%#U;S$}eW@#i@;ESfhY8*4vu)pO@qnOEo{{Ynn z@a2)PGV>n%v3dDGP1`y-DZ!C6Y-iRmY;Pf^36xwlpSRX4{cMFzRqx_hY-!bMRWNnZ zn1ZK8e;=JE{$#w{n}{I_zOa(VAi$3}Hx{%d=C-&&-K4i}s%Wl^gRDmVdsrmCaV8F| zpL-N)XV<0D5C555v{2DK00Exq1niHVunBf1J%YmQJxJn+HkEBF0@ObZ106u!d@?B= zKT*BHXe?7D(cYGRNL~jaAJm-$Aw%(^o(LB}K0^ElgOEdoVeW*8SjS-hu5Tl@6V*^# ztxYdYcU%cnJEA#w+JKsiF!K?pQkdfGrYIEQCyt;5^7sV>pDvehv3WEp*LT^B>I)}( zBK=`kO;Xa7oF7>D=FPS)6Ca-(^9x7%Ym4OLpO*&06nBaUke}Y^0{{=M;}n?4u`!abaFCj?z8YWk;?VSux@P(5UZgd9 zbX<&D5zF4Hf~YuEclW|LxV4q}(`qYkOGd~)sOddBxcT*K=f?H)Gd&fit}BKonpb3o zCG|^KSbKPwfeJ#%#`6haD-Xu=^|MHv0{&=B2R8YOj5?CZ)o-~dUi3P{BR!FMOOGG7$n5&5c}yKVHxy~ zow7NkR=D25)!wku<~PRV%~v;-YlHvt5#(0I?D`^r)}^>Yh zRIIBOK_8W+Q7@?&*TQBxvPuUh`+WwV(Ll90So88kH)5cE+v9fZ-kan4tv9C8ln%T* zPVBGtz&CZ2_G(sitDBaQF3NeSu{jQqdSI^U4V8m+>1dl0K70uV6knp(KQ1rb;IMWs zdIqO9obi`47Wc)o1b(ag%`YU9?anE~hh`XuBc8f1h6chSmJR+>5wQGFk;6Fis3cJ+ zPUyI#DM;&@O{v$|OWPudRGZa%9$!5YW!>Ewo~xB4%~w~a0Hb-(yD)R>=@%&x{b*A7 zNa0+EZaX$vb)fmwWe!WyD?M75Lew))Eh~tjJp_nh7 zs@EAg!|6cU^4S~jfnA6)gUmGN7TR`RfX(Mz47=)%R0!#)^&=)dUa#h~?C9uC4yTsv zU)T}vw;Ubz5!mOo(`kHVqV8;a-6m)J81ej@KC`X@8c_PWoMK9TQ7n#xSa08L{3MFS;5poZo8UIVA6}F4@FHOC!oZmp){(rz>5rHZI_)%k=#NNJ0d^fJAm#OOhHkzIPQzz4c@JZP zWy+@?Prti)z`R0VV$2g+5RE$53xz^gsqADh87r%8EY**LO0D~3X>hL9wS&%|H%0m} zau>9-#6yBn+{-RR%cq2?{0gSTs@1BqHqnMX67ft+~`cZ!fxKxoilD#XQg*>h|F+Cd&M>8%~{&+w!Ipj zLm8042xlk$(#wyd#hW8bP}XZ-MOobOzK}L3>}G=TyH*ye0xkeB+7p03gjYb4AycIR z#gP}=Q-~AGkAEhcF!s?CQ!{-MJB^}yZLXD69a*8#G1~iSu(L~zq5orxwS@~u3ZC|D zzQ5K~n)plcMtb9f$OxH}&!!u8CZhMWySQTdicAoEmgu>afiyFZuWuHTIR0+ocQ$Me z85&Us1peK@X9om6VZy2)*3V8u5A=}EkeBh?A;aK(M$SC+3n%2*PN?qdcC9T^_eYq{ zD#F|tPcE-W6`sf`$wyM8Ng;bv3UJr(Ln8Nkc#t*7D1IU7XlfU52RDFT4Mxv` z5=e0b)9|++9a0XFzL5w_&K{{BT@SM1@0VwqwtlT|K$_k^3sA^>&g|S4+E%Yr~fINmYc60TUTzlS~`FQ3rTxj@c)!hOu}wZ74b8qsQ(pFJy93Y z3X)0G(L-f}MZo^EP7T2HaP2J`fW=_{L9YMefaVp%%2>A#0WI7U*nhoP15Yv z9?QDUy{)iJ$JWt34C2`L&k!l`|CW!G8r~0qd+xG-QRO$aidBR99ffK%rn!}J>zqnG zV$^gBjVpz_`wK3MpbwjT<+|}`5{K|#V)_0O7wq6QLYp9m;GrS$(9-iK1PC2m)~KKZ zae9vMZ6iZy;9TTJcIEe5kAD!3QlQ7MY3-+*oc~Vu{u5~0omhCyY8FQK91@2PUvNnf zWz_g7auc9Y!OiERwhO<|%*jz{7$*1IqvsKd@~E?E*};0BjaHySvp^R~5vD@(hj1$% zlyWUO%0=LGX9|D+Wm z;W)F(aiV~@`meD1DDf~7I2ky(NEw~;fvWT~z@GY$IO=2D*m;-z-H^Q%tcz4CD?6PA z&BGrHC^^z{pQNfy{(UWOgfo6qrl3qc-6INh;~j7tCnY_`EOjOs-c)~imGyhkhO{A31|2p1bjG{Cm z8!AW|y_Y!hVVQ#MV*6xX+`Kd~wK46G=<|Cy33XZK@4Kk}oQ~6<)nS~+nX{cJqB zvjyj7UbRrew4g_p#HHv4EEM+I27{D-bls#*B*bI$m|xeRKRh-|rjntnS4^FqRDAn| zt87^oBPqqjk|XjoLb6_@3FuSks?=U3t9QxC*~R#bbu`&Z52KmdUK9!P=6fe<_xb$^ zhQ0nzZ&0~qmFMy5vaED$%bw=ZP9g!MqH-T%t_nJMS>YOU57`Qm+b~IXe-WyK0F8>+ zlBRVtlffBGOD^NMTc;Mim|q3vicBl`5&19Gx$0PT@qC25bAW;D!&(@EyM;IuTreL7gdTh# z>~K?9!87Y8%JkHAATXlwHZl|0YWsH$-;Jg&{W`MMs>#LLbXCaeLmR#KaC7W-&o3HG z?-YrfO~-D>wj#?*i^h%le-xLSNOCbq%f3%ngy@)8xATNuwEE50RA^+repN;lWKs$r z{bMjgv#}NCyC7ai3b{iERqcr=M|m`spJJP<48FWQ(E^7Z6$fQ>w)jrrT`3qd9u!j- z4XL2NLs)oQ^Jkk+6{0Y5l z2eweEWI<#TCyUk`;d2To*}qD z26p@4+D-C9OUsByc(AxjGDg}Sfkm8(R%AI`mE+N5`WT^$ezP_8NB_8(+<6LFcO^y& z-g=at49ax!cdij?1pa?ZhHAH0QBkg3YcT zg7fa-%Xh0VYj+-p7I5E9-%jzy^Vnr#gbHYXyPb@|v8piZc`^X8{xa1029?RbIlppA zcl!RG&1Wzwy}KN1jW*jymrWfF=X`#^pb)QcDmQBxMANVO1Y^`_cqA^UMPbx%s(b^} zuj;o9nv4hUP=}=MOlQ;}H8rUGjP?L_6auet344~ee<*`|z}2sbL`5hO2L^#~1>PZN zmypV!ph<{qLP%v0Bz0#(6_-Uac_|6@a?48;EN=|x>^4?bjdhq+z1VvmSD}G4Z5zr^ zAy%i_u4#?-(^SpB#r}t$&$(a*Vq!Q8Vq*9%xb)ykXaYHs7i+ud*z|n zkxi>np@?xsQF;1l8b3#}^r)1JKQ%BZiz zD)k^UVXKRXm|e$v)zh-mb-^BxA&bwA(D$6L&tD~+KN4WpHYx!=CR!R1dko;MN>fWbCuzR+e$D*Q% z#wt;-%e9X%kk0Prwv@G<48_Gqkik3U6C zcENsZ=)=13?m~J3{d0!1`I#_18Our@t?^<$kk>^^AiOTJ>sM%Drx(7BLprt}X-huZ zR`BSnl&fMSPHKh$@{xFn;jd>_T39rYLbrcHQmV5`s!rSG8OGJ;dy^oLZ_Pv8uJ2K| zKJ4lG{l$A44e#h7qAny`e@9*kcg$H)G#We0_O;BvRV@T5Wy_9f{GdKJAoFnL+-&T7U?K~} zj;i)3PGNC3xzxPk$KCoFNxSAk1afPkl#@EcIras z4$wivi@(g9+}pt#(7~y7QVh#sL0OraI=P&+j)!ZVU1og5r?e@bvBP*wX)`OfRPyG-AIsvf zN7a2VQOZm#Bde-yb3BtedFH@tGkjREWL&TM$jfB#G{pYx!a*?H9N*coM9TMUf3wiI!K!+>Z!u-!sqGP>ZcpfkH~oF z^%7FUUq#OKJP0Of9x@(a{@s@*B&H1;x&_ z#e?>!6p(kYf^Q;QGoeA)Zf1Ly8^vP3C*=29!)ZN{)&_KG4bq~yReSh}wczA@9IFs1 z9gHDT&57Dx|H`C-2)N;ZKtU_rQu^=uTFH7T`vEarOeRr zkiMkmvG0yf>0-sUkf^Mh;`UZTF6x<69cwd_E+WpR>g$?tklMJkSB2)wzRD(rS^rK?ClP0bW@rqOuU z^(o9P{Xrsa4HuYoKC04b6C)oXh+&^!D&V<_t6%O`F1dOe?8quAf9zVUt7>S{g&()` zGCv$6AA(oXwR#-s;-iF{cA1!ib1yejb!;wHGT_>StIIF452kV!X|f-wzUs8c+(CIX zsBvxyC44)6#BIkZKf%>-n>l+c*;o5%>yb@{vi7YJ~BJ=&f&$_I5lIERQV5HX_t zK??VnBT2t;2@xp~(K-V1@QSieJmcXwp&eaUmOsX!8uDHS}=t>`h?Fr zJ&K~%cFO@Ox}3znhf8`#09Jinj5*a^Z|+R0M(k{=frxw_YYloVd&9k@%(oJ$0?}3p z(znKlnnHTPLm{lHaBh1;AuZ^F)4FvHQVHJn|2Y2=^f*FKlOb=1A)#r`zVvSP9P`e; zak|Br@I6k3cjta#&9t-a3A!CVdCdir`Bybt(p2Z4;X!!K@+{00wewK_xossr*^^Ux=WM4hs*bBNrs$a~Dk zdZJ7IRHe>Cr=MN)d|3_5H{X-=)E>X{X4Z)jh&giq#gdu>=}^s6A|#vL@3(tH=9oE( z)x%eaa|P2%WXvK*jkH=iuS7Kp=n4|CbgiR~rdnb)>d6QB*W&eUJ}NN&PR!F2(V*a$ zC_lEcwy}Cb4#yt^S`Jjd40x&me>`2*da36sA8FQeMvJ-!*+A);BvKdia4dP~V&*4X zZ+|&wVOWf?cqko{ADg;qf2Lp{B-^POlVW0fDk7hwgp$_fw^#6qJM2BL@>amZu2bu_ zRdMslcuZxVVZzBRNem9CSK}}FY~#SnIJ6OvF16x<@Du2mYNoyakyd&H0k7{b#boNZn2m^ zJK%4KCsKZAVUrdJ^mz2PU_7Tk`{i>^F>(@1UQLd`M_V2R`c;zU!@k%n?o5F>TFvKH z><(=qjjH2B8HdoaTExMz6rPM$CK4%5*P)KRHiKu@xVheC?cd%st4I7Ok0TwrlG-~A5K$wCXV%&?sDJ;w=;j?7{G9HX|x_h8i|XUSddWPUEi z;Nro(QkslnDIyK5Cb5HX8|@nNbY@T)#hBDD#qy6s&%@4K?X&brQ2NWM*Z1{X^*SZ$ ztBL=uIc8u?HTw#z-7kx$B=Y0pUBIZlYUVC%I; zdnh4Zs)}WNQFjsAfm?A+m-DhcQg=}9aJW4&fKhY6jb)wP1`qle{_>jelF**pGg$v+ zBvq}srn1XHQz@{-E2(xO<5|_^wBxzzn9@3)Tvl^3u=}5LB)OFA&Kq9kYi;K?z3e!5 zNW=9tXq!)5oC4+~yjj4a%q5x;5H|Z}MrTMWDJoFX!q3WvKkY*JdAcl4e zjuNIhD7gIO!Z-RWH$Cf)&KFpF=2M$Y{U9B1l_W&9jU-*U=M${v;T0JmzQvR?KW z=Q*q|JLJ;hMDAYS-_eR3GkGeYcDRY4x>9_4bM>MOxX|Exd)w+-$Wrk!4hkaC%lKCo z&c;Uqf`%}2yagsZs!(Z-GlB+rnENjWjIdRjfd&FJrKTZa=? zb5_fiDxd6x>T&XhI%!u+`UHG%r!=>w%v{3jdImnUZQc4KThRV&7I7~HRhD$@jJlP4 zEiKGu7mb%6V})pD8i9K4o%ulbA+02SD!8C+zVsl?`g)ndGkhH_?#|u;QjRii&4*~q z1&;*yRiCHejK6BbdeF_6Jk`0tB}G?uDC)$IHHi_c49nqTHC8=@A@Tl{q9?x=0sFpg z^lq(I0>ULLNVno6jFyMh#Esm9*iODOMqUZopw9Y%>w%|WB08u}W#5+eg)F7TUgjMY z5Gh+-r%H!h>+htZ7IKZ!+WY1;@gsm%IhBA^2z`pg;e@gH7}-8Q1A<*Bj=v_nbA^(w^0+ZB(xSv#LcRTT)C*C}h+ryYC(75`h z`=S)wIvb2fQ1Q<{`frirJ)C+%_6Z5DcRYtqES-}4JAU%(ALB>CG z2L2yQ{q7g5hf|M`@xSipq%j}AI?R`bjW?Hlj*=)B*Lxi*uJs~2jtU$CVHcLRF=_X! zi{YTH>~(=WZi1`s8bl7YTA5(+o>w0+{_r#oCyEt zqYs;y2vf~Q<*_t=HnzY^Fzine%SNNXm7GjMx z;De8}FGDLMDzXf;A~g59B;PkCa%u~7XsowFmdg8Efov$jGjYMeTnNqrWtDwDwA!V% z!m0;)mx!j*RGR|eZ<4Ta9eAHIa6h-0+sYA=_4|GY1^LNH*OpCG>ZWSVhaO9L7q5SJ zkG(9a#Fvta+A+pEDx$}QHNCPcp?ru=xQE%b3)mQ_*?T0rgDO9&75E+n9&Wtx8r^;+ z{){-8G5q9-l!1ZrxuTr;>76EKzkjl$n8=7pv|Rg+XhYhR|{v?-9D&3m;K(+q$wW4#mVbrHi3<+t*kB*P9<9*s#T~ zLx&GqK?zjLw?7)=}c2R}*YX>W4Lr;xpyy zX(^XiSAC@CPRFZ>4R~0wrL=S-0;w>P@BT9d;fh-_bw|M*(wVMbH@W?AJlnGH8d8v@ z8qeR7IE(nXXvy9@W-e-j8(JAaoiotnDUnAt~dM*z~( zy_&0tKAkt+AGWEZHnN^i951az@qGo>U>%*8(}g?RAvu*av83t2IEOWpNACs%pm_N6@j&aAYp_>InbHFY{Wq z8^9UUvc~=i3S?mC#yH5?cJrKK)^W}X?nh=Hlq9RP-gTB77O7Joa$OkXp-tjQ*aYLJ zgJKqu?Ecy^xL^Hi7pczm7$wKm0Mo6SO3&XmTrA|-EnUluGw;slq(Q;^e)-xt)cvt- z7SW`|^F4?!Ygoy76h&Yku#r)#AjO=r|9r7H{q!*;z5hFhey#Nt=TCS!j$vcJ zTre{&?x)mgk=#4SRFbixlb=}qkp&)@B<3cUW5)$JV^k$xPoc@JN$wxT-LbQK$#k~U z*l9zdK4zP=FlH?_Kw8ji_v;@_+SxeM8ap zuB(klpE$25YMr>q>_O`R`EwD6#VFp6VK?YW*=L$_AvPmF|B+_dv9Q@?U!_p+r6TnS;x|L zTxFtP96Ctw4a|3xkEAdGE9lw#{(XTY+@*XI6!oliwmoE3qv1uA(o^Z=t-88lQ4K9S z>UP+d^7kS&`?x2GHprPr{~H7a75;~T{g>>6WgdxvWgfK`qq0u^n%@97&k`$py21T)V{=pR&bB$@9)A(%5Jf$lym?EIgJRcS@>o%raFvK7#hX4wGa!Bapwyhbe9-H7#ImOgBkW#S zfmgUNR)vZzbrgb50*X)MyYu@$+h3j+%Yg6n{n6#VFN<%XvZIJ^_nXOq_cP)E9`2Rx z@56}nZG$|aZy-_6cNp-s^>g|C(`e-02r`L)!1w-GrOBc2#qIeBp;?#VyYC2}$KMKB zV9IMWb#j4K&r=^z+1?&y^1+CY5k_0eNDcY^5Okr2y|cAasmKrLR-==&$C!A-S0~L= zE7j1fjcQ|zh_wq-cY`Rtvd{bwtFbq0>AF|?$xMtr2^2i*TAA$`kvC&;k)+g^*oZvN zl|4dn#{wltC#MEiEvijncyXQho?NBJT*)zZ3U`$tymS3U$j0QdO>xp? zH6zP3Ad6}Wk5utdfLLGq2lHF2ETlF0OJ$B?swHXJn3oIQ7Jn5WM@mOupS?)&v(|ah zjK8XE_#?VUt$hwE7Ka<#Rfdg~YT6>0tvUOT#>PB2qNN7ZbV-}_BOLDW7!WrKePV_{ z&5}t46VucNpy8*DK>q-*?tV;dX!fN4JFoL#&VF&`FgYIgJ1!}E`yrh?%fi!PVp63> z>n}Mu`**x&&gOmY$eqSoKp2Cllsg^OF4o5Mdm?^ zu5UaaKR`CgW%_W5jl`umU#x!f2VI{BfH#bTfmoT1u8yl_d1OQSZ3H0H3kCUT(LXOu=99Qf9u{eSn0VdY>vk>y>!l@osq@O^gL~x|~^5 z#^E5hebezaaTqa`JHmONwJVS8l&Mlf7}UU#4!8fR>)yM7C3@q$;8>8#T$giH(Pw!T z1D{7?=j?e@CkuIcW4RpPY5JMcH-dbRRH@rJbrrbiHE zJ6B--UEHo<%WnHG80OCverKOGJDg9_K5V`HIBFna{0eyb`qvHE zSi>Ir#-wjQUvr?K$b8CKPg%dtJfzHA+Ezrb=T7nRV*?RJhLSIk51+} zIa@YGvVODhk;;-;ddg3HeH5uT)q5@a|1HU~aY#JZy3t5p#SG3Zbu)0HhrObbg-$w6~ z%A_4nIx(10C+Md{n!IAFy!k2|W0)kyd=<=ARImwATy84mt9$7hS>F;R52e&z5D8mr~uiX6oi?YFg;T(tF8nPjPwyoZ8<-OqoFnivPNq$_qOZh`7?#X>h#Oi#5s&}CG zQKgZ}=GTHzzckk_Wlg?w)qNO&h&~QPhH^KuNf_}G8sN8Qj>UzZ?YRC`c0tv53tvko z@Yk=Mk-~5G#D-ci=A1V+RPrg&*U-r5p%3{8F%a{7Xd!@#ovAamu?PNhqw!EO65kn1 zd^cCFH%$q#dHNA?4@bX;R(}aSqY0S2)CUvdR9g z_ecr;?^lz7V=QzRWcknBmz}J_yEi8s(h3<1dBs80SGi|_WHx$D<3wrk=j0jHE6DRD z%5r5nYw7$qqQvLT(fAyFh*SN?@Tu8}B~$o=tVJ8nB6P|^13QSKxf(pKfEM3@0+Nf; zq~k5so)d@_$~+v3F4_}3+vL=c-C>@s->u*eok-`fOct%}DA^;Vi9u(hmdiO8R3ZMNg{WfK2%gje0d8UU>BL7&f)%-q(F5P^#eI7L6zd0VN-pfWQjbfGAdK= zRXX}G7BJK(XEDE%ExL7~J;f2kUlu_RA;_ozF4DmT#6X?lXCl5BloXD|QLyUe!^mFm zci?{N@A2(FolwhMsX$mWZ@oB}HAyV5iqN4NH%(Yv$@>1V5zdyX436H8V5lYvGfXrN zwp|)o^78f-_^B8@W{|E!~Il{$hkn z%K)1H(rlQHd%b%S%$=s8VAh|Ncbu{QO#R7q6aC-*Lu+?9-H>qzcoTiYV9y$ZwBV%` zAZaonk#O2^^xl2=xRE>4zOw?l$nyLHJ2sz`EM%pbkSz$CB-g{i-p z$sx^eaaU%_A>Z?zlj*NFOW>W5d#2Rn2RIfc^mLnyysb&fh~&nvhfT1EAv+p7m3CdOhpm|UA>H0u zLykD>H#4S+w6pBCu3x4E9cB2Mzm{};YV<5p5+blL#_K03&{GFLWttOu*rXJK;Io9k zuHz9*2K&8r`aIv>g?3ne75xhEgvTv&4u^mpdh5V5 zf}`5%L|%j+93{}gdO8u_7-|@Zxr~!u(S7x0X-mg z#Iu<2;3Deog1p>}n#K(W1I(%($+;Ql7R@_&_6UoYVHK@cIFe63;>sf*EVX&I?PP$n z>NP1D$ypr9atvca{m6aP)J$aa7y9nc`HCB2e9uqRH>%8;WS!1d{4|g#UEc)K;!P!zFC&A)UsW z&if)L!jx<#)7|EtjXpjS8MZKqql&%cV6BVLonW}4@hyoHUTUp~s(Ht)5ZA4KsFInE z=>Pk(44-KcczKssPMpBQ>f-StP>G+}@cI=EIM^FsYah4T>_+XXwUqrPH=-- zw=e&wEn7>ILZWQJQsF2I5Q?iG%3lfJVx_*>g9a2Ws>mkV<>2eIMK_$KaTU?(l#Va} zxQ*f15vy2Q>)y|sQLwK`Bnhey{+Md8eQT~I`v+icB~VaiBqlMmxy-ynvd&J)Box_e z$ca&Oyfh@}%*y(|qu%|Jh;ZgQo30Seq#a84O|AB*qU@err*ncjHt(Y}^F0w`yjMFt z%BeQS%E)Bb3WnNgHcH}uWLmRB zo@}Q;ULz+SR_BVW_N%zq0SPaX3hOV@9qUX4p1EinB`b~}3#fK9OCn8DK^>I#9Vxq)JBf9q%{l|G+k>czIU z(xm_upsrs9Qk+F2fMnV&w3?W{JUO%myu^PpGGPbR_{gL8Re*o<| zR4nB}h9h;@nYa;4&_=lXD!wRt&jg|SP>Iy-s7}i1PCKr+8-KzC_~e6q>C1R*hh;N3 zuQ7rLh5PJ~q8eW9Xp>||(pQ?_oW)j%5Ku|)ZYn27Iosr1I*Zf=oh;dUw$~&jMo~to z(w}MQ zVJy?ztt9$XZj0BlHiWl037RU|gm>{5m<@#!ETIMp*kwJ^@|?8=248gP5k%@e*C`8Z z{Td!#Pp$3DE8|3v&%JW;m#5QVGiv9zQcB#bFIivb@hyTeJ7UFN%$bVNkGJ`Qw3g--DkSqbWvl+ZJzL-%p2nCy!QOVY;K502AI~ zM%da{7=ArAu)^G=v|}S89NM@H3W3qG(xRkd*;M{)5_7L14HzqIo`UjpP&=TNZqh~G z^YXKC+?d#W7jxk%QkJe8LcN(N9L?IAu8c(D2MS1Vs6rfZ--s96)hP2Glazl%v6IU$ z1pA<94G#=d*hor>cJxQI6&7Ae_pv-SOL2f2qG!!F1w1+&2%diJ4<_La=D)r#0`f3+ zDF|_Gg+7IqZEWDt2I@HKf0%47;JKE14n{Me)t;G)QZ`_2=Eh+IQ=fnSG}lWoFSWDj z*+|w0!-*2kz>}xDBQZiE7gfVue_`$lK0Y=9cZ|8M2?Zg{uE>VdN$03j=38TYW4C z-`6aoX%tf@(OreJgoLWL627G?>^n8S37Vw=gn2G$1iL3(y=ED zTwyl*!FoQ~@}9|}f&3IFVU16YH$Ix-8I+;+^VeI$a%ARvP`XU0z5Fdfhbk#X%A(S> zX;flPMQGMJh3JBz;Ad*2H11bw$g>7n@MO0~kWIhJt|+G1hOaHHu9^HLoPBYxn?$Fs z8oeDRxSqnlPtomhJKb*}V7rmJtpBP+XMmOAeH3Zk6z4vS0;hf=KU~p$kd#I`gls{# zi3ooq%e&#$BX^W!M6JQVF^pDUtkXPKv&X6tgd@1eTF~_IxGOFED>NV?!0mEHWRIcraZn z#1>W$IEO!xDgXB-jD0E&))Cv&BhzdTf$!MpZibBkNDl(yu2L0m(|q(|A43So6pbE zn~IjM{(vvnXMCTh7cr8P6T@#FYtI)@=iyO?IpUL}oNip6{ZofVNuwuf%|!gU6BktD z*PjA9_awQ!6EBL)u6&dL4U3+OcM#R{#{zXy8CA(N&+{~Rz8HvE`es}V`#&MG^IpIG zzNd4b((RELil|@mx2MaMgX6 znFFrYP@W&DiDjz_T`_t>lx%3}LT$E%kvtvrG;IhFVEFzPNEonyN6%)C15P+IY;x7j7)G#hP3|2p^ZXoBc$jq8yN>JHnoXffBt_lbHap@hC*G!1n zzaP^CSEI1IHXLliaE#G#t);9A)F<*Y;9^uuLG@4|IvUZR2Z9!fdOP9Hdw5w2;o78cQ-qzyK`AMRTcIka&xG_&J1RUBh@ni!uF`7!u z=!r))xS<}lE4o}5>;h$kT##}{ZRlTqDZAC?TsQz#E+iSZ8bd|6_s2Exk@f=}}a799mQWY+S(Qg@~ zeNWOMgmE!#%)Z{fo&7Mb&?jCxk5NjOugUu;s0T#C(CZ_$Wy%%YJU**{Pij)qGr#*6 z$Ty+x`|(HrH|tzJN4fdjH)%A#(BsgaH)*zy?kCE}8AagE)2iZmUS3O$fBaW6!Mn7Y z$NS~z`zhtT+;y|!0`CgoZ0B}~0*I=%AG$JgIKRQ06m?Hte6#j#g{s3(858vAJ}XVu z-7-~9pEs`7k}>z5fw%O{ZCx^#@Z5(nXSPw`{a0h|(BLY6{LqRn11*mXX%#0^E=|Tv zC+76kXmL}FkZt2p30K+?Tb`mRmO}NwMU^Z-2hRF_&%S7Usaoo+RjQgEq0pd5X%0LX!1oHjNrEhjJWk zKVeBO^^)OIR)dpHG^E#-VUpbI;n^>36I-9Kf^ng?dj13RfMxFd386Bja!hC8@;Huz zJN=*99XX?CRa~@Dj^7~BT@yWE_FA-(R6VA}-s7PCwd11`D=sgEc8w;xS|&0_vXwk&?@7 z8+m&wS9(Qcj{8*$u;+YLQFOkVmSp@9Mv^bFZWd~C6O<*(UcK!%aLdZ^dn6X*v_{+@xZ1yY`00Cq6j7deTg zJp}2Dws33uGr0Z6YkXDWN&Kbcs6R+_1TNJL$JBW&Ar9h#ol9@};{4qXzvSr0{GGpR z7a{)5f7tVY_kYOe+$cWoC2=)A?q$x!wX5^Q<$f9u|K{7}|9#o%BVf4i@7nbfjDIt) zS@rTRLwjr8{-n9bewgvcHAbD06L*}R2XAv8#x<;-gLJ*Ekr(RS$iai_&+FbdTJsq1 zZ1G8h*rJ6TQxSAmdhPxGXL9*J&UwK%te){G|8cS-ign72rQ=NjnC~%sLsLCfsb_I- z{}HNXG(of7Q7psxHYDfO-gF>E*k}A%&#)NgSg0=JbcGzsB*pcM{~}X6viR-ar!4V@ z=rsHCb#;-8TwP4DOP-$~CRp2n0auA-co-qwWK23vb}C$NIDXp=I#aqAUFy-uZ-;$} zQzg&5Fn3tu5+UWcZOnP~*mBm35{0ya0Dq6Cfuw?_29jjWR$w4{{wtv8@8G4yN0!Q^ z>kjqi@km;zx+~1r2AjCYmA<%#&2X6wuEdHOf25M4GP{S4Gl$KnmEqt7ad3(H91zN0 zD(E4l5jsdsf_QvNBwr4Pg4tSFp#}l6v;Dv~c6FeU@^$PY*d5=;Nc;yh_-Z)ZobyJ` zXm5(+c+qAevr@iO&kX1b#K!;GZTLM&NuCnTuH(Y&?2>Cu)1|{W!MAXxC%CJrCA!kg ze>Z{l^lHalcBX;8Yrix2GPkHZ6u1lEy&QErE&I+UD?$zCk4HU~=AWc)xjWob05GHw ztlfdpP}>;FC{e!6-2KpTMVI50;$eqgp%{T6gXj7iXiNxtJ7r$v2D;Or{<+Jantw8Wsv-;ZfgCU zgvH6`JF110FHO|!Am+j^djG6r=&~O$&go48*fyCgA(~y8@9h!)iHX zU25&CN!DxbaqyFPBL}|6GW;;#`8%Mi0IhpjBzJ87c{FEz!wozHuQ$!H=I8EdIM2Wj z--=J6F*bf$6B(8T!fZHvhebqM_b#BBqQmvr3Nn~`{54#88V(0U0dzaUG5zuvF?K%W zjkT7EUTGis7El1G1Fwj+$M@qLXMyelQihdz6671Px6X_h+e`u7U%%ZDj1>7oT65X z7;cV%3mP#7(|+>7sN)`Yj)=2CRgXU(*j_TV1*G=4{6re8B7nuerX87@9ow5>XcFyZ__*Cj zlgGrsEq3&(XAAv1`~zp_!o60Khd_is-o^P+7cRVizK7L|c=Bxqb1;jxYO$6oKjqKI<;U(fo8Zs=RMy2s`6E^1VE zUclSMUfp%U>yy`m?Hc{;PwkO}wBSHNX|WY;ScXpDfN}G1J?9XUKL#XuxlB3C1m4>t z=0GV$U-xMFu{Te$YBdog#NroZnpm=)b&9$z(BO}h7VTjLCKkIT+&xnHz=&1NgCZsq zlHs%lA-n#ir#9xC7W~ZE*2vVpxBpz$TT;&ODJ{U)iMBVbbkKmZDuK zvk2}J7n$8+ILwZcQPPccgCrt!u>}+trcHt+EF8M5JE5=T;h?ao`Ycdmp;yOW7S8_A zExATW_$g`|6GdCuYw?O9%Qj_l`YuWz=QYw#?PURc52wxMm7Z}evm1a-ADFeEz7>NE zTSps9^`CnLBWrFe0r;z7Chrv{uGUOWH?OfA(OeR{^UKx)@+^405e>*RUOe1_nW@dFm zL}~}|1p&|~LsiIqZW!V{FJT5QA2+tK&X$mC$8nWjJ$ACdaGLV9JO(k|7+XRYA?|M( zdTQu^H7$-_K2uh{VPw`;bRT~78gzHb!^rUL%^~Vb$m_pi4rhZj(%z3yRbAbelMKY$ zj@JdU2eAd{qHlVLl-25)E3%yU#^9XA86qu_!8VjOUfPH2~1~Til6_M_GFK00&>a$I*hR=wNu31gho+}&7F8U z`Rww+*Un?h9{c4is$_AA8y8<7D26uv4xUT0*WZA0jZIRVDML!fb z&T6T*5=u>TorYGA6HOti0@#~!MAHJ`R?xvzqK1N)|S|~IU<+i8c3^3_h;x> z@GZaA&jewU_JyFnYuYPh?iGVe3Eo z5s7nS1c2F#ig)JKYkSCmf~!9>)~tNrV;ECMQb$o9CzS}tPig*^6L zfvALftZD{?h@#-^<$uSmu}A{WdL*60hhgDSqD~D^bSC_awl@yZY+w-1RCmGddwACq zlVpirybfYo#K{kg_3&k`c z^hM zM5%mTt~J98^|~EnPvT(94P`Ve_t?jovS;Xe55nt zVzpw3FtGTqLPUl`AU?N*)jXT!lPP_nZiL2zLT$%lx-tjB^IUNg9di>Zj%4~zmYVsr zmhOAMItK>8aPbnb-zX!Y(sFExk!$d@T@}C?2LEaxUdg~*RFZHTR;N$n8L+4tMHmdhZhJmccCIt^!3UIB6D}@fv9vi&XQRwwkT|y zLTi~CWRAFjS89KcH`~?JtW6_~Cw{nkLsk;-g8)dl2;BeaJ%zO1;4%b07ysEi0-)p{ zv&dBKKX^d^kK9>AoLlH*K7oP%{2s5H|5=DI;0Gm7B2{Yei~C>lkFv($e+fH2aYExk zCuyZSctL=O-2Y_;{r@%**bO*qin9RhOImQI$pl%)Kf?w)%}S7Xt03s!jx_(dU%)E~ zv-HegOp~uWrj~cc4JAC-Mo%4h^O}ym&e<#uZ@OG*d+*^5D8_P*m3Km$V=xx8(!!^(wM=&)Lgj&q1NPrd;eBx1u%N<6cG~$iZZDp?*GB8d-om+|zQECW< z#nS`y)T4PsY__3zyG;%BXtmq%sN+OAgf~*HP6@gTLwynCVfhfu8iX z&BJ<}A+Q@#;tj!#LMoJTzl|aGUo17$%cuSpXE>*XP#Dlk2TQ?kgWQ%%&lK0k9sX4^ z4fr6AXDtPCvmoAQLP^l8QJV}QV5+fbCSTt8CJnaHit z{pXq8l{(i^mZugk-VWjxO;V;^yc*EEkd<-nwgEp!#d;7w3(2zs#fCkXZUBP4F{LM!^Rmn zcSPt1N5m7GEqj6D^+oH`K6lMplda2@CCKstmk`S;ev2BMVt>I_T?c7~bZq0@D(&v? z=7LS$n%l*Ko!bPhpA$Wf%czxvQOAC^{}9_@X)#`4DhB*O@llAe9U>_b530>55e3OD zV_HbZi0Eo7Q&ytX-HHD7V=}B{!%)zm-tGO#TEQJH1_G+Irt6OTwQWkBU~+KIAjK&@ zxcs1Y8C!u?vd@6+`y~$2ZbvOf*?w{m5qh}9?S!pQWe8%&z7AFZTlC>?=~66wW5z^a zk=t}7yG0-8BVq(@(@kLbu}9gx0!X&hgE;B_oVoy0?X|4?y6vl@Tyrk?sCY*vJf~!r z=_5`*3__;_N>1+`TH+2o?THV4IZI}=8;Yt*ql9z0_#4#PJ|@JGxmjk;3#pWxNF?-Yz@1mTg-x(`;ZKAP` z5H0Wf@htF0|D?lN?ZOv&jeTU^?d>?gKm9TG`>nR&f`cTwp~9KC{rumFPOvFABQt9X z=`q}oyFS!=U$tlhZatg=Bq{W8ap-RHQ9!!Dz+BvkWA90T_PEgPg}NfR^BQlU{NOYC z58jq@j$*@AmMj=>=j!~0q^X*R>;Kjl^F*5ltlY&ea4B`}w)n={=&J5Q>$@dJo|IzR zhxxsynFiTooBlc3dk)m?!68;AN zPF7SH!|w5lK2l9=x*xjYSPXcok2A`;O72}&+q^I94_zUt7cfpLy?A)VAf*4Vz6P$REIYcEECq6-QaW-nQB3-Ej1|A#*mX_}&^Rb_L$E2Fz` zGdu86KLn1-(lG3f=p7(LvB7<&oLp^VT?URx_{>bJGww(zB2aXybS?3Ptxly{{0fM7 zeo|OAYxVMhh}+A)Ts8ZP>Li5lJ(w|F@P)tK zyyqW#qxNeAB+%pPM~`r*(I+Wc$*C4|A7~Ui5N8@I&BMVKRVGC&)?KtdJ!c?YAD`2G zsIG9Dc`xHWe4VC=h=L9#?9_?C`|COUary|JO4Khg0kEo6wxPj?OPzzbtsSNn5XRg- zrW4f+vm=~P$w`uiXSoj3k#se0s?Iw%YNVj>OdEOZtyk$Buz7Nr=S&1c)#o3k6~GUF z)4?eSp7utzIJw@iElA}8y^84TE1C5rJv+b1Va_aL&~3^_sQ8F4ms|BrvgRGDG@+zY z-)SPtQUyl9T;6@`_I`qofmq#=NvGKpLuXsICB#r1IJRRHE-Hwdw4-8h+SyvW#Yb-0 zJeK2%_dWn(O%sI%x))P7gFE88Q>>^4Ey2bd^iM~5Si}C$^wPyQhgh`mAoGJ#Q5On# zrv=0o=Z8ip}|%JifH3~ULV%Ak-h zC0T%NVlRgjXX2T@o1^=H6rlDDfSyk8Tfk@=&B{^gGH3TJXMQkQ(9=~ZF2EF3w;AFt z$X|sme`bNJj_t9*=Dj#_u#Q6qk+8yPXS-k`qk~#L#FbhhUDoI)amT?}9wM*r0P|6% zr8*x>MGT9|jEMjz=8jW>Gvk0=Fb${-=g^I?2NwoZA2bCtSFHy_{A%ANu3{G}g190Z zb?sL<^o6Lh$MQr%G3OdBO{vPL)y-|B+@U~>_#(k!BT@=@{|b<|ZOhv8>)!UnF`AXX z?TOD~y>A3+FGl}FR^}-WuK?$N_>wZprgH?&sl zfEgLEE*S|eDNSi@m{9fyZE)6<_0x(`4z%C3D=|@4Bn>6lx(Smt^LK0Qi}3I`p*zB> z#~u<^1y7+*;Zn*hwSSypu#CdmaIZ#33$~v{a8taF?2NED4Vg*Oy3gj zK=~Rj+rwHTMG-n?=R*<9Myn+zv~7^JPy=?Z;F@blv=ACf+dq>yQYG%UsH9w4{^B3_ zx(dqv7vXSyMzm$g)FB)K02$HlrxFH0^@HG9Gsp_X&zd|q@xA-<{V(9?*`4ivE14pfbJrQbp+tTrLsmfd* z_{#-RJU~@pC|9Y8A+f$nFg4sF=O0~qE&C)5^CUI>hxMm_f@SZ;0n6zI&%E0&XQTVq zANYcQ?I8vd6(D0f9W~#_hHW!m#%u*CYr7js%px}bvYN$jN-X;uQdl2B;q}Dvq=_Z# zQM=%K7o0G2GHG^i&St}modEYrPlw9=TMTz0?fG0*#wPL6fO(%ftb3?x2_mpp{Z;QvD5x*)d$r)iCa{fFA$Qm@m4jkK)LJfHC82h~^ z@tdAxU3cZA?F7jFxOjaZy?rObQS$=TFDr~`znPuGY}p}pRv@pU11=do2Uqk!6R3>> z?W|HT2O~vk=Uveis?91}K#VNmi@RAYJ~(fBM(EuA&H}c89yc}nHd-V7@dKwt(nx~` z7bu4?(@g@)g|p;|GPb(ED+%uPanPJl^#XtH)G%*Vkk1P`-Y*)73=u|P)+8(9nV^$2 zfaDVXbERym5Hrsa%&SX#7&58scBd@om6fj;sVVEef-lC)#8JE<3GhJS%P{WSz)d3z zf-VN9&0^v=L!5j#971Iu!?{(vDz&-wzfM;9zQM9HT+Ix5}jjbT==ab1ahRA~6k7af`+&2^B z_kwcv*>M0=LlwE|VU>wTdd0~A``0Jz2d+c@am0u-roW}Pb^Qn%?YwCp>htq<`8c%W z_3`w!uRtCymk8A+-r}Dn2hjr=s)LO!ZDabtPQug!uiK`L%s29vg3IOy`n%kjM(YQ{6}u2Px^*m)}PXlb2y)>2JJB&(68#&u8WzB6U#kQDd6 zlHkfN@vgdB(uDu@>@OfAoi67s(7u9jnsl-L@O$6$LI)>u%U_HL&NUD(k-qm`&-Hh_ zN>4Zal}xGd1bqm~X&(3@xnaEcxBY~4S$X4q5??i%9QW>{|1c_l?#cKi)ce_Y#I+(W zSfKiCHRsYA;D2rn-DJr9$|j~ejJKu3klK!G9U2gi@yfi;R{Zwmz?+Q>Ky`OAfy3B3xA6F@-@IHc2ho*+G)|a_&$uW19{|7@gElF|wpE|6 zDPA>d)8$iig4X4CnSb!^n`A5rK_bpmWt#v&!yfpvX#S@b6!-Kv)rYOq$z}J|V65h9 zLX8YKjQUa#)Dai9fnghGT2}Vd*i9s|{jA?Y_rabFMrpb_3|@?Td4Vy=DBt~WyBZ^+ z7s}k5-F8>l_;9R7H_{ix)K}S>1zNia(D$yK;v1;#92$2Ktdk2vzagZ@0gVoAFh{AnTD=nK$xV{#1 zj(d8S`n8BZU~95Igt!0HDBf+}g}RIh7t;3$eiQv6T2oj*-B^(f^;!(&pxz$8@fH!i|0`* zn4tyvw*jTl2sXJjX3V`mcVvcrS!Ug{na37MyH??IHvX~s`#POq9X9K()*T1zoD)oS zOCP<7C4~M_450ZXZt+!KJzcXp@j2fmYcUTWp_-T2q$dFGH7yh z!f9}ytON%T1;K*I3RhgM&%p^$(1(H)ik!0ZsMqZWjoy!;TLaqxDd%eUelS{C&xOvW ze{+-`wbXwjqF*DnZ*ss0Ir}Sk7t_0@a()9n*h{u_;YBjAyQ`qJO16A}sJX?gn;m*i zY~Fi?osq+9E|Wfpg`E7SO!Bx$?rF zr*S>mZ)8RTt9^JlkOCSm5+bAabh?%E)yR&DX?ea(nNa}Tys8^Zd5sn|jsT(kQnjI@ zRCqstEJ@2y&UM1p@s&Q@o$g4VD^RyVpYT?^KdfO(MhrwVyQdp$ZHnI7q zsiO&lAM?}PCzhSNQu5=5wCZ466QsFAuy3^6qYvRk#O;w4&!5|rZ~c(@>e+&0&I+w> zJomjvw6fgX4#-j^H+CVkRjLzP3Vmjq(H%9LkGbL4yCcUPKmG(W)D0(emmhAp%!nx~IGRgtyh8Z5nmscBVJs#60(+VCe7G~}QJz>AvZ{!p= z-7vCDMbE0`C&1fgUaG2MgeV`Qo+!g3^000vD0>^cJO(SVInQNMABbJk_R)u1r3drH z3@d9qj6xl3!^6%!2X34D*sbuV*gAzZ9_tEse7x5507VvVfnld<7fd3ljM`mCZbI4uK2SXxv=% z(a7(elyqU&85TJsuqxE4A|@y4%X0Vf*&F+EN8kv2lRccqQEjL&KRNXw%O%=u>?Cx^t70|L1 zkr;6f+r%6E_GrxSj0+Uxm@z5t)K?)fYj5Fz_qW%}XjUY;*}rYZ`oZoC=9Y&3Ynu*r zjTp6K(}ZMREwBI9;dOnYc~^cHUQIAlQkezb+r<7tE4sbxwFs^BOQ(Jmu8(Ht+sxR| zIxXMLzhJ_?O7*>VSYt;t>3Jy?t=;-{MZ|vnRbAQv`pJma`MV;mW#XjiH#(^%p+RV) zRz5c#7My-_h_vzVx^|3Yb=ghg7T6dN4F>E`>aPI;D!n@QzU2r^)&WdA;~$9HaDG>4 z@nXyFk!P?zF$N~6!I#)+whF;dL$|!fah}U08HAer?GBax@ggcDMpA-P%QL zGJcbcqrX(d`62(_f_)7WW1{Qg^#N&2WR4#vm8X910P^R4~(eON1WxJ-;X#4>P2Ld5CXS`ejX>yfw`++tYM zDSno|5rwfkI;!nhD*heMNZd|ShTBJbN=^b@Yd8HEagjs`^DpQ|#8%aA;X8V$sg<6{ zQbp2YyM$p&4|xJ>_1Udd|q(SRyk#I!WMC^5E8@QD8Mckd+ah++Vq-m9JitU`+mO}TuhHp zjX!N&_F;5ehNsZtYs2iS`p8x75lvQ&{?OUsaN-iJBWkEf3iEc^Xl<-Ud7-5_8|~TY zJ!`e%1OaZE=+8Yi*t_rG4e)s>HU-AS(EC8 z!dS5mOOe@oE(y;UE11xVy`}E6o^^BjR~&kRy0x2&h(Gj3xkP#!}=Qgxpw{%Q-CD9U)&oyw0J;iot)6Ml+8h8Oe>w zvAeJSo{l(yT2kjMrx7c!uD;7y$8r=Il%QK2>Pq{TDAhoCSX6$jf!UpYbSw((>hr*#h%s z%{1?;X(s3P&XTj!(-7%n$~XBbs_mDy%$jKO1JeL-ZtSc2} zTM2(h*}k!%?V9F0vC`=evjkFhi(Z0IkZ2w_hhSQ?1+W=x5zY7m*`j3VDx0~zHBXV2 zAR~ot=*%C4cK$s#!>2Dhn_s$}UFr!TrJ3Mhx9m{nw`Sc<2Q(>I>FFk?>YTah{&p|( zOg{oo;@A+6#}W+bKn+dPi7A=u_B-OyS#R4a%GuRrwomZw+z*=-h6IBKkf~7kQ+CjK zfgd5$#!LxpDhno++eDSDPN9mjPS<$I*^W4CEs@Q+Mc#oujJT4*tkaD2hV%)+xX#{7 z7=7qWL!Wd$w8(?jd2y-88~cGANz}Op#ttTHbP2jgWR#G5OGj7+m_+9$IUXwftA=wd z#tvm>9czn=&2eesk86SA-XQoovGPeUthqu48Q=ALcaK1lzfxKHgwMwE_jc?cbujGf zm>HsHjS8FFRJ8>fDjPIOM@OO`=QP5q?3y#8u#DrLr!O%{uA`?~Wv1sBa5ODop3q|! zo)LbV-lynC zCmY^Dc`2a!`}(B^SK;77uw2@2a91X^=I}vo;9L-@*z8Imsgs=y(V?t#aCW8J_$7nv z)N~u)J?}vKRq%k0%q^*H)*9=AAyLMq2x{1=HA)P4#!o#!?fK1GASq5VJZoSf=cyg% z{r%6iPkkkv_`T z3`8Ax)aL=ltR@AsOHru-BIiDyH^WYs9J^_f3+@<58`9;&-0k8KBym&WUI>NEg*oVL z(iPntwo9OeI2*5$B$_I$vc*na_39e)@;*~+|82W5snErsqNCRnRjl>jiz>mo=3l6h z-k-}s*To8%avl(SrkX!{XPs9Ma@;y`Sj@NZms+5sUe_UcN5nF!3pvt))C>) z2j=Xgj-d-b53nda^V0O`zNBcr_op$js@$IC2%cSMQV1}MH3;WLVfC^$3$j}n&e9CD zBsh%8BZ` z;T1koa@!TT%E5IZ!Z(m*YL?xpo8LXzfCGy>S4Ug{7kX5{Zu4gDc~`DQhI|fV*>6Lr zO#B;x7v^J%o4uYEQ=QNdpsGpZ;g6)@bl>ZbW5eADZaWcXOG=4kzjs9!p)6U2WMOP* zK=WrRMu{^Mr|Wj+xQCHj_z zb_0(XG>}WAf&Wv4pf7qGlOAb9j0m$!#>E6aMM%g!IwshX8Uh*BBFA;(0P_XU-Slxb ziYls*kqp!OTB}}@(11&+dH#IcKj)fgy)8(}*1^eC1(}GUhJk+JFTbCY?VcqaCCg?V z07wRxi)|_Krhj2?vB}Ps;M#1NQRbjsNHo$^^BUhzg&Nnl{3j@*x(^PX`pGFOqoml? zHil-CCaQ&<3exN%lU7$fA*#WSwx&gqC?Jnf*FKwne4U;)SE?s>kz&(nm}Qyc5~lhM zj{+LO;0QRS8ms|hNshOFb5n_Mp(?sC^u0x~E;H^?TeAzrxg1K2Yy|o6!xPfJsnz}1 zrDI$o_Iqh2b)kV~l!A?_tbGM&))^Z3_kyWnD7XunU4b~g(!z!ka|7O2SXQGe-?(X zq=*SGy>Xw=|0=FQQ(>-YU+ql836D!?T;suLqDwZ6`!?lx3s;P0C)y1D-Gi3?3-qKz zUg1J@`zGnn{R-WB!nsl-wQyY1@a{1@a0<_C8=KvX=AN%A?`a8?kwDjtk@$I#vjB4V zz>V>8IMS0ca zc#GT8r{EGW7K6DorFC{YjsIDj2*lJ?s;Jlk*ZyN6j0N}gUo~xH05bU=B!$#6Yj6ld z*>7;H+8zTIdKcX&@eS+F7B)`;B@l!==yt*jow9*=%Wkhbm~C$*$~uh0fIbVV9}FYb zBuA39kH|&^@+IwfFTsX%#&9YypxE~gS7i?@NznQIjWxWNXG-h|So8%-lcUC~GThx3qr@vyFc*uNh};c=TUzVP4-N%KP>+A3j$~>KFxC6+E01JikjPU> zNKuJNAwJ^a9A+rJeqxEw#;e@&)vs}zXP<@5n3_*Xe}{CFoEX8l*x5tDW)~h!D%0AP zwQ;w*a*KR@&9J{ZnNOn1rWO!!x1hx<*YMS=ac>1r35=Uhe32ZkB}S%D-Cz=ps&yE7 z-5cHg2{kEW<5uy+E8tU0#iQkkOMovR1c!?&oa|7I%ht2QI|!^n^HM4JGi6Rvih;eW ziJqroA0xsW#5-`;RJvHEQ_qd`^*ITZnFcyOhj)e;@q%h^mqiQO zx@cKKoYi(Kw2|`p*o+zYjN$S`y_yUkZ@HIhq4!1qR>5Em52CskV+)Tm++`L@Mwu-%H0_hm!M9{AyF~sXgF{eH?cKWxvEH?{UUd#cF9tc@GjeD zvGyoQjmreig&LEk#$IEFmdG*duyp9cX9}^YFz{$WX9~HusXZE;FNxYNM)>fSqWdW@ zyk4fblyIX49GlDz4W{WFdI<$temZBVZCT557OrEzP+ev>S)s1eRj0EOAKdJOWh14M zBOqQJ@>vf7wxVdw>=y&k)vLu+fnkJhp)u$MMZ8$4yYL@fjoruJUo0ioB|17-4e2O3 z1f^$dW9-bI9x~Y_;>7s{Kb=i|>%=2|9uOSW^p9_>DztftIK;5=G&eD_-B!P?)d1zxB z4l&$L`Yb~bPHH*oW6fgm6`(DoWOHwe0X*gkyMSCId804KKwdr|%R_F$lno>9EI^bO z5uf-%qN*7k6wa}MDzTBo^~&DnB$5M}KGK%-nrpdU-Bmg^bY^V zPqwZaX;>4-W=K}`&i;Iy91wz3R?qmkVk`7?A)|_wR%WdVYUI>l8Moy~a-Z47OtZ(5 zjgJ=Dx5Xbns3*tCrBuu{K{QRO;l*%@Ef;|5@bNno;f8dX<#W$`umRH9-tbw{X|)oK zQiU#NH3E)FOHQK(kK76PEt1IVXQNgxKw5fr5p&viwq~h>-kRMJYx>vUflqepZO;@i z5#i*eSf;a0RPStn?@EYf0JNyMp&GiD@b6MugfaeiHO* zCm8ut*pHA0Pn=F|5T$;jK&2f=uY0v?dP|x;YA7b6c`9iIZ_u|PAs~~DFT(a!f|%zg zWWC2wh@;n(pc|34rxy)jSZ&_RCePigmB5h3b?b^O4n?^d$X72Tog1O$_!0x)<_1`MX<5EUVM)-e0}h|)Y#EV7e*sjr;XqIZ+lrZIr5A= z4Mo0>BeL>R_qg}^W=?n=0Ne2jmL z-9=)UtxK)}O!=XLbdh0nOPgTmCVP!@y@FWc0B=^q^8@IfcdWc!jSQ~DPNZ!&yM4dKA*@NrHYHK{PNXKG_DY5F8A=Lm&}Fy(v{qVm z#KVR8g&UpC!|g~|I@1Mrg9E4CCi^cY?%+g)sW>mYOmqsVFGf^I^do>K7gu3p^tGmB z)JAeNO+~*b$K-ia~EG^0~J*$@3}CC?i0?78?FOq$k8 z7Kn!>|D+%91^9{P(#Krg@P3v)C8Y;foJMXR@jM4m9%qSNrp}Uu(Ij{wP~LmS#dN93@>qv60h-1?Hp{89 z_O;bcGuQxt{2el-B{j;k>eKQpersS|rG2 zJaiyrz0JtWZI1q&tCy%%$lVSXYnJk9=ud=3CqG&kK&=AhtS1w-0`9R=D9_j_&vFV> zD&#w8{vQAcK=;3r1d0iDCOh>bDd%9c!sO7(k>r|@>0bVX#ZSfziPVN9R)r+`!Ba+` zSOYRWttD}tAEV;qacVuLS9!#8-zV31xO+B8_8}@e5;!d3mFaaH-S+&IZPRS@*S)V; z-9;Ga&%+plM$ALgs%1ZQvC%SJ1LP% zCqV?pMRe~B>A9pFqL0@%>@Ml_oW91FcknH8wr&`Nb#Ui_6Ux9n0?ilLEz`0m=MvlQ z=_z!m*R((@Rs<>Qnx-8iL@1=Mnl=W!;Fs=@AWmRvBEK_SM5_-mWxQh;U5lXhe zgdBH(tolZ-NfBBC-Bo}IxI&dDA4{7QO*e#HyOX4Qi|``!H6IBLGA`ebgo z00?6<)3ayii{8}r6u+3IiDesIb+d+1(`pMpIW7;ZD7dJ$cXS&S2=+={^IM0kRyQnT zxvQbWbIfhG9{jO!3C7ieT&mMk2`*7nlZf#1Hb!Dxt$M?Ox1Sn^`S|5;V64UpojkAf z@4h(LD_y!1oFwv3x)blkKN$haFSp;!ciO``JT%Ys}W<4cGU>3IaTj^FGymMieJiw;< zuKCa<;h_p{j;ZW#(^GRm!XUyX_y*(rc2_<+M@vs{T9EInU@cehG9S3KznFQe5E`j@ z|38xM`0g$bUwF^WrJQ(4{Vy9R(s{dRB{R1{Ir6@`e`%zudr+Zg<5lfm$8Zo ztMh{0vcjDNkGnNV%|e%Y9+Ha+d1&Aqjr(g1)m1^)qlYBw+R#h-9^&p(x!?a*i1*^cIqvPQjBky(#H z$K4S9lVKMkf#=6beqSBR-{a>6#&QsYkJ3zI{0Cxi%S3-o%%XhXy>Hz@HwI|_u+Y=h~( z_yhkveq!`~o-Et!hAA7-XtvNasa0f23tDCRgzRQ~1VB&W$B`n83Xz8hwZ1X4%o9IO$#Z1c?OL*V?20wxSJ#NQkpL4YU;4m8c8C z7HbIa?y%NLADd+(q^fAiPR30{SXIq=cm%DV(O?F9t5fCB(aQV5kd1KSzS$pa_c~8`w({a!VE%5mYz?%h=)#<78 zSDD@=JK1bn0qR+Tqm<9A!b=n@3Il>-4RJ6ymFO&P4wU=8xbgiMAm|e_(KK{pW9u@! z$U>bH*8EoAfJrVrbboGGQIm${Z2-*$1_{^vg>U~uj<}VK*e#m>bHD7^z z1NE1$2)kv?mefK@5FH@msg!(Om7zHE%H~Lc%?Puz=GBR@uTH>kBYNWbAw2+|+dk~6 zikbT4cTb)@dA4WLj}rAw{VVvt&JS<@_i{Dh83{z$&768CCDBzMj3Rsip7YKkvle2@p3BMBs}2rW6euZ%@|I8TFQrIlChwIFv&(w+_I;zp3DLG~XBP18*Ie zzievPBH7BeuFcyuFJF78<}$EW7I(3YXGwqXUbU^>`hDER%1iC(y+5!h=hwWrur1^+M>BKwk(gW1y1dL)RB{L~HbVI2@7%4nKEA)h0=6(JxwP zGvg6A)tvfLb>?WVbyFzvUt*H?f{@p(RG?HClH~OHD2Pi`CBsQIqNZ!$)}*=ZeyK}4 zM}@33r=bB96nHC?ykc{7v=*%y)LW`v8_7GmCA1N3Wl|mjYP_khdA&2xHy%vc!tW>w z5D5XmGG38qi)T*{Ugl71ww+Q^E-?;abN(4NY4dpe;8AF;;cZo<8Pl&#RY7A>J|P*R zY1s@vx@)*%2_;l#D_TfKL_^xjiK_E{O&q)up4L)`f{kd3vRxj0F3kZ1ChU%-W5LsL z#i!qO%|D|4)J{&w8NPWcNh5?hKY07hG+%wJ(~3}_R$dGJI%Uq|I!?D1fjR%lg!v>d zc|n_E8`AHtaVDT8vFesL0(5qAOB>WT9a66(zdrkL@$MgQE?&Pn*AgY5gqzuRHPQaa zy`S0LpmEmS**H6Hy#pYd>^!ujD_gCM7F)tTf_v=t&~qA5((ElX_}7tch=*!kh$x$+ z0VOFvw&l*CqMoex?5aGzf-^+dsywdY&!<;j`sE*bmt0?Luo)3O04lwPeF1ukfa{Ac zT5aol_z;!#1015)&^!66pi=SdPld>C^xP92Geh6(&9ovH-(HiKT;9aNU`_9=9%2Hu zfFjxm0`~ghhb125L16ID145z!)(*>B9F+(h+RB9#PM-Yzs%Z=|_ z&-SXrZ1;P%<6r|c%}Q@YOYSHJ5kRIa$a`KtC$B2mVn&GC`I#_HGcwuMwP;iqDT7@z z$T!v6RuGd{=rV~EIGnJL>h~uza`JH{_5W0AB~KRQvUJVUc(r0pgWR%Y<_kfZeB~=5 zkHQqe-+%x1<=f|=Cg%ci0@Lja&0xCnLls$UXbESG;Sc^=6`xZ&O`$v%4kIjMsJPt z7X{50Gx5)Z4a!r}E=W@91xc_b$UxI${Dv*lZ(0*7zs6MC;OOa|j+>}m2C2UKxcC%$<$c%uhF8H-=l&_y zZ);PCWs3EZrLQ+s3sO?Hh2nJ}hD|KXYX*ayd)Juj$4`93*Gwlg(z>{~cE?&NVwJ7snZN%MNiFqs8oU72ym;n2d z2t<+smQn`-d1S!dq&9j9F|nK$&&f9u-TB}Lvms0&RM>iHC`1Tczok`M6u8!^VQl7m znx1@mEuwRr^rjloDWc{YP<-fXfP5OJoO0WQbcg-5?!MDs?d#s>G3yZ+Qwhiz2h^pN zXf$s&eO_WJgGG4KO4cm!r$3j)ja_sw*mo@0RcmNz-?7!u%9@W|>!(3+c^y%`w0Gt%<>%W%8Wi8oCYz3Z?h*!rxd zGrfKrIuH6+RAY6$hUNnDks8?X@Ga%BGtEn0eKyh{u47u2xAI@b9DjXY=iT~$K7r%DQ!eZvVvpvOvyr$- zuDRG?kyuK|_ul(PK7sc4S=#+c_vSs8Gfl@kTo38waxI{Y)8MQ|RWX!S*6#qd4z!^q zQ>^K*b23p?rrX8anpM||W;em8$WcG|#p?(62>4pI!2(lLnbswW;Q{(NPvg%Vx5ebb z*R)RaB-yujbOHAvQNYb|O(SsYygtj!LkSEZa?Bg3Z!X}vG?MOLXlS#sqBPBFm+Lp%%H2lrS4&OxCmXSI!`|$Q??A+0R%pM`t&J+cm2=*`9Fzh2qihoc)9yx zkT6pCk7v&o#C}Oz$P52Ev4Aq~JGm5>;?viXW>T(sZaqu^3=Ct#r+IN~lZTXVidUTUj7%80oXcA%lWhJjMs z9l*=`hOz|%=Vjha*)g{%y`((JRBN^g(lGUynUsiw7^GR!=0@9yMj|huzC=1CF@*TP z#pqH=w|?DLbgZX~#VLvq(GaORKfhq*uQWS$Uf+mjL#!n%=k>Xo-bEH zP*87O`$EnfcweEivJQ#@6DUr(k>hF3_&o5vjL(6$Gd`(zG(KrJH9oNjA3puAz$VOx z(>Ed&CF9C1TEdDzYr@(SA`9}`2xUB6bSXvADkPr546pk#z0>+^69W*|?lq9&31mC; z2NcRNk#DA#D(D`rmqtBYiMBhY2kD~`b9sEe?v4j{btLVyGZRzS%znWZJ&?N%GjVZG zbJDj59|SMx9#HkH5`;KUAr&&k)AuO7Ffz&0JPDGMvlp)5&L$9DR*E$%nz3`=!@%kd zD_Ii&IJL=;w&ra^tK5LEWvl@~!age2=)HJxeM$0$-?B!MvpGf|y~Wuf^E$}vF{ZbjRen3wH!0nO?%HCFQ4Jyep0t+ff# zc%YLA5pwS|-NRC&yO7IxHga%IsQ&03z2&*pZLk-ILl3N}G~PrAl}jly9t2|0kGE}T zVu5zlYb)r&kg27C0(!CqgYQXeU5T7swabFb&Glrb|In8)`O2B>F+4@$q74JSf5|Gz zK`+s_i`Jb0jYhyW#MWo`z=*C_zx~sjZu}qOC7#&5I%nThwf`NG(V~#M$QM@ZhjnyK z5=>azi41^_g`s3-jq7R3uXHLdUMsBeKf@)s8{6Uq(>v5`$`fZ|X%1=v>eVWF z7ZVK;d=4{5@Hr%m7|}m`9V6*@cj3dC^ThVUhdtFIoe>z)_ zWQKXWWTj}fGe<^$y+_x)G2TwLd!W@!+cd0F#r6?(F1K>AV)pLoZ%P*~1NW&1Vei1! zlF~wm7s`b1vg(%H(niK-Os1<@IltvnB(aOW5w~v>n3W)e{sbY{|4#DYcedc#c{n|6 zRg}!JHiZ4hfNXYWV@KzG7KX;LPfe9#P4%4o@3ZfJ`_ugV%d@lZp3eX4@4r3!egXgS zoAWQvf3pAh?fLoH*>`_^{pa^rul~-@fBLR!%NzXHPiNn;SAS0=a(@2huakz?)A)F$ z=AxP7(mw~Rx&$J2fetw@-6iFOnA6f9fz_v|*irD-{s%(C4^(@xqtsZ_WrfY zZQRNihx@fZ1(uVw%AP5y%I@35Cr`4fZFf>9w!4n2x^vEx$=DDHNk~zI0)Q$jllkud zg^deAiUdGO<+jtri^Q!WH*wwA*WcDI)P%EI7nfojy#_Ji-j@$0g8yAZKR51pl2F5nK{(WDiYxGP*aM5&GXWpRqF^;TS$FTOMhS%EkxMh zjo0gfW&nK;nkp=_+cr0V>H7gH$5%paa#osWf>h0kaUHy~sQmzrVoK3B_4k-eaP4NY zHU&XOsZ}pU1OuX)f-YSp*GI3?NYh54B$XrLN^dA>R3lwXU2a}V6NJw2F4zFK_&8*l)?aKcw z(T_rVSo{4l^JfS@Zlr|@QxKYvTqnkOWdV6ric>UuY-0oa2}7|=zVeDRfcc0MHNgO2 z!`TXn60gOQNw8VqHO{CE)Gp*WyH&4(vktm~_*M>ajriw#4zAT^&#-C^K#!=m)~+%3&^RN7M5H(-nFo2thCHmezl{4$H~f49S~8 zV$4;DDm4h0$Bkpoc7wdHW}vf^kKMNF{B!?y;1Gaiiwcl3XI?L8WY-RWVP3dWiAstDYq(lk5vta@J4UE0StEqI zpY>g+OAadqpif~>5!^TR{<$Hmb%AAi5XZwmd**V_?W)c+F7pX4)V90&>3J+fz@5}a z4AxCu4%HSmQWDyrGm7*EAgdY4bFah<*ii0DT}D~`$JI~IPZ6Dxsr|1YNlV6F1ClLt zaJ|d*fDI2r@C?mCnrfab-S@v$16ObUZt`2N%)#&8s~d-_2NG(7m^b`@Zdae}{T0iv z*Cd;oGCc3KMvO=)c?4SPib1p$&Gk`%K6COn8#ZC8yBaQ-VM)qpq_RzeEt%A$buC$# zlSg1Cv{hLpW59QS7Y1owl!$zw0w7TI+``ge9cAtW6!l596Qb49kr4x6y#H1*`EPFP z+vq`EEFa+wqqcOFVgV&s>LXKNR(ZBXD=H+uB~x^*1P(uDL9b+q0GyxXs4685haIUm z-S=S{d?Dp>jzpEIOL5J~#EL_)&ELgP4O-BenecV>r6txOUF$qLcwOzeG)J->#0_I- zWmbzqu&@y%n>Qc<6Sa1#;&kqrmXD`P`fA!dW2Y>CSSsdKnjIEOv*C#M0#%ouhx@LWmQ zRKl{%ltAKKUH@4+FjMVb4xd#sg|z6sc+da`#fnIrV~GcaWWUC)WR?kafeG_MVA7H` zohHE66!J;D34?WpIRX$RIYn<+qKj2WVuPp*h14`Snfsb%mLx}c1=4uUOk>W#WRt9t z?gq{1O`Y+d)#0@;7hSSu+-s=_l2TGmETDm|9u5;-m3m`0r#xW$p$VvScWK7Y^)UBih*84!OXp+Zr|A8hxq1nA-LBpPjlq8YPQuc*I5@jt4Dxc zOFxk0Z&(2p*)C&?mUy$jiMf_uC$7W{bZG*gn>QdQ8sA56S(7!bV93u=njb3NsJhCeO)GUYvjcPug;;BZh4}xUY zt)-fbp)@a|mHd??&cT*$b0yV==By|P*Ol3|_Mf&58t^4hYDN3JPM!J3sVZe>k@VeW zavGoa*F70%q#QYVf%04!$MxNF{r}4GUa=a8U?)N_)C>@-D6};bp4>E#N=w&6ey5 zwQWY!)~{iKVA=d{430z7t+`+S1jQ+gcga;8=R|!enIw#K3bpWXxiyh!a!GkUSz|7@ z+8%N0dY??#&kam`X>i~}tX_`a8tV;HXWc@^m*^buuzJbktXKKIMwya`Xj@~HetijR zUHC*tnI4z`5EV24;p;yb%@g-Kn;5}xSd_3YAS4C7^~(r~pNeL*PwEIzqYgOpGrgQ# z2fDWU+9{&_)=0l$RI+knGSZ!+!pN95Ojhg9xJ;8`v{~1PDbQ#CBNtbfhE+c*SqaQ9 ziK?deSjRMXi}U)CDf;s=2_Fgh7pU*pICTO6Epv4%!0pOnfW5YfD$59QhBCkCZ+#J% z=~-c!EtBG;5fTi*MTNvgQJ1AQ+kk>WJQtA5k0sT` z$xa*HdsL4l?O2t0Q;K@*EChnM!c=v4(}-4*t+1pSSf}XR^Zcw_=>+TGPdii~I)HFv zrWI6d4w*zq-Z4a1N;1b##B`pN(Qljejd{CZXI7M|^XFw6CRAfTX*-%OWUspEJYLon zW6Qy(32IG&*U(k6m6{ij&osGHwSDhM5w;@QhDDH3iK!hu8o0aYqSX1@R9G97NuPL> zbGBX^yYlMsX>(+WY8w%RZ-{_!51ZDZmd=*phdONvlP^*&oj`P>o%|1E-Nzu20TU4Gyyhi zPBxTPY4c#>Kr5-asgOplT+)Rrjb}r?z=g^4#B*S{vXEF-qM59ofn(^1t^&9A_i?-G zXx3CNG#rDc@pgJ@5!y4o2at;ko7(BnEuGA(PjRa;!YXt0a$j1=ha72#W?1V0-- znu`fY$6Xa`*yeVqmP&ovwyDvW|;TP~g_R@>mM4IqTFcrx=%1xs$lIp0MhLD_e{&p!KJfQt+Dwp*2eaId*E=So+}6KBfxY?3x`g4Q|;R z*;2P;8^OHXZ5sE3m10TPw!k5rbF&-JPq?5%c9B0qt<7GRr|1on>i=FssO`XgsoD!h z#2Y5z{g1LpdiVz{k+F|cuwCnLc*sQ_*pIp|F098bJ{9P)M2wr+-Qx(DPU`MiTWRf@ z8A!3Ce`diMZ>-?ay6QYgk(lV+rs;qc795W*eiq6dVM!JVwyI`Z5xxNYoGht8NZkhB zQM&E9uZP7c+BU@gs9U^5Nb_mCY0^dDp<3whk%J)g3L?Tkj_SI{n(j(&y2D!PSmCwe zI{&SN3phakGeA+LTujjg60}?t#HV5HcHZZb$*LYHMdKbE67mCWqrJ2I6ZjF>9DScw9>k+aS4u)R8S*8Qh?ZGV<+^GMkOzB8 z%E{SdO845p^{aO0+TbqdEKIc?bS2I}6p;?Qpu)`XaYvuXFH%H<39L@wfrM{GADQ>& zC^ZZ1WKE^%dphF;JzwIoB!vgCr7d>7ea2GaY^i;87%5Ir5P2fdk_m}cI9t-vL7$Wk zCM{A&X3ct;e~Pj_u;kE*sX1h}XI->MAN#r#(d$?iAHd|d-><7F7&ZV?qc35oiv!Pw zH*J3rBy_oQrc_$a3FmrxXT-b^q}#0WJ2gd6mh2AY4BZj%k1A;dVva-#60$dXh(OKQ zYR%b-3Y&6k_Qb)!N<|ffx^Av;QI*XCq~AYB67vO-iViBP)rDMHjC#7NE=H<73AkES4iTKS*M-B^`U1VgjZinZwJ|MQ>Uq=V?6)bI4I z<{jfjen<0o4!=N;6rsnVkt8UjgDU)lRj#``nh%%^c)U3YHW*z^0#dVpXiiG04mS_W z`j3FxWAu`0#08)Yu)C`nEo=CWdI4g#D|N1{c|s7WbHX9bnaL#*bX64+mxNWK*ao-L z$6SQ?^B)SHym$2R~0S;HI_2}A zt`mzC-?%ZnDCQ+Aa)HPP$@>e3B7l2dt&||^=*!>lVwbvTwVkblO)fEZk@@Wu>N4zS_{zXd?&9KCR$e1~H?)1=<9f*}06IaLXsR)Eu` zf3JKM3=kaM9MUZSYXi_`8T-VwyZ*T20zZ|DK~We481Hx#YqkJ-1gH2GkkiP@Jcd)zYEGL^Yk>e+)(1nyFqUJcUKo6fh zoT6*Gq6OxK&VF|g0QR}CNt|)l5pczyKSYlhhaP3PL2 z=(42JRWA$6ep$gv6N;^<8f*1}%2O@r>5r&CH@*@eO3;-q*5kU~YiLN)rfg}ObS1&n z7P7hF+WA!fuCJu1p}1tuPJY&Kj)m1>KNRPvqhJ({?9$iM_*V)xfu* zTCIuw(5G=gFARda=|jd=D-AZhbf^<)@Q99c95#kHXN@j>oV#j}=XvjuZShz8cuPZd zKbuUsaOeW4HNdY1RRl8#>cT>Rnk((hb?WTx%ZnGUUrtx~AuNiN`I^x(oH^R6k7i%7 zd&mCZ0k!8uVj78At>;m)H7nSnySCPViTIU937z1Hn&HFMnsBk(wnn3i+o}nvN9Lq? zs3)5i1NpNL{K!IKdp2VJjTzAg3(7ypr3bR^mEmqki(wt9SCYKH2|iSabVsaT%&e`wR+Aaj1g6WUdMhj4i&Ra8xy% zzj^yiQBd-yz8)KNC;>p?+IxJEe7y&Uw$^qh6?!{znu3IF`8-BqyDDgTTVqtunc5*b zbCQ;~cK>pM!`Zy;+u@)ca4`3mgRIP8P*&&ckCsiCU=LrJfA|na7m?V1=8-3;_6P7> zDl$RS7%n{8;I<{YyncRt+4PI-89$v6Kpj+&`}`t(mnn?qII?QC#OsSnE?-aq*ih*L z%Fdf!4u&oSWxNrHs)-J6)SAZgKVDoRT&dZUrV7V_l0iMjYn_=jc!cJ?1IA}5+{fsS z8$RFgV?`|QSnlD+3ez+_8|=pl`>#GiDE@&r4V>tI72g#tsiaskKFDv-c*~hkDk0fQ zpD-|E(rC8r8!+Zw&UXpcAe*2A!JJ=~N-M#Un6qa5^C>FHo#_*`@1x%uvm)RC7r%R! zT%B&G7OS-t)de&UwyDv-r?Wl!_p)Uv)kFP16)yDjy*Y-NuQ~ncy=%N?J6bQp8)q-6 zL?qe^mu6&=<5H+~Yx+x0jFa;OJ(Afv=-)ef4op#%zPO_Md83!(mCJz%b?L}XK-ksk z7A^C&1Q~w`&6ze)Iea=@PsD@y$6%a$i3E<~+FgA2;u0uqElEdKV8CXNJcXH)4p@m_W+9 z*5S2^mzqNkn=ir;^7__Dm(#}B*v~vR)~2^<#+1DA{FflzdzZ=SOb`9<@X;E?-#i`` zq`7{eGLVngV#K8eyEDA!Tj2!#bNZ~&Goxi7vH^n1@oq( zTLKnIxuoUZagXVzw(opJm#s{7+LbMLU5Va&f8FrZ?Rez+EhueI-Jz;eIq#i;d$M5)DpDr`~rQoiENa+@9H+Gc1{83}o(OG>Lhrb++Mqq@!qB(kZ8?VMk}FcPNCL=$F{p@dzA zc6Zvn-{|?}i@gsfi%b!z z_JcUTTk8mZT1RjHTQY*m;I7T!X!b&7I9UJCTYdn~u`d`yhFZ9q<6tmad1vXqxrpt} z-kgLp^|$^{G?|EfS{v_m?Tn&OtHMgZI)?#P=nuEog(X&aZHhnU?5@1SJP*|COG
    bBeYLTg)3BW(nVMHwg^k6czNApt*qRkKtSriz3X4P$GYZOVum=p?Y@- z5(FVSt}`;B)Lzp=WDc_}!C-2F*g$g43nGUh!O7S7=9QzWm%#wY)Qkrg3`%AP0!(#d zey~wp&>Bj3hULIyEN{)kUgLy1tIY^0`<{=rgYb-1(nc`ozemYx{0D&I(K8f%V^|%J z0bupba09p=n)1mhA~;*3Te6*`7p2zF9 z!omoGsT2W`y!5OzrY;MFVk^bM{H%*MB~rcI$x2fvk>rZF*d0Y8RX5f;yhrh1a~)Tu z_l+U*3D}1O8A%)him73R0Z0-)g{b(OBvvE)gq0# z-rKFI!F#!(?vgU;Nc$~`WKia%W}Z{@+FW=8HxE`auwX?cU7G6NH>YkeFk5U!qYYS+ z^=&$M*yc)1zl6=J!!L-Tm6(RqVG8<`+ih_DrD4^V(yRIp3-n7I-A4`w)NN=`U?SO+ z&_Ue>?nf&S@3BtyRSQk!Qh^$heBuBZM~X*-@?*tqQw(>&uw!Q7=F^EA|Bxz5)&P2O zbve+Q;Jw*nPw<}jY%L0x+M?hWr+rZm>!%^&PqIe8zw)%xweQtcS$n9GuMK}eB{OE_ z`tq#-oRcZq7!P1H_o$Eo{YXH1yX{sHhuaTbL-AH^jY;pdZp?-N=g)NUHe8lW7tt1{ zx~}YiqI1aHRv>kHu`}HB5Dt0t8ya?&M7AU=JXp?B?R&K{S4O1~VHTMVON_Oy3|gg-CFazMJwuhNkR`@i zSBB9%g8OlYW|BK{5XqP&2G(j2#keKLx2_B#9k9eC`!siHCiyrATcpC47?dT**M9qD zSzYmw%;?|M{1)THWgZ`sbs+ke7c=B=Y$2?|(h8qYb@)mU6=QL@((xBzbZi zif(qg4?EZO)n7ws4O0*hC<%()f24!EpZXWAoYL37pGEr{?r!;1TqpyG>(JbRHfB?G zt=qh8{+MGKt}SS~`l(r#e~wr+4YQF*bqrMdj04rKO}RmDwIKd%NM948)O7q=BGJ?+ zHE?k!Z~ZY!E#Af`wJ}Ol$ zSO=>8E5)gqGS3%T#;Q`@uv?N4fYP!h>`(w0z_x86#g+iphh1{_8^VUGoGLYbk3D8z z^#ORxoaF8giCfHY6RAc!MHh&ZjIOD=KL8R4e%HIsR)9=ZVx=I(+*HXc{j$!rQf!pN z1G=omA!e(x*%I2E{LT2IQyIP<|Bct1M$`~Yu^Ym$j^Z^n_+aLC@U$k*dop(mdQJ4cZmM3vY%8RNu;PnHp#aDp~Xyhym_GBK}a_( zu_Cwd%l~n%^xw>!c6G-9Af#ldNRJy(x6|D3RpS#+zd}n^aWQ={%%>z7dIE21lakf1 zB`OFO5co32>b3fwSP z*YK@UN_Gz%D_zDHnA?cOYJS`@LI?u||09 zc_~F!S#VYbK?xdTfv2s8Y&{BDMg-iUFY1!zW!OgvbfQaiZy-cn_(66>e#UZt(Fuu( z_?#@Lkb}y)>KB@uQ_ymuYvZk|Leh2NW+ym^x0aZ>C4KJ*_Dt_sqkn1G?KP%h=KIL? zfs|60V&m!}USp2k^k;0f!s4-Z!XTVQZXttTsqRINi+UEk*q1{$rz9mVgTB)y(m6bi z*j;IB#prs!&0UESOh(_~?3R?duImnylW$ABGT`K{0yfE8v?GzS+BB^}bS$o!7OM`O zsY(n0^;HkY_>a)0e@-_UNs&br%|i#V3Bv2`48ty`cPfCtJHIbrfa5~3m9Rc$(W6=$ zWwt#=spypQ#`@^=J`iN5(R-H9($L0E_$+Vou(vp80)1Do>=wNsAhiD+g^&3xKTa9B zu?X3}KoK%N3+vgRQwg#aUJTfd@VGNNz}eJ40hc;fk?F>1`_+=&O(dICf=p;I_AP0^ zw`2=}fXPEPFEn&N$-u}9zbj$m?RPJJ6fwJzCsv;knVn^1y*xA2_h&{&o}LvesbqXW zMRxm!z6CQ@6b8+W&CzqRUOJM4D-#@f{2#aa)S9vM0y!t@>E^M5U-YT+HV&T!L0><{ zH2xfAGmc$+>!VH6@r!t!G^%*{2|&#>Wq0t#b;LBcOP3=-!n8p)6TBv@oZ9WW(V_5Rgfs{3kWBEto_ z{2@szGtV{^vupFChoK))L;C7C{K;(|RIuGi4=hhEwXEXf^Czq8Hr zer2XsJtn(Lf7r74 ztI;QaM)>|6k?8Y*??0sOsb`n7DF$`*)0E|fOUFerAVEq&!3pTavVbTB@yRYfK#Nsj z%j_$*prz#w%y_xym*)TmxF{TUY2-Pggvdz4eee*-xqj+2>i!cPKEk^bbb8_ zJSkNfFUe~X-wpqD8Ks&j9DY_cqZ?G5!FvbMGFL+&qJbZ0o`?pkW0!saZa|U0ql1lR z1l=rYc6;SHPZ(FfmEKRuMw|ifCG9gatg9i01MeMsV_CuM+RPIC4GY0Ciemvnuou9} zRbOK{Tw{li4uqZbfoVCa;}*87Z=S$m*5Tc{w6t?`@L=@58dT4(er30wpY?zZ@Dtfw zGqc!M>@)>e>*bC|l6y7(O#O#@_cj^9vG?!p!tQ?}Ix^5N zgBj>Hw>&bRNqOf3bFq7~&Et9GVY+^f`BAgS2jqdOz0WPXSF zUo$KGI8{mZtwl1l0#rPsJ26u`J*gupEBjcv*aO0D)AFv9v#kdpQ3i9Yd-A3e$R71F z*6kjyWZi~b%sSgSWnN=gkJMU-~S?ua3A8u6Af%lYm5Wt0b&}PrBuiO?(5Fq#nmOsIDyPzT!<;^z_T&a z9i(ccMJv-`DamRr!@Fd3;(hi`yrVb-memTE6Lo^YjpyB}!5L^@B2#3}s=ivm@&_qx zx_o^KNE)th5@tJh;89^(`IpfBg;}0W96CzKPo=CV>jkt+VxF*tU5!il` zAzbQ2qnNG028P1$t*>}SmUu&1#WiKg&_na5hw84}vM{{2wM?Bf4I~~EQ1^O21{?aq1lFSC4?&v!HlaH*+x<+l93U8I^z7e}na*r4b5zihpm~88 z5Ykj?gPdYa7X{&R1$(I@k1WiCl@zawYIaX=XV3n4b905(nBx_Zglhx0>S+i9DSj0! z3C>9ZTlu9+)zloPRLr3k^uq^VKHO?geSic`7O$9#aZ3M3p| z#8lClSV9xSF!WI3K9|<0(2u1cQe6?V0++Yp3o23LQ4eJDhT~GGJJI%ZkHKC8rHLcu zMs0>0)6jMnu{~ULRS+x)TH#xDhex=Qm(Hwoe(+Oi{0JpW0xAR*b|ti@1Z*1-qBG9! z1Z+cw3WjrZ#o3C;C8>mSerUZFuay{pJJ&O(ycKo}(xC7sXjZ_>L*6#N`1t6-?4y3rnnh0O_i z!>-l9jQ=5 zuGLb(xR0{*|M&X3PHNd}?f>uf^~B4IsTUs?#r|u5$f7lSboewS*cMfHpC*b$I=COs z#@SP=(oWGECJji3`3n_b*b9jOP928(@#fX_Z2Du2E@9g=3a|0Z8r!yQ+qP|6Yi!S4 zW81cE+qP|gx$ozB^X>g+|L8h9$B{}WeWrd?s!+Z)$FtdMYnq2FK+-M*&XtAv1Xi@< zWJUyQA^L&{PK%|(!RH=&wirGq3Hf()^yfBF_tidPOp!Xf=l zTaxS~8Q8U26Ry9k8?&>iK%frahaa;*A24<4p?Yh|U`l1)!}_)8tbw$Z1%b!@m@6@@ zi`nanI&v4ftoyWfDminvMfyeos~?O z6rAcPJ7U7oS>b_OCD6=ob{q9R&{9Uo6xy(qcriI^zx0QLC=ijEe~pKPj3tY(1iAxz zcBZ)@mX6QOWf$A&y+wZUR=bC7uL!7BD!n&VEb}wZ@Q0ed1JOZjtDB!ZrT=+jV$sq$ zF^aGsFZVLM<_g4^h06W#iu_Sf7KBo`QPu^JY4}_|4A0hPYyqxak8WbNI#)hwYuXRB zz6)2cd3Bq;bs8h0`F*>6C)6)_P2@>tv^PvSzY2n_8G5jl=*E{Oxaubv|1py3MtT5wR46`jM`W1`6`1`XzhuM84uCy0v) zt2{(5wbR5J_=x5b!GvIQBJLAdI3o|n)T|Yc^M*;G_c%VQRNJmOVDgXb6Y~I7>dfYD znQp$n25aoa^1pluO3+lU>{#y5)-De?7DRE)7<@QCW^_A!r5PllCdYTCerRjg!yE`B z&^8uK0A+_Jwa`!`mG9YvV5!Q^`Cmbe5K1bgMqPJ-qN~tVuCo|Z z-}3P=OC&jYvr(^I*@P@pEL~G@W~%}m^MyTTKy_spW>ZuEGHO{_`>fd}Ra+GK*~vy7 z6Mm#CB$<0mxn~hpdUoGRuq{F-+Q0#sxM3ZXlME5iDr%MKJ99TaO=j(Lnk`xzP9b#a zO)Ap$tRb5C?lz3Ju=VAXDkRnRFi^z7|LR9B>O@CATI8r3s-RP~gPAoC`w&!;IrqU- zIF@m54j{xzfDnDfmKv=GR9_|eM9GZX2>=`iaEIf(E6hE1p?$B~(|}}T$+yUq_*`J= z4ZY0RdDm(*FOt+?CzU-+N>DAsbj&RngZF5Dz7sFC<9zJ(!a3fX5ejayeLT1F*q4b{ zoOwocn?v?ZyH;`r zMtHB?1@=v51okI*uEB}l;ta=vdhucAr`_Mjy{P(c99nH7x?#-`=eOMPjN=-QMYP&T z<9~-=UteFF_PuJqzek(3v){k`{C-Y9%kPP$`aJI!nWekF&OgVg^0U1heqKM%i>3N` zkH-IaR;UOWe-FOCVizA;2)bv7e|&2)KYwS7z9%A_aCT*zIn<#BWM@z$+^osb<XnpCV|h7oMj`4&T$)4VrXhm`9+HFQ{)fxNrMSm^>A*IdUi|Q~t#lErEI2 zuBVX0+gYab3(ry$u3uE|QzFz#6E1j*lFZZx3;fYZk;#Oa&Q7#Mts(f=Kya)wUe_3j z#GA9mqw7_ETVf5P4k7s`^%+C-hVP9HzdsX0ReU>^OhcYBno90(4vcb21?^ z4Xs620qp!+3?Wg-%u~7eyzFC2Y={yQk)A(|EK8~lNQRyud(Kw+C}!Tc#z3J<>yH!^~LN&70KnD=d&(zYW}1_Y$?V zo1k0>g)qD?61?5dBWMtn{N1_#B5yN~x8K%b7_(Up%@> zuUKy7Rb_K;%yp4K=SIU^YYsqF$ZS=#S3kvSi)cZ+Z>6ubkNI{}3R&(1BlzNl5CgG8rnQxwH;u?SM5W9*#OrD_rt{CR zWK=}XYystqv*GWuK4rkEGVg9eT^r%6Z%3}E)v$9Rd4{;t_MeM&qGD=$+2?3vjU*ROQiKtf;x#Z zzj&UiFVEUwDR~2VD=-GbZ9AcNv^GLO_%3cS8V%@`|K9Oe#(KhshiVyBA;(mjd@jLv zrL&oU*4X?RF{@4D!(9XofSQpoER-V=&XSOeP~7_&F~eE{*OBMtJ!^sjSyy)v<=%3d z)(*dm7~{}GRw)}rq~_gT(g4G%QMOS+^Jaq;13oHl;(dJD9La&AEZzObDnS`mQ~CLc zN{Q~&eyf9W*TRmDWipl-AM07C+;kf3qS)0+`5~#xFP=l}Q`|0jff3gl$9FI3nCmGhU_I{crM`yBl zSCRsQ57BibWPl9#btL1s7WH>FcoR2a+by3wbvoO|{LiLDY@LAhRB__5b%?S1Soh$z zqGraovlggW2DQK8%6NJewNdqHY4PumpBa> z56<=n4m!h9kYi9PsK2=87b!b(FrLEvqBGKx-0?N>_UiD%E?AFC8S=3-@K2&^K=Ip}`9?q@<viX&EJip6HtH1U(^LFVj z(lx319%Nl3V8?VZcU7WX-}lYGjkM!8Jf+G6=UdT zCiP#e(5`tfpr%g!UXMKqoj)u7^GmmDZM9yeU6cOE^*4_6>2!2Z0M_abxH`>U9Dl?Y zd;*atn~!%%W0TYq!{F-8sW%`Sjrr&{JKYyz5cUf_&ql{UG_cJ50XyBzRt>FLENMxP zcbMW-+ql*^zuO<5({JwApYErh?uQ@RuifFpUH+fvo*pt@L3=10 z5iPofmO1NRT3Bvi;J-~tD12XijlRWf zZfx8CjECYfEJthnx|D#NX`KwEIM#PIehUd@2t_B>CH1F#!5DoS|MgkKPSwn#OjHXp z)3AW_l0!QRdI4T^LR1cn;ndE)L0*hXq^wx69$=<%k-ZPxUjLmqKvBle5r2RNg7ABlb?kfKeZLQ|I^KZ(<%)E3J3^Xcuc zk?+^|;rhUok-{L2kw&AOk;?GSNvHX>avMJ( zgAcv!YnR5C`y4gNKt{*lsQJLBaqRCk?*GZ(796E^1*ybhIB5k1p1193IPmufu%xUa z0Y&V>Cm9V`!`{?!gfvJd$z}7{FFfz%<@;L|XsAK01Q)R5rQK`FwShyyX2TbPOTJw46yybqPBp;ks;Yo&1#KPQ_f=bR+EeOHeF4Au2IkBk3AR8{8+b3cOXQkjRD2%t;_O1M_QVT?^R zMPU_9bkY`;jHzeT?Fc0fUTO0JMjWuamT3US;b<++EV`PBj(>Wo79C;?+ee{u%SO+? zp`~sPR;T<69_(gxYV%e#qdKR?DStB0T~bP+*@(#tXYbn4L;K|C`2F>OgeqdGcCZAB z8>Jk4i1y2^gaQejy-3S6ayZOx1$#s9@emg^{`UXK!Ya{&vQM9z* zDt3@V;MjXj7ukVw2_1l}h}tkioDWP%D+yp2!4k1F>MWlkQ^JbxVV=;rHit!+`$xt# z-={YS^W+c;Fer(t1{qA&DOo@CZBvNrNPIioREIL0==O<&=)O!rQHA8>N%ao#VX%8! zsE8=43@sDK(IF3#TutX;GJ&MP1$*;#@JCMY59|9!2JC|u3&8`)z=WFhsO%Q8<_7;K zyBr}K-X7yWc5fG^*)7URmQ_iUB&S~EiK31sS=ZDcBj(K&2-7lV2Av)VgyshJa8 zJ?*fL6!MgFxzah=V+J|3J6%#kJ!?tMT6!xN(MW_w= zGEe&csvVSD`@p6+Eg?^x|KUlZ8r!y80hyN^?QAxf?IzIT!&X!}oA?KYH{ykhqs_x5 zh1E&~5AsibMcCC5=(KPEY!h5gl40qQv0*)XYolQU*Nqgdx9*ASucV|*d!T(q>Hg13 zKidVvdX-+SgoaR=i0NQ+mATL+7kb;@AW zG>)N9xAk7uYI2nWVwt@RpNS~xS?C*~se1HlhVKFK{?#B`A998c2A=Qd-vls70_Uqbe<$%*KXSkGHT$h{yVfx1*+!y5HvW6W+#4BpX}l#f>QycKp5Gh&(c8X!UT5_^KmC65PwOV+Lb63@ z%cmZwU~x|NMB8sC+1dnS`-X*KiI)nKLSrb}ROwiBt9b9#k8j47B8vZNjy+mbLPY)5Y zD3sb@^LE{4kz2EcXYIP`u@{KPM|oZY3f^99v>vat8QuT>0*Ex+s=F&#{!i+q_Bf@ctj7WNuzDf#&e|Gp+8g2uF5lP(n{0 zaKRo7PcG8C%Qs)atUC%4m+g)HxSPSxG@aiwiSqDdmB9z2ZGv&^OzEWgC{DJ%T5N1= zhc7c|&6{|>u?l-1vFj8v@&eF4xs**5_U^O|A>!(4|3)97x#4ye4-0mS)dm8Y|N9 zPBCJwhTSDTT(u2k05iSrp0t!;-wQRD+OXPOqu^diVyJ#qUX#;oc^f0^@|80}o5THy zXznF9hWaAi)#@g$GT+AZ2@*uKd5c77tUs5iMm+~Ee6X|$bj>C2090`J;5;Ow(6bYV z#KRiG!<{V2B z?W(g9(q8!17x;c0u=;u1z$0p(rt-X(gSS z;SQ|)XjOP=Q=g`ARg(7CMU%>La0+Otbrpa9sG3Wy!3mht!Z2OguTxN8zVaiK-}$F> zKzZjZ^ai*?Q)yFB*Ij?q;sjL&R3&N@Kttq~i@FaTxDq)VBKFMqr~)5wnA>K#D^C;g z3{MxVkaQl;GkX?&y(20X59Hf3VBP%qt6gutCh{A!lIFxHiY_tdr1*60_Y146Gell< zrWCM5W#b?yGv0}m08E#NS76iQtesQPNs2slR#7yDx*pbQ@_E-;9H}k8=KU_{(&OIC7*I9{(QO~llH^TjBg7PoKJ@i4yyhG5|1|iGn23wCdZ^-` zBk%9;@2$JO)jz-Et@^p2zka?yM_-%d`n>N4$(1q#~;Uh|6}E{%A6Q|YF z6W>5=i^tPJdh!d?OAl&X%|2qr*fYNd0znY3Dtw4D1>q}l5+GU4W|3kk)rS%+k5-gb zp>3@;PehIrzA0}%Q`(!(=kw=zY~NK~^aPWtvcSUtrgZG7*VzFc#at~n#Nyk$?2}M- z+)aMGCYt-)GLWkT`KN$gNbf;311aI{k(S$ zX~AOZ5FUmtdc@fu*yh5wd9M26IwCzRCX_qC!WZ3bYNe>Ft>f!_@^WOh^X%_TIAxb4 z69$l2BC-`GvwJEvSfLI@!lZgl=dz`}JgePWiKe!z3jgs3@IvZO%#oSR#zVC#e09OV zkHt8Y4V)TCTWIMS7EMYGoBYTG>^_nsp#Hy>9~%0)ka0h%EPN$wbr^k^F+X zjEdJwjqs{&g1)TDoYJ^bRaL2-H^^AfJXtAr)~SJTI6W^4>SP@t1sIP^WbP_KKJzL^ zR#SVgZR7N*9(u)t!f&xL09$%tWy6-Qt|s632;AxE>JfZdti!lo5A%GDhI~dUgDO16 zvAJ0y>3F|?$!RRN?W#1RrV61x1REor4d|!vY|uK@h)9$w_l;FA@`wd%e+5%urX{m7 zT1@FXjkykU7xA9#@mi7QX0$PMVN>^0ZJ_K}f>;c1rdyZe#&V`&Y9Z{ZHk)bD>yc5W zxMc_IOVMGL{`PIxLCH$z|I#4&&i(^U30PTEtF|uNMEt?w-&z&f#hb3!xNDiT4a5H7E=%4-zkI|+V_7tdgi-e|34}c5eJKDH z*FKe*T2e=mj&_(%yuAv-VuSQLypcMAPa$v>VmXfOX^D&>A9%1xPfdwp`xAFPbh8f0 z+{=n~5PP4a+iZ{e67O8zzjJ+>cv%})_0QzEC8nxtJG9|JiUPI5zIFTayfqhJzISzG zkw51r&hbNd0YSoq{VFBNXTCz0u>z2fTG>P}1T_PT3ypeZs2N-^$s?wW1PBV}AdWpR z!_{}jvKfM`OBZyIN z#2qmuofdO7D-;+_G)3}(DZ=i|kwM+jd*W5J(ZMQgY&T=XG=R!8T?#IBBC)*chnAPt z$M=by={kCP;}F$bkC~+s9FiBcATXXW@!V4KZDU=(=oW@|1}4&1JcOwO1|lvyXJh^( zFba%a3&t^sbg@gi)V|5T*HmX=jFsfO+fK1(Yfon|`~`Ks@H&lh%}J<>a%&suJ0R+M z#knv8%`U0i%KWZFCW$}PN*H}lHdFTd4|vD}7Oh*jl^yiK7yA=pFFo zM<;I_NOKvlqn0faQ{9D_v2!}J|LEskY);TVoO%Tv{Zvi}MHLqk(w>2!(oLyuAFEFG;U&k=FLzY0$iMQ=;LiQr?vVA z%u|NAr4MY>>eko)LOff zp8cDnL8)~?Zb}S}n#9*pqHzr|TJulbnK`-%Nulkct8=sdukdHQ*Pc3}tMJQ|q)=CzLYxxa3i0Z>q5B>M2mF>_d$+0~=)Ei8y*d|%j9BJ3G(W|-{~$SQR3BwxA)H96Z< zC2ouhr}*tNjfMA?w?j5+pr&LeeQEu3%1~?Mr~@D&D@il%m$XKpqweie&tQ6H{6jC> z^SaoHB)C+rh3rOw7=tbhqrYeIBK#fk2&WQazu=AwhtbQx4hSErjqi`)AL#7n@_rsp zMI3_0zQ2_yAeuM=7+w8%T@njr`j`&$T#G zK!GO{*xPbVWdodM+Ep-mDNvqq29p@vQ4U=N(m_lX7$xG-T{pGyHz?8trP+uiQO z%ID4v+_xEof987L_<1nGz~=U$(tS$|b<>yv-*bNPS{)P3?z6Yv{tfNXzawVkJbd$%t+=|3)8<$ zXjs~Onb3%Av>V!xp)gx)d*}Ag=2u!JID*$&k`#&rxh3MNFjpR#3R^*bK#*JTWBTMx5g z3dkD}Vu1OAq@UMPhT>jmwvqY!U@@kHk@%p9qzZ@Rg(&eiZ#s*twIXssE5=fQB9lJq zZ4Ef*JgNMRAAbwDHoOA%RLY`{=L9eJ^sARUqh+F zMNHa>24}(?lSt~As0*XOcG2jd(B+I9nXk8%%(WVG^5rIaYd?LoiuYcfrQ&`!%!&Qy z#g>=j!&JQ2#t-|up-|uLX)@M#eV(5CKePXA`Q6&|(W;n04E|JsS&xe_)Q_~BdCFkf zqfpz6R)jOpvLKbvHJc(jrftD6o~ns8z?{Vi1ijqjRpV<61Y&x>~8gcLRl zE|#j3jI&_LWy6|=lx7`Jim6i}CdM!=m)(sG7PtMf z{+OwFhXEF+c!z=6PvLPz=`p7iUv+Twky7JFlr^I~q1}@Qyax=E7l_2eqJueaPOUJ; z&X(G0k_j`@K3UIWDG_#zE02LC$R!>>Gt-VG$mO0v@K4yo{aAV_rxzc#Z5+HU&TWpF zP}lSBY#o-)@^+2uw~jjqS%AZP?h_4$)i0E!_UYj7IENm><0QKAXOecykWOS{^ywXc zB3DLi_&Hi_B=P@8fa&M;bCg-z^K%hJuJ7~sv}r%n`}J`Cj(#dX+spfV0`1}(Y%i`& zN^+u~mAmVxwS!jl+P=0s>+1abo8F|~?j3#DSzdFf1T_#d*rU2&Y83SH|Bo-Lul^^# zT#rmvVH&O~!8{CxdQNud{@(y|*KT%6R-8{lN$8>f>4kr9V4KG?&#v42j8j0ko0Kbf zKcqvN;OC=1Qlbilb9mv;wyJroB^-o?MHK~PE{z1Is*dmfWo-NvMxeTUh1EqQy0rL( zO~F`o1=|E-6kXt+0*p#84)*{)p&j^4@opuF8UV0EIkwEa*z%CxvvP}&O*EA;{^p;% zNUUvx|CpQUSZlT(6CJNT+mFHz`KI8!zrWQG-~;cNUd92NOR{#>mlRluVlq5?hJ5Ct z%~adyZbcH0Ugj|OyP(vybvk^)%B;o8k5%uLda=1!z1LfbxcXBOd0ciTx^pnl@NEwj zyrP{xb@r@W_bu;-c0-#(rN*t7F6Nc2;d*T<%(C>OO22u(UGo_8{;`)^?KZL#QI(Uq zJzaKRTO+77gR)jflOu7Lfwt;H?lkV!@H_iHbJX8iWq9ePFOMoyPJk^KA?QOcF!|D; zfvct?T(&S6eFXWEGpr$o{1+$Hql-Fw$kgz}Xj-WccyaBead0PeE*v+j2-%~R@(8XC zBDp1X1y!#n8wd5Ug1Wo~rt0WwHC4&u7nnErY*m1xV@e_{<^NqT5LlJQ{ivVt8-~ra z2CI364D3S%rp`+X8gG12626RRi9Fs+U1cBh#X>9O{bab+8?fsJwHmPCgn0J%GU#NJ z=wH;)77=CDrX(HM>ktModEGd4R5ovXyrbx*oN?+U9B`s-rwfnj{9#=*Dj;LCh5C{% z`@V~{v9k~FdA2Gj#3o&4s=?GefHmFv*~#=yKuwMhew9t8JyWCR+o^gF*?P$mWF3Q| zdyI(JP^;FwY86{j9}^X9`xhhKrSK{tKcN`49IV$O$)+dDyH&_YMYm}9ul)1mQTkVT zdM;bemF~_XhP4``J?{h9@fke>-aTF~UTOTVRie$N!7@Jvqfz7!d*-umYY@fRvvk*d zd{%sN;CD`jj0#)sVlJ|j^H8& zP1T>xeh9i-VAZ|HL0&*vx#l30Wnad~;?R(PR1;Ly%9e!!3LPj##`;D_X^1d)3?S4p zvuPed0>7EnOtoLl90m)T-{AAsA9A*g7B?fnEl)S3HmwbL_;X1z!243hGHEijv0|U_ zg;Jd$-XtP+LTVGRwflk5#29)Vl^+SC+bS|w!ZDPy)^VS4-T-3XrB-vzyY10uV=Lrv zk-Yl&HQN3t#Cyhc+rfA3_vspMyAw9r^S!}Y%loRodJuzda0F~4v*7{qVcP{Xihbw7 zP7tT(i5?TSjVfMbqpOidPZa~ld>c$`rHgj5qWrhBA;D!^<)dF+|d3)k@CWt24!;4NGLm| z(pm-=;#3e+H?A8`xVRm%#kuZGjZ)I$bGtiC4tw02KLr zu;okO=E##YG;pK3AhLZOpE(p(sx_l|FtMe~!lkW)7HW+=){}*Y;guD7ulO^ge7t;y z2r>SDIX0!ID)=E`ST3ekQAD=nStu~4$lBnShWaGpO*4i# zbqriZ92^H$T4Yr;A0TNLjS!irvScIZe91()veImn8gAUBV4hvOV?y_^$Y%zt^-<Ij?C?PaPFa2=mmOIBb7VTQgH z;HHNq~C`o?D&7rd_FbwcbCVQbmdbgphHcwKB%@P!Wus(2>| zPz&|V*pCjJ_UaEhjb9@EK^(O-d-?G=iw)Fp^#7WqQWB$fVy|3ILTcW`LA%aE_uNC( z!4TPa>dXhLHQ{$mdDh%vM}rf;!1lKEI^p%{q)vQ&-vdlkF`jMEjYJZO#4uLyfqx8t z@;YccWF>!GGMMs#rKx0=iZaO$`tw)O%}bSxGJCj|6nR7&r7FC^U2;j4(IHo$f%LZ& zN)Y=sB?sz;sD}+;Ht#!%P@6LdP%Vaot4?_Y7WRB9<*8o^U8&^u2E+=q%4fF`J2a#m zBhQj#n%yEBO-;C5)zyL};+Ju}U){3c%Ra_Jfk4Nw!oJJ7Kd-x&+6@Q^kcb-350#7P zrll%W6^V0flQ)Lyp_KQkv@lFc2VB^IaV*jGsO>o^eBqx-_?imxKH~fN1J{|t#RWSq z=uWkofHG{qwtkPtc#-0^oxmmN&W$WrF`a2wSlnlT&d_C>%UB}g3a@TKCEutF!Q*!% zN=|5^<7!sGrps8oQL+og>m%$;a$%$sZ|mr>l8MQ0{efrL$5%$k)kxtTB1&!4Pc?S zLsc8*YzbWWur!v1TG`}w`$=hKb{GF@!{lkC_AxZ0%Vjpx&fy z>}tKD-2H2DGXW4F6j+9LOK*@>gm_c)*4IsaMA^$k$mm@J|FEbLnfy&E(vOw12%pN;nVW zPup}5o=hH&zWxaW1k>k|P1|${2u;9cda#Y}8GDrouUsGc^Sye4!#yxz?p$U;!;XgB z!d*xiK4sqjO{3^WO@N#XZBMTWe?+x%t4KrwVX01~7O!II0c$QSiRgw?U^`^&o*}ps zU>e@Don4oDPD)v5FkQAaGG`ai3S&M-WfTQ+ojEqZwL;_FXVmsR9NF7_N;sWOPF?uF~a?!z^qW+76Xj)uuS z*z{P`TAQym+$>uR+bmmz*eF|!mz>jT{7UfHLWbr$nRA)FPZMT7nJYS; zqj8il3dFS$q;L@%wUde&RYV#5)JI$uRR0#M5|k=u>$C`j-qsN zHi=a+S3YJPS;)SL*o%gqG)-}fnV1u@yq06qP1nFVX}#|>g9q84c8c6Ow`RO%-vVn@ zbsM}ghWn#^y++>A8etDuU(f8nmn7|reZUn8fs~X50f@xlmfv6R1M^?Y8DS zQEIh53oBwW7mQs*`{^4!D@iK z7F)qat&VCG<+IT&=@A)IDTeut7O*h_UTrvIrBnx?aDS{CZ*!@s$Anyx z;UKRpe@qKE&zlpBcgu)a;4>p``02y5qv!XN5wnch1czqE&N)0HZWtdCKz0WbQUD`?n<@!gFNw{w{n;0PzdU0y7^$O;2rwVrB_`bqDows z!qWZ8gdaeTS*Y;ZOyOT_AKnH)8AVc14X6Blx=XRa&uMvAVj2!hJr=%x-rN_I3pF;a z7lq%q?}qg3td<7FBipz!6_EA)=eJ0~L?N%BaF$k)inh_@x3M6vEp^%-+$SZ!%Zs^~s3mLiEb=~`LMII>cFPhkDUaSL*1vmqBCu&A|0`{;INDchVk z|IpY`4M_*)WkK6SUQqJjpe39D6Eup0L2Sm#d1i zoR9!|*tWv;96lOK-Am6$Y`KS@ASddT>)p%3o=-pzKx%peV4NTaQ^~lWrL$^kU5tWCR%`+y21W5t%zGTHHnb5w%vCGaP&JU`(aoV?$}Sg|&+o zzbK$xI#E#1KK2z{Y4rBCe%ub#kW^V^byj~PzUQEE!ARhc{kM)P8`#u2l(Hz|NGP!u zfdpkVVBgJs>n>OU1q>_$Kt?p-!0;4jfaJ-vlEO!M_rzDs4I<{_r`j2>=TQggQah*m z{Aus|)+=Mg^pIcGtHq~;dMdSpc#r4vm2>E`e}W>17?j(X1$nAx_K7=s=Ij|I@82}; zX$d1|20U_m?0xIuowi~MFz@YSDjO*0e6c7QKztg%ybixuVn%Rp zKExJy&2<^W!iJDzGtz7e$J3RVhOI#p{mhP0Bdd=YIU)EEAYq)enih}3P%KW+4lHM8Wh zUp#q-6|6KxaZ0i98cn-1kCGFjRj;_QkmIBWOEhOwG*PSz7cL>WJ7^)OXqjv%7Gxsz z0ZB`~LO?#b|AUt@kl(!zoyV!Bw%nTqCjfg<_yRkGU`xqI7GV%*GBm^iCc?-ozsW`v zVE`stYqcT<82n1bNCXlC_V9#7i2()$U=||c47MWT81sk=5z%Lm`-rh;ki=XA;jsv| z|2_6x9savj+At4@Wgvoz{SVtf3Hv|dv}^f4DqZ@oM%_35qu+-zYWB@CYW&-KM9I3s z|9&mcju?<Y5(skS=-;V0L|n2LPRU{PS8OAk zl7mQd$yTHRBY{SYBnoldNTdO>6>*&C;Vs;xe(}Gfg5H?4!2h9G8MVY@td&LlFKD#I zWM~}SD*(Jo>d*L1CUxX<}1yzIPj@LpkIRB zEj$E!*AWVHqNfnEHFO?*+Ej>sQdOR-G$BP~A3l*jdyulgY4}4wHwX{UGK~|s`b0h; z%0O4cN`K<%4fwVcLRU}C;9t5#WPoJXJcPK^)CUBf6sx{z)=^Os%4l<%E{6=jkw{*| zw#EsGQKVs|NZB4HsLex!+O9y+&fLaI;>JsHaEV(wTM6q+9o4e9Ah|<)Yb8<*?WFoq% zn8O%*fIVp|dz;Z$dU9A~`yF{Ya|kn>)Vw$FS309M6JPJ^4o3H=y+5cLG+jL;Kr743 z<^*MXfk}RGk1FH$=V;A&W<{$QpXYX!pvg8l4P01d(XaY76mV4}!4|I3n1r|k5a%$i zB2e7)@(T9p>ctmD4S@(_Xs|A579E~PZJl6oR&yzlitkq?B#!Ut@DXPq|3dKz1Dj?M za0sOq>?`T_R0)&b@{nxg2v!sMs+U5;ttb|Ko&0>m4y8ea!T+YAq9FJAKU7(R>5>j;O$zl}0Jjk|j`la5mwxNE}L%L1id5tOUJi))H>O zOU?UV6tet*8>W!s$(k2L{uF~SXHg_D)N6lr3x@3GKu@{|X zXzHDZh^cZm$-ZWr@<`%aNU$%tpFWZ|fg5{vLXfvpBm13rd zILqKXD&z@r-=)~Z)st{#?;-0ZXg74rb*vM&daZi+iRDgVbfV&1V013q8DFLB@^!A! z3ciKju|t{X*W>a3jj5~)=!8r$9uHRWe#zmm8?YH7^8+^LdmB|)SOhgP_WHrN)}!Ff z>pe^~!}yum4e|s5>-;nn_6S(D)}rg7)^OW&1Zgrl>C_V{QaK1xn54A~g{CNqb)|4+ zhYR&qGWI#6(bn>EZ<9PN28NQ10#1Op9ypIoLP8IB!)4)5^x79KFy~`DB@ICPhH(v^ct|$+EzQotQ5{iM5SsMsd(Kc*Z>e_nTmvW zz{khv#dFspqA= zD9SESHLY*rFQfLi6Jj%#!1mF51>y}Pc(UdphmBp&1!UGG4y*%vA49G2`1j3)nnQ|& zA+3WkKZ~;3LcPwrzi85m2%O5OaD{_Ezd`ktJiB1=t!Gs8+SmTK8`6h(-rk`T+0-*X zADOJebdx+9fudlu<=%q*VP^+Z^-G0u+gwB#^O|9K*uC;L!MkOhOu*B{&HtCZejB5G zd`wo%?jhYNF>Ot?RJjCF4^~^W^=H>7gQ$~`61z!1GIv|OH$~>jqY5kgop@0|A&2#1 zpaGMh8lwPBK(fCw4_HYChO!;wy<> z*@a(jga!T3KV8n99xgIkCi-rjyoG$bHSI(*`V%#=JM!}bMfm_XOq|P?OXh$>5zc#r zBiiOvKlAzudGRjlloRU6m{RC935O*qjMS#XfocvrVkV*_bxYc4{Cmn-2y5II ziUeQ02!A^#cg2UTgSU|QU1mGSvK(tveu0AqhAwG6@1x<-nIb=6w(Ka)zPGpMkNfsC zpwX(n%NKXr>dbEugOo}xcr)y!wCAlIa|NLg!FoMfdfee0`2AX(HN4hKPu2teCKBkK zHM(eWDtOF=aT~VgR-=RAaz=7}uyOE|xQ@w6h)&&OzMe62>VcaEvfyGo^l=&AUTy_! zv&KKFhrn7cPen>SZ26L$T;Tdh1d0Z*_b)XSjIEAYVwf#nd087w^m|t}55T39FGEDL zL0wh)w6vswD6NAMSy-*1dch?w<^d%|sZPWGEZNOxxI^SDX5!Z3<}&jHT6L$D^}X`# zUC{B{xox|w?MXaF^zoRpH4{7iGw%FaB-@Z*oWXpz+KFY4*Kfy>h&m~26iY49LQwpH zKHi-wYXqP)RKz6vi+`gvtD~+-xv4l|J;P_?>L_Dhze=N?nrTAjSZ>Xg3OYT8WKVI3 zpsfi0$t}9`bmp1qsuwH`yeT*>7)@5t7uI6M?sUNB2BFv{x3+6cm^L;f|ihc7_DAP2wb(5fR?ERy#ID)g-_e?;!Gf~y=(zh^8x%E6q!p9=4b2eB(+*Gr8GP9&RREC8NI0jvyyUH~ei$ej zf?YCa?rhEZcQ%`%dIooC@pR+DoXsn+T-;`sEt+M71$GkEG{8rW{!j57$v2c0p!7f+ zJZX)0W^3ZZ`!qQ{tKX|o2{(%4);%LhldgihGnaLw{Dgdh(xgSaZ!VIj&;N0LaNWs{ zUaS97DqLnPz*ps%o`|D5P;TAgO-^prYfE8X+;dNNnzwpZ`wW3drb)8E(iZB!LbOlNG+XUc!sFa_Sd@_%A2A+{Z zn6&}0`wC7gG@Ip=#i|tPaIxOiRU+t-ZGbk?MCGTgYVGD3)KOVg6jHR-0~vjy_J z`5Y89 z^O?314pI9wkt2|CDpz;T8xZ4*Rr8K@5U=->lk4BkkNt;f*(B^IK&(RqirrjqqF1{} z7WNa;^&t}FR-T{!lXC=#-Q4ab9$mr=te51iQ-tAGo*xqr9b-85lhW6*Bb1UIfMbZd zFv(MVc)FWX>GXiZ>bzZ_VNEmV+bnN>H1+s|e2b<>-^~pp9Re7fcF%405Ja-^2(HQd z!)&(ZN1m2Kf5;_l{?R>^UazOU{p;u9c(YK$Az$wT0r>`@2NzRrdU-XAVqF(STm7)v z+t^J=mnC$&t^SWp!4}t~GWmR3s{0<8k(_MsnPV*?^9wJ1BIJ77-nwJ42nAVG?#?h$ zs}($=(DXABp%pzozxT)fAl&x6y)fL)YgTpW+aHbL#toe9=1lqE5}eH?yg{}D>@Kro z_WvKF{`_4D)Wr$jv5kstRcza~ZPraHPQ|F$wv8LxxJhnor-F)Y)w}xXr{9_BwPsfT zw$J{az0V(TK4R+pI!=+z#9zOic9YG7YwN{ByA{cudd)l9Hb$YjB7$3JC0j_LLw(7a zZ3Uvvdc`Wgm?5g4p;pq-R(ZVRA|NYv;Qu^OcrOmCX)dyBE@EiH@0XE=q$73P@{tKt z5n!q{+B$&mY*&1fZzkJf9R{cz* z>UFp>iz>7~>J2NOc$YXwZf38S;W~x@S-y<0T?bkU8X+EXFADOs6=xnRA*09gjTc_q za2#cwmlvd(5VvABAHJ)rko59^dS}9uxMX7TB_3a!Yx54tjs+nfTv0+-4T13F+dk1u zS$zex#TiCvhVG!W(G_)!%0ecj4KEk-1sLDbz+9Vi#-q%f3SFzwm#3vKI(=4!n#QMCM={2-FerPacbUl5cqld~0F3K4y_a&Wo~EZTf8BzMwN zuZxny{cYsQPal0G!+`V0#HuvcrP54(ldUMoWqU#K^s0{&f~a58u0$%5p;3Xc?AOc- zlA`++R>f}mpP#n%KY4|GGNmS%NZt$nea*@nKRwR>8{f%c8+W!uE(wG^%IZqS`Z+bj zyc@gg&s(+zD2~Q6mY47Ik*|-Vo!{=DghYc>I1}ps_k}wH+2N4_K@W=Oq$p>kQaq$c z4{c@@1@A}BqV)-RUw_@c4i0{OCG7USz1>|L+Wxxvl2t#_eQ9`p7X_viPz4hc`=2_} zZ^jC?W4T0Z7ItrCxQ`PN{?T)9Mh`l()Jw{ls~G;xrtwh3_2d#vSs6iCb@vvH!h2v@ z{fe&L`AOQkNp~PV@{&h{ezo9k4J6)r=46MQ&mTHdH~oDB`Gzvp`%8@O%uyamUyEQF zdMDXD;l8bT4pT|{-6OjZ`moPu;Jo$dnsQuUrb8?QF(~h#)QY|YJ6dGOv49g}p^xjmnDt9M!x=IT?xK}&aqLZZcpO;HH<;&ykv@0dLnBX&4|^SC^s*ho-~X$g>p=> zjREonmkuJBGpCxlp>S3Y?gjbeayP2G@Ym--?O00ID^TD~@}BM`Jb`sQTI3>~hC*jc zo6l8WLla4h(|MDdWnVUvg8U~G@NVBarD->v**HWs-fnn#*y^2{bD{ihl<=9+>N z^%0F?3p1~}v4%W}XpB3DJ9F8RRp7PVwb}Rpw)GHQrWa}{s4QohsLmL=lPw9I#+$}`FZD>1gks+8bCFgA90zRqNp@f zFWQ!bi^;qq;Sk?8;XU-T4N^eHs*If`!67G7QpKgS4U=76?I>9YPDvtLrjnAN-$vF^ zhO<}3wjTg74E3?1-*u(A-Q*irqkt@qn>k4~sxHN)!*4+uh`IWhOW(pbmJ#Gs;;$?YT<3 zWEc3NVzj3H1%$X52J`GJ%M8_5U6tH&k{uNgNxo9-(JMueiFK0+-m?ytG-tm((J-M( zdw`^OG)Dw${*1w`Y6H;uIIWi@&gSpS;cLCTGiajpvoe!1U=}-2GkcR@Kj$Bp-vVQp zsv`krAB0w{XHCUccm}>iuukh*x8aa6XCW+QMKZUv%VSas3rJUtxQT!FwXD{8e!>m8 zPpfGIJ;zW6Qw5+7?6nKApdM-&*NgP|r_8k@e^9j8$Sdt3E792{IjDjz!I~p2aDV6*AfuHcB`xxzq>FF zpLe~m4*JkBZNSyi0XuI!$3C-9*#}2`Pj{SehMZh6*BfA_`K$@H&DIu^ zk+W}2^{UWB8cDW^={~@$nYHVCn4ySi^Hu?(1&dbRu47i-1t`}k`vDw1su9nin?7#Z z;dd+VguDwYZ!&Lxj-K~1EANK!3agwflq)j7d0I4M8r4LO?m=ab=;0+ff!E_8dwJV7$@qv}yFC!m&MFgM3dR8I z9Qwr$oKpA}2qqIcuWp``dUr(j1s@n~pd?Qa{XXpm?Rp#U*v)pLFkos0KOLvoIf+Hk z62l>u1n6jqa&5N;*Xe8;i<#p8H154=QHp2i5c|i~ zV5_IQAMi(zftejPWU`AeS@ZX8Hls5R9=ETzRlwb8qvWJPX0HHfSRH44&eQz1U`T$vmv1exWZUZx|_C zMGV=bMFg^*B^MTAgc9ku%r}c1raL#(`k4r!u14-PmIOz|o_Zm_u(+NHyTvAW&j%uM zr`Z+ePJ#CL;DAv*mh##T6Wtge%6qENla~T%T0B{P?d&ueK$C}iIzGKo?Xiz9hK+?N z;xvWhMt;iz4*0`Jpwdz8o6TO{8>x-Yz1n`eoB}bGB>d4|dP|~J4D8jT)t*mlTiMCtsF2 zDsQxF>^(#ASC?ijb7AM;5(PMmtyT9v2K2`~wW*SDg~>(`-HUXUM8pGx47%|kz z=Q9)|ibUIJs)LiSqH0Ys9u0O{(urJBlPYbTGc=JctF& z3&O|M#>}tS*Cofe7@KC`cKvBL=kV7UakK|YOmXS; zyx^Mk>?RkaAetE;wym0z2oJrRKnC21-t)5%e?t2E?nlu=m}Zbp?I|}^s#eIq8sJXZCt>+`7Pj#&8@-P>^nkkb z^FAt(Fgb2$^`IbDI&nIK8!|8bY$!AQe(daSu8~-#wYr1@tqj4y?mjZ|afx^v?tCtB z3~FDP?r)~#A7un=){guhX!WuQe{cHDZE=+s7oswM{?T)fkAE!0n8ed}q=H$*@MA}K zPE)!>gW!b9VF0+4DORsX(^1g(!4L2T&-Ml?JO{Syi{fWG60USyC0>K{f`jHOMhim_U?_uL$j}`FkHC`C$sud36!* zJhE5|+u_KQB)DiJ&|3+}KDp4Q^H_mCXdr$By>B)iZNMSo+q%eMU8uD*O5ZD@TpBtV z1u5)hQo5?BE3jY_&~v)#pDz~X)t~P*H=1w57Uv6BUgsB(5SpopzyHx+%UC$|4Sc;| zp3CELcx1m!68JFBl;7=__)I^VkmFUO_!sRAN1(U$VDm^J8;DPrv-qo93}iP+#+JS% zZM5trTnvgY)BzfqG~<+U!Tp~m$6iD_ z`#P3kVT<#Z-yblJ{SI&W|F!miy>RU_HoR=)SkQ5D)@#eb$_Sq5^sBS!lOyYhMzrY- zv{RrQO>STg6&NDrZuVtq)(OmbA1SjE`aAsNFASB5bJUQ2DK!a&T0AD5EJ6zGa(3Du zhS%%Q0+B#b!52W_zwzBN%;>8}g--05fsQ^BWJMYT31XuRdt5X^R4>Lr+vBbH zn#xZxmQz#Q{_-sKJ3={I^1`;@?RIKl`B^o;F%KuDZtoaXJNFHfXU#PYA_Ng1(<}Gv zT6A7zm`#xKE#=iM1PouAhRvj1%W=d@6;P8(UIJEM{X5(5YPTTVv+85uw-zqCy$2Om zBQ?Ko8a6x~T$^0?K=E;jKHQ7ry4PgNsn~k4?FR{CHU!eO??pS6Ly|dI3!x~zG+f<9 zM9cV42B$G+3_bve4rSb`{dX$1em-)?>WSuq=61Y)J>GsFAIqnXcLoj-)D6qc~x%l9rawiD_U7zYxLyrR8k7oWT{;p2xJ4mmGHI?iBS zs|hto@n-Y{ivzNN{QNN-q+YvDWiRu|nDe-wfv`>R zpIc^dJR9kebfXBe&%^0vD&)$$QBLmP%YLcbK1o^5pwlP0tUrn=6+`&hs6c;D3w$@Y z1QhUQ{^@ifbB2~&t*aerdh1Hn_DA=2C+ialaJd}c!%b=mLWJ7k+Y3#zpy{HT zir%*mFl5oQ@l3SMa@-gTLDSD)#HG?&#jmC@m(VT~rmvT0)K5W?!+pJa=)^m;1iBu3 zLv+CQ?zD`k6*wEao({mD6Af5X0Huc9<-(||6;>{fJ!V23h6g3`4%)7a54Z0^w%VRz zI}D2)`%!)Wx(K7ReyB77aGE}@f`3~ZmU8m#l!-*u@-K3gZ3Jp24y-7{$P#<0;gXOy z_Z>V9p`@%sY7nXoOIkgx(v?N`9u{|v7H0pjNsPhvPNQ~-+_SP*w<5+4=?>P|sS~)| zo7@lLLw*Ri$>Q=5601;T)X2J4T0j`8)vR~ip^##)WkgrBhZaYxESSkIj@62$`8^%etU;*lj)UIVz=Os$?L~kImWV3LGZt%cf}K^5 zlUkOZ`;;wJf=)Hd$?9PB@OOc${tOMQPC9hYk$MC3q05O_OK?^OeG^dv1{7|x@`{I_ z@Q3c|%erF}70G?;F-MJ%D6c&d7^R<5L{Z_Kg{A27mKz4=u5w(8tuCZ9G$EpkC*tq~ z{Z5U+)$2l&2XVxmJT4dWi)` z>yjnMib7Ryo55|6H+0B3LstPURvbPk9@YlsLA3+TQ z;{5EoitbGs&H-U?#8=ME{sj_$8iC)Tx|v$wPy7W3wi?(_7+}TwuZ^|V6J~BXsF;Ld z`VEhwMVP=GTovom`wd|-o{Xa1BWR8#U5FW2oMqq)bA4&3NI zz3q?w!9d&B2>G0~jBa8|={d(%#~4xls#<^xSYQUk_oK`T=|eW85tv#QR|K$ed%lHFa359gB(U3coV`kkZ{O1_B@c45}fKxZcjLuTKVhkp(U~Z`c%jB}wOl*<>mr~kVe{m5& z*YpO=7banw_}0C!%W`6nb&%+uP!xfweKP85?tH_dwbqAxPMHXds(*h+lbrDwT` zanW8U(D(Yy`<9fAER)W2W=RA|X!;Levsxhg-I(U`ZAMK=SPb}f4@I!-XJeHxpkm87 z@=eA}BPix&ajh39htvyrU_YPW)(bV(K?CHz;3p(W^3D|~bC=EL`S#5t5OIIDGO^^< zMVI<_T73t256X!vW)soYM{WOsHT)5UK>}mk940J%M%^K4#c7n1+pt1Ef{nz`HsTY< zDIe(t>C(syLS=EMlAp6eb#Y{rGe}cw)8x!{u~E=;P9=$;_&5B5rXk6JtI)UUG@(WB z!astbXcW!r$yf>TrL6HX2(_7AuKSmAE;8hyFsat0BIH-=CR`{X&}tT0@s|Im{hz7V(2M`Mu7|T+0+fHdiS1f00Sd5WOcnaikZO2x#eZ)DW;u7{ z%2LtO-(#w>Ef%S>4^4rU>5Ay__#p_U+lZoeNX5>6R+%DUL(Idr33;RMgBurC^fjv^ zQTaVS>Xws@$Jdjc-p{LeX$bDVR1qZ3&^cQfLl(~hpwc_thJiS_6)lA0RMvWmAg94h za-i9Yu=MXg7HRAft>qx!Osx-yDwvKz<%^2XxCNS$&X^%E!?-F&cB%#{nSqIQHs0g- zZwri>1R>xKUlfZylr;J+YadnfRLZ5VaUm_K#P8LfIilE~&D!FMW%9{CjuFpB2rTr) zU%ra4wZn{?fQsK3!F=>h+E$D@rUM5{qOTn$RY? z$Vj~&wzZt${NiBkp=}yAtWxVot=MY+~G=$y_*^=-J5I-{p5b(2n5ZaQ80*h)#jP}V2%=N*tlAQp}}X0r@UK( z@glbtIUk=_RVrRIP3bzlHldN+NCVAHXxbLrDkAEviMfuP)VyJzbP|gb&jl&lD4n1=ie|cB8E|X6*GkpI@ z=If{s>>q7*Fc+3b!siYqfK8KuS$qfKYGOQl0LK|OWdge&4DXUF;cu?52uUdcCY;tw z?urcs#u+&!`9X!zG>W6$VMNg>`dR2HCne8kQg4;=L-?nxCh(R(zA}=` zc!!`5(1B@a%aBu>-1&KS%b#AKM4wLJ7}fC;5@x%0Yi4&>#SK#l46s~U*=+Y&oSH+! znkna@up!fX1Tb9HRc)|!|8&4DYvuAiFy;*Io73(!1D-p~rK46dRRVLc8B>VAc%(Y)cZ+Md=m>{sB2;3{Ca9nY2zI23QNX6H_hFEp z38P(YIaE_Dv`UP#x7rrH3Jml|6ZxY?i$`0t@u)jl_Ry|7|E6aZgh;?=(GKS>B8I)4 zJ$2;DwTM(RxC{9=zS&etg4+Si#~~asM7as{*~dEjTzy#nS@2*tw13vIeC9fe7&T zJnOyCA#7K1GmiHuYpO)RW>^YUi#!%& zn98&}_fi*eqt+Xg@^KGguf=0Y2IPk6bTVd~a*umFlV)-$ z8wOw@);(k+f}5^LJ~Y!m@IF?#EA+}QJr{8VuY(bj;ksC~P=}5;d4p3Yt!*p~8?>-! z{vJvuW_{BlGj}{9M3tnJKoon#5$Be%=a_uxOC`|%!iPDir09shlb|PhdjR+ z4)eW1eLe^?DrHc(me{$kUF9!}1ai}vIulEuQCUl@Bv7`yU-CIjuR$WN$QAj3eAkm| zJ@QoRv3_r>yW}ELRrEMA>EezkLqxOiP^W#4D9BIe%BbqrAZ3A*YZQ_fQT0<}O$flT z(Q&AogGi#iimQp5uM+9#L8vY{J6+BK8?T3jXqPlg!&11jXSqij&1A7#0rB;9q#|pI zhX&q@lafGXc?=DbxJ{wWDfwv}gvl>yhE^r3Hp8DbnZ!>6Jx-kmh7^5T7;Q#(FvOWM z1gVNcqtOiHanZ-pV~w-2=t}h@$~AL!)KlduX0)z0I@EPIREXMhNJZZn3mNOdZI1(^ z`2x~1*s}ANsm$S;^74b(h#oDB6cXU>60^CY(#j>@w!&jQK)GT*K2N4K$^jq`x*k3F zy?*W!ftZwo(VH`pOt^p@iDqRWgi#bV-nbbn_wq32wcv{@Fm@Sk4VB67L;sgWXR7tk zLW5OT=UzYCNfn{YYkr0RiZ3U(4?eyQY1jK7pPC{X#EyHKE&WqY~t2D%qCsGKBos@=88U^GddZ&(XymMmv7aTyTTe6{BD}&QI6*LTpmlt&4t! z1q`*8un5Y^IXC&NMi&vu8<9DnE4df;w;zOh@576Trn;U%U#utdxZ?{A2pD+?3l3DR}8DOU)Im@50S%&UQ4Yt+xZw*#Ou6Jh#5mh zLE$|%@WQZ~l$p=4MmzX#KY;HtEdKQzorU`7y4NZD``;>5=1*;3bRZm)nPQjL-T)0Q zh4O<63!^5KQ&fhUi(g|5QW4)z(pT1G&w1=tHSE@cABXi@p2ZXaqk53;+?6{Bq@Llj z(P^K?nv6*s9Y?7QE*_GKCD|ND{_C&!iAzMKl>s?Ha@v}5zM~^{{?+W~%3L zgnlPzUt-i_7!_=Sjz8o|1UTpdP8PnjL;(+wyV4nu{->kykOC|D(ZhI!u7Z4)x=V#J z=u%|Bmj1)UBaD9=dorSi*Je``;zLem9B7!f+=Fl7X?JsTG9C!jEvp*A6g6#Frg*IE ztCNkaQ5IpM#X<3fB+XaNjFp71q&P)j;(;?aMlPlXdsam_!#T~nit+A@&YEgLa;w1y zRLkNNx*h)-2FH)op7%w|$C;9m&*!g$=l9CdfM1RLFTk(Y@<;cLH=pe{e%H_1jecEt z(|D~nPl9@Lso>3j3vd^$LYW3_-`Dq7Mun&))2{Xao>oBysg2;wycP%*;}P8H_j-N@ zbiLGq!{mwNhK|e>OQ)aGM*d>`TgR+p!gkD;Hf=)>v)eN*&rxq}m8+H!+=15X{jvd2 z0Qw8NBS&u|Q$$C}eG6_?D^O`R9e`f1-cWl!$cr-GxXWdHlq>uw(GmvQefM#uOx1;E z08~9lyVk5eg(OfKK*1%9>l4uqBlW9xo3)1H*6KNUGw;kQAYtI}ZDXV?xMwHbFb6l5 zxy&i^fZLW=c4bt!d*4^Mx&5;;KV5)ul4hb)?D2)c2e7)_EgWy~(HY=vd+H_8C$kJm zVtvE!xb0yxZSKv1Dtb_}<>_ksf$^BPyeV{|Pk32x+s@3&v7rlc?dIYUi3BgiTKwr^ zF`peyUhWK|nW9!cr9A&BRd7ZdOG-V|1sRottZtB=h-to<#WDo|vw0T1Rk^FUvc9lF zcdW(eRv{nv7ttVAIS$JpUW>(0 zZEBKKI*nq>YxbE>o9eZmXZazZcFK{?UgEcRluN+Z8&u`?oaZCyKLf^q+#l5;P1v>^ z+J2XR&x2}yHQ49kyv2peu5c*k31VXFvk)u_XgceQG}_) zByXV#$rgLxALC}AN)s&{c#vN&2{izKJ6Bjtt{Nns6gv}>Vz!`%O7ye3@Px7y)~2%* zV-~Pl8@&3JC?>G`dluok<^&*H->bbyf1of6Fqp1iE4TgU*WZf_A)2vxY5S@$k-Q9b z4m(92!r=j_3U9*6Ra}Pthi1SvF|XnsHIv>8LYTeQQ}5l~_oi4U@Ez?70OqNSut{VN z1W(d_)RQ3hTDrLe^ul|(iLhhd^x-2S;G&Yiws`!ha}ic+C^~h8`Njh*j2DG+jyx+T6N8Z7EKo>BPE|G1tlpWzGmr zB$WO(tyF>7L^5h+rZ`*8+-)WgdIPo!bo((oWvH=%pY+{uhJH{RKf-VEWG*wt8fX!W z9JhvO7NJNf?hU*?Il};q+Wy#NDJ}mH|0JaG?$W_kUB{I82(4E2gzINdX>!bxXD?~) zlS|^0LD~*zcNXA*FuYBlg0dVoV>u~w37DFpv@u1b+VfN^;24Q}&&RM`#BlG&tt zdPBuQZ`dLGG}oJ9JGcYi^QwY=WN&;KrCCfdN_ypFV;kID-`bE!sQ#7IxP6!s)FDNX zdiCmv-`7hYx6it_f`L8^IWHHqqNl!Sxo=@2b#1|{BwKy8)WiVdg=eTC=^{~~DI$u|BSY`~zVwKbR20M5}T6P-F0 zHl&4#?c#2;LqCF&O8(A2ET3Q0m)S-;4VxKR1GQD9O++G!6Dz+vw-!>xa(0Z${X5wm zlj4>6@@T@39>turuLcNFh=wO8S*pz>Pc-{l)Gtn4<;zY=*6uo5DzjZ;dcC|Ypy2m}I`LKq7P($z4MP;XOc80>qmR~us4H-mw!@12amqWi! zx3NVhw;67At&zp|^uuU1oiTcvhM0>8<(~cC9n3jdEo#^}BlO*>;goj-#jZ12v-&B1 zN;^$bBeeHQxmz`cY5Sr;ST$n5S$B3b?PcR}+DLhn|Me^W)ooCz<5G(c!49zQ6>64~ zA55dk(d{`{`FN4*F(R#QKV=rN)yRjkjf>BpJp*#r&szSvO|XlxvQJz4c7vw-bpmKk zghUzQ#>uIIf$jvS8f?le$=Lb6r{3`Aq{s$3eV!%-wn;O9Q)`3KbTWQ`6|$3Lo$1{H z7JaB$?N@tn!EM#XEZLTVToPm~i_?q)ymbq%gP|7fM)qvkh*j@^Im)X3^0Ub~-{Dsq zhGmbHt{PeVCAMbaFPo*E_y`u_&%tlrdGl`!E>lvglq8Tba*R5qi%#FVj`z;M$8WoP>>#Gt5dsQ80NZS8_2I%TF5ltU|Cm!9C z8+CroP%3P05Y(#vY_+oq4>ve;S*C3A6j0L>TiZ6jGX8#S`W$i-LTv-saJL)t3*4P!ew+&{nBc@p zD8v!11{mcz84Ngvhp+d z7fl5IXGi;Uf~)G+Yk{AiXn)M$?QuZiot^3x#9%s+MEM?FLNFq$9{(AiycG>skAouF z?5f7;V)I);xRUB@@m^cL?x2XCC8Dn+#g!XPzw1_?-H1suF;R058^T_l{4$9Vci9x| zk{M63e7ov>$vGWqS+}qZR#{oj1xXu3bm;mc)yk^Uca@o+C5RAwB7#5B9?=`O*tP5l z0;0*{csltu@t(`ok92C?>u^TRAj3+Q_RljfO3u>7mF%UTZZqynD=q3f0|=?~(6>eP zn`}DOVQqd&mAs2F@4fjD5pU)ne*Z|Um#ifldYc8tm0aW}zku;&bI!1R0%So~`r{4H zEn@*{7*3G8dP$XN)R5BZMnvI5oAVU}R3P=qhR$SlJN)%(jPZ_$@NKVRx?QH0A4H!{ zOgok*QN3PCo3k#+Cr%o>oh&;yR=hqPt}Vd%UvD3uwIJ%vfA;aPGXKYtyECU`h4*Ut zia?frw?we(`Dp7lB7$#a>jJv)4PWOF)NqnBD7z~hYF(3-xP&5^3bkmY9ZELSbaw|+Ey!QFeu1+0cX`n&e4ZG8$hj3PhEQFn&5ZY}`a z<5m7;1OF5W*d^p@QHe>%!|SlE%O%_&@Nzx^Hie>Dn&v{N`B8>QfYJWCIyD)`{}U5Q z=7amRkMQ9=fNtF4^ArqO9&295e0s&8afFVuthr_8&i>p^d&H$N1gO?x-qMucq#`QxMSqD8pg7}btMMn#A*_zV5$I9Y_Hotz`==bSXdw5VnN1R1@- zCrZHILH|G@;Mk{`o*|8?uj`2)?BOEb)=}av8?qR=i>Q5_%(Wb=*SC>|3t@2WRILwF zLDbU}PyboheEFiENNa3@^wE^6KL+Rc3nTfKNhZIUG9_nAX0Yt=|4zJD5hJO5TdvGC zY}Ks^81`EEeW-%Ah$Thu{$FJ`!#4yv!gsFrNfZze+8e`VoBWKl=x})A5k5$Zjvg(0 zGE`RWc`;$*!*bJ4;y<9y4yIAR^baDI> z2%9H)-%7bTkp0{klv`ow;fEisr89IH1h(X-R=_e|=wikA_;$9Kii|h$)-2j?<=xF`3Y&!g2A0F;rCTrfQc6H#6q*2E9=d<2 zxJ!8@F;1F)TN%qiRQH{-`DjO*v4O zvjQ``wr*pi+ZnY3HTL@=H)R`x+C~8tbbtjDy;7=5ZJ8`&qr~CdEU9UZI3>Z3&l` zPQvbDfJ*$o^5~oS)`#ji3k`Q2m?T$RB5U>@m71z>hU?$M(1?qdz%pa<%z`FEs|X8O zlf;8%7Pyhds?#W6ZyEg`r>yR|<)Kor{KCHiKA!%(+a_AzxK}^NtA&v7)1S??QVcm> zzA2{r8RByABu~qsZu`@OuAs-xHc45-A>64gwUaI{Hd6fNx`$|$Sha`=%_i02gh8E+?E^?s87jkPdr8|21$ZRulxTG zw{`q&+KrfC{Am|^du{wsq)pSf%uCH^uIuhs^6hCF=b!%Tw#|tCzumU2bhFc z^oqyZysbUzLKn&=7Z~fpc3}^K1;4P?YQdAO#k<5~e;wl~tvKC84Teq9b9r|}&f0VI z@LK*?Y>WKYWbYFGM{$pD6pS(`q z=puh=SLeJKzPEj$)J6z3MYwccjMz|h_-KxMxL&`16Kxfz3{cZ}s!}L0KW(KKE zN<+^v@+EvhtJN|5o%u#oz-Avl747Rq!fo5A9QFi4Jzj;jOp^8abGK23Ql0g2ujTer zmywv?wr5Tta;-g&K1xfAt1?zBCmez*0nSFTkYyEETz}E=L|>Bz(Q*#)$fuH|3-+zT zM|ijk=^A@izSZFL>ojJsK4K(GTYqG6u3>gIztxT&H$DvTgQeNg=?ER2z~(}hcXcf- z8FsOS#%VPmyGXt{*q(gF>6+PFqdxflzf}8J2X9TomEn?UjlZ_=6sBcQ`W$<2V@MxD zjJY_`&Wa+D7BvRHl_J4OQepgH1FL~)meibzm1D!>BADb84q>yITd{to&eCl6k%-G; zGWz^yYbG6sC`VNMKYc5}k5tVveS2y=MPA$A!9$}C`4z(3sqN#cQ0^1iZFF^9(ww3h zXbygw;i@@aXMUMSV9D@ZN`@b?C?u%`B~cz>&3}jad@Zg4u`WUzo!B&rqk^6rXleLO ztYcN!;AUAq_c1}hUeEB|^mZ;^(!x}v+s@YZ56&5T_i+UmFRztYF1J{>1)2D+tg)4-LjUXxh5_6m1>;ULWp@UD~r?+NBu(^3?dp-KJ7 zFy}h_^#?lb3b)m$ZKM+V6}$b7ncXcVR$U!;--qy5Ym)ZqF@FO% zPPsuOpF}2o>6y}OF+_vl>1}LTV`S-W*7(0ACIG(8RMpTJr7)@ z6`(HT990Y?E?B$KY!glid777g#&hG126NwB{`p7L&}2L=2x4syvYp0xXbtvC!X7Gf zka-qVyIbg;&;$Tu=H&wnf(_IEMpJ5IkqY@i(${DvLTF`(2?jN-?J0Vvbx;!MWSou!2?ovcK<$>c=f(p`GCRqhMt#4Tmww5ykE3tdR`acBTXUo` zG!CDzpXjt=pT9dd_}duZ9X;k6C`XzSm}q#1u4eIdmSOORVW>P7L~S96>#ZCXH;!)ju-VH#!De2@}t7cZ^fE#E}%t zai88Qg1T|qg(iYbv|>ztwjTO+^}1dJ35VOlnjap(kXQelF*^gbZC|28wn$nvOcs&# z@X55~svjTg0gZdtux67OJgV%fLk4(=`~l>@Y)4LwE&qVD1R0NLqpnhAUm!TpOdWA6 zsw=oBX38Oh;ymW#xyZuK62$(WKKnoAR=@K9G}zMF=&}Dvo)Z`r{Oy#Ij-PY(|I^=O z{}1CHo>fXZ7Afpshlpla@b^;s5A2@Fqy!5ewA1Y%n4`t&=d ze#ejYzIY)vw|mv$IOZX^Cev)?J+hoygZXumqmt@EowH_@!|~CpQ`Jae%w2@oONH&v zas+*@w88==)Div6z@9(n(*Z;U8$wUNS?)G}$ z$N$2^zrTzAdOtg#YECJ59egK~PBY!_-?{!bq4Fyp2H(g{ zhMNK6ToQyM&SGU$&>AwNlOY*d*rgWyQu3Gftz$L;O+&B;9FzmmT7V*@V|9J@qBSQK zVA)X!UhxHFBx>%4VgLQeVjH63b0iz&yGPPDB z?h#ZBl0xir3z=&m_ zW;c$pcPuFK_7o!)xA;KMVjYa_DU%byCc`gzRu1#s0%_*g`sZ48Db$DFuL|#HA?Jih zpFO@01(C(C18)c>e&uoLSh5NRMZtG$@AvOT?6br#$|)6cGdo7PUj^)@wBHzp+m0g9 z7->h$jw=I%+x2l4XLdx5x{sM}(MInTJlsJ<_|zE;IVt}?ChaM<5*lqwt_M?vg zw9{1mpFivf9JX0bI|!=L^TsiqVkp z@Cmbr(9SRJk!~+et8nAn8Tbuuzwq7#Jteq805T(E#d;4FCK}&t{svOru^Jx{Q~X<@ zo}fE24ovAb94MUyx>70ShL93D3K85oNw{8SZX$i6Wo2;7M;&+IjDues;P0za0g`2`&_%4kwSaEt4%C$1 zbB9$(#gZ6dA#KB_KHRKWK&~M`j%XA%v`m#28-;M~C?-pJou-H(ndCj}I3g6q2=+?) zakCpOjN1ta1gBW%n3%)|U|p4iK>|;L%-fX2fhWR1sboT96>4)%M(-Ux0OG#oMj>K^ zqP6_ZPC5rJM|~%iQlU@@hdS6+XCu{MRi_Pj?CWV2&A3#FR^nI}H(iM`$M4%8@7DXm zgFb1fbMzrvYpYP=6B;)JOHyk3kemME|5b09wT7f0!@yYaqQnKFUwMu2{5N0jJNCR%53r-{oEXL3r zT##=zMptyBp^F4jw<`=r`JYEp;0t;F8WJEM=`J#A5t^h0n@CSx-vqs6gy)rr_OQ9U zPxUT| zHA)_-pc6@}aGu~X*eP?f$cs>Ul+;nLwi>!aUEx>T&6a@J?&Gd?L_S3dGe=F>7}u0H zg?V~aA4z<~6=cEi4TQxqm3O?XrO0(N14-J{6!L~1_^#e?=Csljry2Z7W|GyiiWL5= zJp~vIVnx^zv6$g34XlOLsizHHE(fyhxgsLCKEM@-t%qHMRHHnl}>?0zrc&pEY{H0ceDtr1=s%$C_kR7%mMtBVQ zDa$#ClVY8uvsymuemz~grJ$t<&Nid_ONe+9jt2={0o%yevaSmWC{bzEI9!c3<}@z4 zzGgf0+7Q{bNh&K~rF;H6iKT9B9UBRcXSZY}lGueE;x=$S!Cc?QFB*EK`6-2(5;OC3BwHxGAitYb?5MF+)1(bCu$xLX68P&R};OVx5;H{J=kee{rt8 zhGIY)2`dJj2#~I%tl2p~zgi(Ro+f)l7LFZ_(Nx@SycY>#FsQ=}hn*9d<&Xub7Apkt ziE|Ee0AH zaqfhouSqy4R0|F|r^4j^Bd7ZbbKog|3UK^Crd~ItP2M0>EjXyY1*Sk}!&Cm8ar}?9 z-opJavfeIIEihxnlyU!Gf-N)_``MS!?+nb}B|a$?Ee2((WRznomVWiI0~0nKsN0exxc-D{KNX#F4IHzLgBLyZ((_wc~Su%h9}5CEI1 z$<&M*xa4X!nN{J^!$0F?0`ND6{MNRq^;t7JsW-BnKOhUFX_jsZ4%JINmHo4+p?5#V z$j@VfL5WmK4=CSze5{@Pt2HZ7)_bJMf2I(+`?a`qF`LvfDR3=v^A)m5Q!zuXiFuVJ zK$vu$W(h$Q29^?CDL9#MJ7v=2DNC`7qy=lJLfn5^uN@x-6DY>Y9XVet-`T1~6XnPB zeOJS3H6%43CAcq!I=C9Jhx+<3?evF5;sdCI^zkHgYX(Ti+ zCQYq~RSfz;TLjSgvSDyBS6soBwtE#%YQLAKoCKS@+*O^caS51{>F=|dvDe{%-kaqN zcjI6V0eD2DDnHfOH|b@jQ_X;+JMg0E$n}Yc68FI>RUxms%>&hgV(b^-ZrXoC&Lks4 z#D#h&=wM?Sz@PGnzdviu>RC42Pzq7;nittCxmZk1%%@?cVs^zYN&d7BD&1}Mf+_a2 z>9|US7bWWxH&5H80++hV#=lY6epb8+P}8#I+7`wyrpS}ocm|hTdoj~z%W97mj?Y?u z3>a}ttcxEyM}`hV%j!Vy;37se09rQ#$?^lzND!%W_q&B+XSH}u&%4`X<((bbfoto=RG zVRa!n*WZTsg}#w$;Swlc*G(gs)x)LY14{A2G8X9BbTnqiDkuqjh)uR}YuyOoe@7*! zrG2wI?=!0}Mw(BEO?sAdGz%DsdZ%`XBm(Vodid_sY%;5s4m`% z9!kcjQq#j#`4Y&!X^g((ubUh(^Y}gtVQ6Yx#`Pq+gLVA^`{(5GCRh4I z+18}x)bd*@Dyk}us^66H@p&q~qz?NwQj3 z316BvZjfD~dZeKcq;Et*fpzQ~U8EAH>@5%R&{=gA@aV=Jji8@y-!eQCxb( zw>vIZPh~OnKp~T4Yxe^1fx{f^2xYHb@NM_2-si5|NSo&)3g6aj&bUSW;Z#f{K)bSw z3fw5aN@*Y+#Ue`xGE>1q$kcnxNlxrcX~fnm4ymEWu8?*Ux`ab}tWS}vG00!1uOn!0 z_~N(`YhG6VUE6*i_GHXAn_ljPhcr1>w26_Ik+HlX7Mjg%d#M%s7oJAHMJCZHhk}8@zgff(9l7*$L#N z=x5&qi)mfbo%ZQrR2Lf_2{`Pq>z9$KSVlRc?!R5e512(tEnZ2s2;*=!j~g}Xjkl8T zdNViylgCt65p^S#8w}$iW}ia>w}Fa@c4(VbYcx;MXa214Mhw)G@0DFY9>Zca$EJsH zuk&86{}k#jZVhIhA$6HC8L>GvF`hPyyp^EUZfqLw>*N+M^a9pLRliEYYxfi*q=q_T zKhPu}>T0%|$p9qr26bAF6fF4XBI!Ac&4P|Te}`i^-K&p|sj18dm1y}5LBa&WbeuIn}xqM_6oQDzj*0X|hg|c9*HZ3Z@7Ff%r_uhxl{3dr8*E?VC2`W1M~6&>VrHY|ZP zHCvi7q%bUiC5~XOgCZ)kvBa%-^h#GotUIt;sA-RyYQ{Q2DDIWoG?tQ0$r(r^@s#)6 zNV27}(>1sShe79wsaYMiSTEb0!pkuvZ?%yr_(}?-)qR>HKcfhPPqIuXtQl2%E{t~F zn{&Qy>SgMkTPpsH$LfTGu9cCgEmnW1k`h0C7|wwo(EiyX;M#)XcEQJ7JX-p3ZcG? z7bTOOanGA!W~PJF+Reb92oro64L2XT&dvp?%Y8`WrbxK^^EKM5-b7`xh=wrhY-(F1 zcalvv4ivwFakDq@&MLxIwO0fh9^u0r_QU_9p4pfJ@rBq|Q*P)|P~oU3c?vH=k73Z)|7cjVIdc zumrR?Lsv{fJBU*_2OkM|ztSnXiU z?gaUR(IKh{%INF=wss$q;^n^$n$~U)ipMkza97(+R_0LhZKf05ipU6GoIJC8mdPh# z3i0>rLkW|kKM0l?tX3o!m{I2a`bPAGrg;077HL<$h_gHR?DJT6(2A1BZws7IZ?7bxWJEbpQ-NiS%XalJ7lITV>(^(4sG&Y%UH8n=AXmQ8ea|6^vWp(Ej< zGy{%bALaA3!Hff&@ArngD@%ss0RAhKKIf}}V?j_>1h@t@%}d6zoId$4zjCYmNZL7z zqE*`2CQjNJpCIW#k_u>d@j(T&OWYt*a;9_hM~KfQ>t%0!I_=m({o^Acqag_!OorvQ zx*hlhaVb;CWy@06U=Ttt!&vYvA(;}H(d3&;aReorvPU=9mNadqSkd%+@7zCU|Pw~a7K3f>e=8fKItF;SX=kgr#oB1e2a99=pF#Z{A!Ov8(i zowUWQQd%MNpt%1s{eS`|DVv>}is!GK^E~KP-g%zlmq2m0*<)zf9Ox@y{iKHIK`i}U=e(N8T{fInTaA3{pv z!lDG~N(qf!Hy^#a1=kK_!$6})QOD}hwWA%|mpk1OmjjUlZs}QQJ_LDm01$=>p z2&|b#ly9g5lk9yDreOwDFz_ehk9HXhD{H@s7(Y(l`k(mz@rNnH&OJ>^S~^*zhC2Uh ztyts@=IA8V)LE%{IGBNWKaM0;w?Q4l>0D!$4bX`uj)m@1_={(OK=)@v3B~nWD@`5Z z0~%e?|MC91%nYtB&TlHyYl@0WMD|uQ7tTO>R(&%5MugQ-(D8SjMlkh zYwThvuuf$k+qqOW_kqY&3ew#IqXaqIBleGmCh*^LpKvFg;n~hiPFb*!&}8<9)v$ff zme7E!S8ki>9m|0>j20jvCZ*g{w=X!%Q4Ntc9pSPwSpR!l2skG8S|dH~T|7;1Dh8%CM5 ztaYYXs$zc9TG;sFlB5%_3-`z*Zi*nM`{e`dG(X?N43qYKpxMPx#uaXbQ}_XBC@mdY0Izt7p~7`cut~3=h(` zlq>0V*YjuC^aV8^mUKmpswv_eRBPaDy{G%##64TYCCsKxavw2@7xdqV)r(w zc1lhmkr|(>@~LXNp6O^Ms@kV-F&*>QeAM1i{=ODOQwQ+%ZJHvYWP5AZt#5}d?932g zvLRgFP$r$VVPMz?nJJcfSG2C*uf0I*`(OS!Lb*xU4dRKJtn!|v$bU_2E5)Vr2pG^M zp7hnfkIK(57{|66xH;G&T68nV#86~XWv8ZbkxFE_e%hx&4EO#>bPaa(*7DEpa5V6@ z`CW4+3FcuF{(vs_myIn^dCFZlt;qgJt~m6da0M1G@z|qgIh_@6pJ;Rjf_u6+8t{-$bW- zr0siT0u-2vy*_oNmh&k+#Es$DeL`H4KP%TY^>$W8!3c)F`O)}npC8IGLn{$S++gls zPYTz+Vp*IxizFc5LWcmZoS&g7aDiy59%+;E{8i@*9GE9vop5BpMgWl+al|W>egm@k zBj!oe0P`Pe_drk-LgK&*z(hldg}I{zRExu)ed^%yh^fvm@)t9dT{HC4PZr)}WWZ;| z)&zt2eOy3)w7}2f>rOCtH;C4aeB#AEA8Dl`)Kph|L_H3bC`rl2)IB{X3m)%_*WwL@ zf%aD_XMNKbOMO$ngD**{22D}=Tm2KcG9g#W-?N?c@@a9mH;sp^GG1e8#Nr`Y&*42= zcxrVtI|C3WtmR?r$tskG@LY1sUD^P5GJ>Z zY~yk#X@)Nw(GI&hJz0*FWjNXUa{j>r%7p5~wf~1IZcp62Wh`7MR-R?|t~pHW4r=gP z?=T5I-GTs2=RexdbV5~-z1 z7QT>K?{DKYUq!_laJ2<_l?KbcntTm^M{0{CJE2E5=WI#;PA(Bh%f?c;advxk8aB8x zJs)x+j*G+`b|k8`WQp7tba805xVGNYh-`;IxYN~L;h^vzt_d50OXE9t0tYatguOq- zp;${58?~`jD=KhPc%YyZ1&qKZm|f(K!Y5Hk=G!U}nU9y7^R%-W)KHL{$)eF1Cc@$- z4NYVmOye;Io)v~c#wJ}SSieb6l}XAFiE2`MENKG8bNwt1tbT9D=&Xomjrf6^*G7?` z;-h_fl?eQr+ht1%oMNMbSpR%-V68*)JBP@4)R&vVW%ws_YL`76F+}{X!Pl2~B&QpU z$^cJwhQ@74@_zMk=*xvLJ~R6JQ;Rji!A%x^H+evWpO~#Aey6I3u6xBYe8A7+3oTQG zbp>T%ja>HBhA!LX7Omq^h*B}A@rjW!8}2k3FA#>&$H!X3VOoF6S%SuaD|hO$FqYEt z0C5Y?{Udxq2}gcLdtivf^!jqp-az#dIi6jjWC*GYg33klyDu>^Wdh0ZwjB52B)_tw z^&r!LEpe=CEFDn(gTcHH`nV|9a-i#BxX_`9S0x#YbHECg2{$O2c7o@N;%9g!0P%3*=4-zrf!u=17@we|iOu~f}Q znUL|dU~&HqT2HVLYsjqbxSb86$5x68*4Mk;^VYaC_qp9X))NVL(rs9+ied5;T_AUFdHAZV6VrrO#vhoxP{+XLw5=nX*##pdL%H z{{ER$P2`Q_O5EbjkW#}YM;yBcrD-4$hS~N9$$KtavFSDw9PV~{FLsHta;xtzUuujv z*V#|Xl>%Pt>R&M9w3~g~L=81)J`74)(y||p*BTxb)Xj44N|omeRsYz#(*KpKTawS^ggj+Qo6WhDFx4Z zd-n4|!z59@v?K1zR@paz9YE+w>={#jkbfGnf%XK#8o=!nN_M2EjwN9 zCt^Wkw!;CEHaHPNjvaXA9pkSn{;}FDDoG-#{T>HjmSSQ)sFAQxeQJL9R&di`Vg23u zO>k3Y7B`_twiFDT5j7Rt$mbmJGe03PH`Oi&Z@JqZv80?m-(Pp^!|wwxF>|H=ia}-u zxswyV-rN~!#&VRyV`?gsf@~=m@2zw+&{z$q8aLn8xuajlktZ@*6CS`7I2z1k(`qeC9E8Y_%8fRh7jzFf0Lub&mWFh*wl?Zyx zeq(&@D)t7eyKaPy)vsfHxqg|#^X{TFGS@xnTj4I^96#=c$TeQsy3cTD-3 z183as5zR-s^>B4oeS4{K)-%km!WO}@*5lb(q1?>g3d7Z0Ef+-Syh&2XkDmZSjN449 zL1;Aczal}n;=2;XB;prccAfqs<54VrMVjjl$c!rvsP#B-Z6iZQaLrD#tWtbuV>dCo z=)pGmA8)LAjXVXQKt0Boh9os}pg3A!c)x@b%UjrDZJW7MKS!DNaaeqi+bMW={x4jd~Q}zb_BahSV-yC$?DSRgd4sW0fQ7vWOV1^phmdi*jl3U5ajN!*- zydKi9z6usvnpD)|bo?XlxA|s9&L&pX0S|g6zzkSkBO|0(L`KPDSj{25tX6fFJZElh z&|RCyMUCPcf0x(B)qD7s7p9fF(a<$^;s`5ap{o#LOjto!*hlaKb}}L|lJub26WT;% zB(MmJ<-vJGS9ZM=xQq8V*RB^Y5u2{t;;C=OeM-;TFWFRP77 zQBkeixgYnJr}rL=)JE{aj{@A!E5(-~JnRGhHjRMZz=byzp{cy`U0bX9B0Yug}~r zf7Rb$5rlT;2^&0{u-RG1^hF4%1!vgUgI+vp7ro+I{e|D$;IGd5g$o&_Q7dv>K}Y|c*xJ|;>RkADxha+iXz{CXU-KSu;J|WI2x|`Q;uv$ev zk2^EyE|}7s2-+J1yPKNJ`xl#Cv(SW|=qcw0mYcvnO<%pcTuLsJV_ zg98_iE12AO#Y#p|{F#hVe9@Z-pA;KWGf`CrE&+txG~^|8G=L3o(a<+Df5%qOY*L+ zg7s~liuhb)bxwGZmmOUQ@1VHskw`R4g}Rc*ux!o(QUcjJchXOuobV~*D z^y=XuVRm3#^b@e^#b`8i>SPlrZVt!TUs4vm1T$aTtwQU>L^EGsPscMp>V`lZSrIw) zpB7C)@J?trGA8jd6WGos}MX~X~^>M zQ8DtN9@@$VgTH=iesY*OCkrARuS?e zo)vgbAoE~!zmLc9DHkS5vfmzO=?_fIEUw@b9Dwrt3w>C56WXl*+s9{>G|~3A6|#_z z`^{m$_jBCLPvoa(|Bt7ol{AZALifSXLXot+1ODCH-w8P%1aJpCI}@bS3oKdA1r`n2 zyYRVa`pAnfrbTa7o+Mz{S|0a zHv8ke(PQts;rVLYX??lyrZ=c$tC^yN&6Mwy=koS)g>flPjVratlm_V0TUyp1pN^|W z5I`ynOgew>^+)iAAmJ7Z+KD#fD>Lznm?Y)81%a+%s3XgqVS^A){H!GEgB;<<%+b$sa6YrxRb6(6PGA8 zl814BGzlj@(Az|@1cb33`G_5bPTF^v{ouMy`u2JQ8+{+%2KWnoHf{zbISA_cyu1xQ zhHDZtD!%3^et)DTpeHl~(lB*zkX6@W|NY3NqU0yqvlvd*6QPv3*cFp7NcYF3zXZ8! z_Q2>*q6ZgSVmY$jdR%Y)a8{uwwam54Ry{)3*Bg&U5?49n#xmhtXc{Ygvvo)-1|HoEvuyZ2Yv! zzA?bQs6LPJfEvkSBwvX=gP7s6OP%yiJ>YtOt6@xM1;G!m@di^)V;5tQs$)a(hNURi z`YZ^FF1d62)wUTsiai`>kCvF^o7M9FJ4+DPsjUBm$1}^eSoY*~GA3MLC-|qS+S^jy zvFVbZlh^ILnkGWbLDWp>k+8c{_sJCY`c(1UBSx0_N)?ib-_pM?3dMpTM~LG1+lJ0= zXb*F8!bwp$^4f(&yu6u!OP&%^4}063aHd)x{y~1U+TiocKhc`asWO${r*{&QZ7jUp zNo> zN4R}I6}HC-A`bZx8GP|k7(~(|8Qh{f3S-|BV95fqQ{bz>o?Q0|C)%Qmlx$Ghv( z;Wa=}%xd}@TH~C<5U)eL926hKVAT?M?cVp$uEJdQ#4a7?D}tXCnx=Y;f5)f=szK_> z=ZY@xFfq!HM|?C`be(FO#W6w-x`ggX-~dwcyZz>i`vE`U?i^QmT4cG_OAuE`qZe0b zb^S=4nfQCo_2Pv{V&V7PgMq{Xsp3=8DF#uN?`?>gb9X6Oo~SOW4#PQ-GWoQu2$g)i z??>8h=-0J}C?4P#LEBeD&@GwGo$A?V`b1fpN@Ng#!v}X=C_mkuoTmw5BC8b<*-XxXMY-m+aHHUtK zjdopRBz+=((+HQeL;X*ozZ1*r;ROFlP(7&3P!1@y(K6B1|r-7P^9zPA%4KD*i zlp7<>n~z#+9=h)sCyx)xYI7^{+-D|^k6s1?pWPi#;L@FQmBI;7Tq3n+b?^viN*r_k z@;;&S9=Mk7`?ya)tfq;{LAkKEHfj5-y1vJCJlrzI%@1YD&05$$xVZ=xOa&oLM&a?0 zU?70n@Z)!v7;qHRrw7_!DbQ+HP*r1q7 zgonO$B)Tbn{#&+&XL~S_VP{mC#{bKu2JSjZF=pA13`D2~cT@l#K7 zg=YSt=q-VXDT8tUW4JFg>T0$T_0TeJlKYzL#PwpFVdhCvlr7C?+` zB`mR*Xnu8stc%)8q|$d?3fn&#+Q8_41}--qku(r6;$C?*=KtoR^=6R@LKE$;^la}} zGPEyYoeqO0OAIdS%o}4mqgm?~ANyUs*I2NmZq$+=0k)*%W?epZ zt~wQjRF6(9(&;}GF(oIVuW0$IuBQzX%MOO3&oPwDM0gHpL=YBWU2FkBY~-8CMs+f} zhWtq$8JBJd=`#G$Qx z+w|uJ#tbucdF>9RT(fU&^%{zr2@AYqQNMU`kOr&;J}x18h>ODmLHUZLw1mXgdasMO z+3aULpwiKwDC5Lf&J{F-6zN;YagT%p)^?{Gt^xL-?{GT7a;+1=7mjOYUKO&f@Xpvo z+1{U(CbSFMO=zk&bQrI$|Ic-9klMoLKKQw23giCpZ$xlMN1cA+prb4L@npR^CV*-6 z&NcRsw@7ZC*KFE-=A_@Y=@;fn${=zP5^K@03@xF4Tn@*oUyLJ@w(2&6Lt8u%Xnsfg z?_PX754knvIT@EtU>@N|EhrsbY#T~M{?o7KocHTHauoCZs@kY!5K^*uYIfz&5T|L8P|~Qr}w`a z5BQAE=lxRI`Kl;Y1oI&uJ8*@V-U@ygLyGZo_=|r7+%o*F%I}d&QokU1JmkHH2+QvI zKGxraA@@AMZw8{>tQRZvVeQR3kOC>pE%sSx)5$xr6A+^hjXb3jBxbsPj;a?~iYsk{ z)~q-Hq3>?ZE%Dren=PmeJ>s?Rk02>iANfk8*S8OIISt#tt)d;qUUH%H@F->ttIH%_ zZs+fqM}@cYG$oB;s%86u64phmo7ifL?i9O8JHqfA9$V^Zd|K)2pzY$J4i11;HCXg_ za~;;&k8vPpqnkz3%v8YVXHIBwe)E_HaD%H5cVh+CbKhH&|1fuygE+t{RYo1Q9ZIeLOHr!M-9UWC|{j1D4zVjX}HGR9m(!Dm-iU>>S zzdhN(1Up)hFiLc8Yjon6^tI&UaUw zLkOlnZ9^;1k2?N_yTr*kd8;d}^IMu|XtE`0?%95=b%{6+1~1p8_@EN`(@jYpv0B6^ z9)kKPuuhz)O`!brYhW41F^B&o!LM;PMsHja;5TnHbJkUmp6wxl3gEwU$JY0=}o*qWjK^-Xbx@NHBU(bQo|LO}k z>LAfi8QpNBxi`gJBNsL_nA2T`R4qtavuxde$Igv{{xfjQSiGa7;f+niqfxU} zmCE_cg_Mp97s0pgE@CINnme&t%U$ds$MRuz^kH)JVRGy6kK9biIg3E!vzOcqMciqI z>F9HM?rVCkr^9K6oid&BfVHer&f})-p0Zlo+g^z=J?;{{|5vCo1mu3pFdlUWB-4k`yW?o%O zAw=Tp?ULb4%UcZODz1@-gfSAIDI~)%uWDYacl3sd%T^xXAhOm|gw!}EH($xqJ)ARpT}>5 z9xo5S+q28Z)hvxnp%$9CdDs=ss5OqE%k*FjovW3Pp%&iLp!TH?iU@CITn0ow;9-uV zu@Qwnuf|9&(d9jJjy3B?xCT0dIP5aZNt-OSMl!@5#9ai?|K(wfv>QFA&&T&

    ygBJHvr2H_3LR}RS>!Q&k-Fl1 zr|($T^pw5&WokFGHMb)!dPkuI%CV>U(NZ_8UViV+W0!>6eVS2Z}1%8RSL|QN-oX@y-S&Cg>ccKTfsH%r?-rXjTCP0rk#>Sh6Dli)%-@!LV0#@Eso3TJ6C#+KA;8I5?xIpcnYmnh5Erg zb5YnZ7rgL_m3+|7iQP}mazc$_sA_g7qn6n09f?0jcnfMQ%qp?^;w0Ke$LX$li8}Sx z9aWx!M?OC8mvv9J5ghq9?`!zdlhXn|y&Kah_eIcl|5Q_;+FygoW&nnQS$(KMHTy#q zwiQf0n2GN(ZAkUEu)C}N0b_P??E^o!uo<2>(5v=VmlwVAJGGiAlSwy{B;P6XypTJG zMC3gD*3u5$rC|Qi$O}RCj{+Q&>A-{WFkh3gBwuvht~7R3RCA{aHwTMgUiaH=wM;}|3fGwlJp z;ZEXKO8##}8LSxGSzmMGWjD2XwkNlWx*iER5&;7^T_aBmC6l!>sZ)-xi;BylkfcuW zG^~0BLF}lhVM=X0oy&*S_KS>R>z4UHxWN}VIX=Ijc(mEObEw2+rQvB(%h>LUNGt}P zCt7UQDxpxY9gvG0H}58uT4eoqeR@z-{C8(PjqxcjE$pGuY?0mNJ)-y|HX{Fo8NpJ%{xQo01ic>m`>< zle|&Rg!}XzTn~TCH8eG2uf^lT)wK7t38daII01{Oi;^wPdJ9Yai93IEfx?ZHXU^Kj z(V}E_jBzd$m&Bw)7Ky0}#jrFEOC$_2_P2ox;EoR9@(&=i79x@bb6LUWLz|O6Ugeg* zz284PbYdS;onyb^I?)9d8e!5WPf;nXzNnwbfx8HD5@xa!!ItsEx~Kh^lh)KSQE@#Q zXzoq_fiZeuLN=BG+Cl3eudTsW9R`?@#c!$Vrv&|EkVm?nsSUWy<` zw)}~Kdja*_#gARi93q9f{vvE$Y%n9wq_U0TkdRgxZ(!6TRWYp7tu@L=mpgtGaXh_1 zJnIZrv$V_6un1M~-zk#PvpFyQ2|X9j|=W__~m+>Nx8mLsfkR)Nd8;txL04#AD z?ejd<;td~H7C~#MXpF`^EUE>U{}ZZKE)TlY5WB!s6_>{DQNob6n2(*Y?}j|3-iGZs z(xzq)wOXN}4sBt70Ewo$FtA2P?R)J5=9piTt6^{;`ufe!ZbeWh=h-Xj?-7_CpSZk6p1PZIgq0o`}<* zhl!ZA*BneXc;HVugAg3EiB7S2@$sQ+Gu0nZ=hYLT0&><3lzvoV92iqo+QMN}(E8Wu z3DN3Y|D*$KMmf$ixg_UACVVZnrD17C&M9imv~>_J-(0=7Py&@UO&a0TP~zK+cjo)a z{9AOu?&2F(rn~h zrm{~IJ{}Bn>`Tq22{m5`Z3eKFag!k6WP!y&0qKgPj77(Tl~ zuslR!aMsjWav_tmTlx>1oKO$5&ZQWTldcAM+ z?1IIdsXC-o`B^j6}Vc%$h>iiZ8m#?#t$;y!;m19zb^~=oP~|T z;$%Zog6B^u1na_*sTop`(k&CpL*oj?g9Oxy2})rm?|9)xe~^;@fqLdAG@^gVxXb7v zCf9>{o{VO#%o$s17aOpPsCw1Oipmg2sN*3h*(6hjK|N187rPC(>zdBEG0F%FjO}x} z8m1VNA!r1jIrwiPRLKWzdB#`CCpb`?2GiutW~0c9EbQCU4#Zsq==(=Ct~U4v^=KrW zGsue>pdLwUfax)xmZ;e~uXqN2cd)5!ECroI^#BC~r4(8qG6wA_y>uchZ?)rJCgsGc zPzr2T=@9OgT+q}10R%w%zx7ZW#bLH-YifK^P^IE!vfZ_n8N2mEx9dzlv)Ae?DE`|OLC_V zxCY*1IZ@e)ttnEn=$Y|q6z0-E=?ov)cEu_K(mCETDQ;P%!N;@|{FkVg2}j`bxIoR; zUNwA$l(Lr06T)V7-Os?0p;Ck}};`jNyHpyC zt+Ex@EYo$xp3ss{VS)nG!r3}caatvumAHq$y|2!#Pg~Urn@w_=lJ8z!T}L<7x*@T^ z495R83UjluX6dh(48;i5&ZjBcbL-ic<9)95T)WLA0Ea7qOn$1Y^}~?&Qp< zfiLKq6)&jj&gi+%ZG}LPso9ppHg-4(u(i3bY(n;hYlGp%I^P(hVgASFqU*DSjB6s?j_SRZ$GoroE?& zw6kK(TIPJQ8J0_YT;SL`AOT9mCC)^dzevz$KO%Nrpq^^U>wtjvqn+HeiJqeTi|djd(>%LzOE&oJt4kXe)Mm=7Zy~OD!)pAXJImI$3dM(J~h1 z;yGKFQiFN{DBCKc4nifzLW3_{ufm~|;F$SX?BdX!Zoyflegiw2f@lK=J^>r!dFNY2 zR*fDIO4=@mIa|?+W|~!8X`ZQgNIi9dY!*9Ep@8a7W|LR4x=(XKa|`c)_7_>(L`fkv zA^ahd!9T}D@Fb)8NR@iXP@J0h{SZmtpCI9TY_fNll{6$Cr6hW{gVhHh4kmebrEm|x zj3;;}q;C(vZ)G(krfv^myvsj2JZ*bKwTk4FiHAv-xc^j%he(qclOi!9`C){yG%e|2 zWWvLkVE++gO-o@InZEEKV@-3BCL|*yBqAg`diHhlj7WTbh@|HSNq8Qa?3^;r^hD>r zH0C7x!A_EMG=+I+Z95X22h*1awXm+_=H%4n0qECTZ1DclmPe*6LqhVJ%k@@@k1kVqFzPwq@U9)K82O&&-*9)R4HmOPYnJODCQ{UDlf zJh-e|>Bz~+#zPp;q#}<>G#)aUsIf`Lhl=835{w_EqYaem^*M;Nvl|DNvb=D=tCcsTgn++0T6JQRf%=Rb4CV<~bm;HQC zc4~$4q~zB<5?_Y{d&eiert|Ib$*y~MK2JcK;Uw48tcBz>rail(CqO2HDXTlFs7X){ zOIID6syZS~b&t;KiIB-2>8Yc0n#QK3?$wDsdBVNvsJkhq$uRFH6?OE&j!Quu-Nik5 z`U9z_LC#e=TK1=%ZhtEo=5QDI;Iqk4lRLsELp&(ebYz<89x0~d(k;iNmZqkaj!G%r zBgHOdZllsj_f=d*?arRv>Jx^Mm^Qkb5SlW@!IaVMWY7el1L>ljiJ%E!6MNo|5vT53 zn%>{qIWFmQbi!vVoiojt_=%G}6Axc%lII=?p2N&-RC1?F-5iY0I3Tfe)S`_+t z=TEhAqxp-gCaVM@y#^q zfDoo9U9hp+vi$;>NnuJuc2!zrHJWB{RJ&^%_;6uehldHiWR_2Q@DoIG`-}~F9Q1`_ z_+QtKLr2K6l1rn7z{1-oO2n)IxQ?5;IigNP3*y^|drb9cHMpb*E9h1(2rbGL4Wb-( zI*bnhjm8nUNP&B4L+XbA$Pj*o3?VqHEy>eT+0=j*s|VI1XH~HQJ0rGbzZV3P0&!=I z7}&hzIGhuvU$|e)s2fo%@!X>neZC0ix+`pRmDFvaVv7I5z+QnQqZwNe3~boq@jG}F zx_-0#IKHk)0%B{mIwMt^!JrtOP&_PW@EoCYS!+;%KvgYohssio@vszB>B~ya*)>mV zlF^!K>un$MXa2r}{I_{(%M0uuiYt_2X+st6ImxMJQ}e<5HWHedT~~(Cn<4GJIBgKE z6-q6X>et0cgBF-;=WK)y+c^R3gWx{@EY@$$l%mV=i)P<*E48ZER7@*Ib0|h(e@$99 zE^??)AeLJv$?Qw8mMP0MvLy>uu_8)6GsoA^2#qBMCas?PgQ@R$QS*GICuB{t6&IF| zC`|<}p>n?L@*d2r1v|VJXSl)nOg9aLopy(X+@4&1o_C8L}Aq08XE49 z1h}7rI398DTRvo;+7m21JQk1Q?B{w@CLSMyhuF&f)>%p8pl6A&M=U|AQ*4?R(ux9W zlCdY*(=cT0DOXgDG2X)SeYQ*dv2Q2t#o}DKqd17rThyz04eJpp$C)sqp8vucwQ=YG zz6+r>syCT$T-+ISA_9Yi*Gh*~<4ms0f?;%A3%(T$F@w=k$oa-THl%Sj+4`~~Mwcj< zPEtPy06XRM{?yMy4e`+jB0p-n+(o6fq4}CF0k4~yhCbMi@U@W$FbyeZWg#~#pOKd~ z0I_4FHl@$gt)Fa3!k=cu6)ZHH1NmWKu`pjpG%#?>ihYr zSm4%ifo=N++n&)vj$ zmI_i2s2OEoS}kjEyZ4-bmqXVe3m!l8 z+xxe{7+6xJ0IR1w5cwZJyV6r4b}k+ui2LUsUEQJ6-JA4FN< zzGja*RCLTN$As`LoI?*_a}o-kX>!75%USq>9kik5h0#Sejv0kf6FpG5R;*Y+?mMt6 zNcaY4EER61&4*CV&ZNkA!DAJ)z@)LDyl`Qhm{IC19fe#j`{dlHMWc;dZdeWZ!i3T` z&J~f^y%bx-P}<&Z!k!(6Teb;-9XnE4MN2lzxdMi{nO3ZjG@nAf*IQad%ayOWc*Dd} zubz`M9>;mCt8F zP79gcEO^0UMUN6@IwYS*vlA1=jXnW;P8B;}_u3e?#0w-0<{isw`NAjFQsJb^A?{E5 zB!F%w^S}x&t1`nHxYvbx0em=RE?>X9d|_XEbA_*YE^ec;m+vlLINlto-+^S!uFZee ztm3RPYJRSol)46>$qUp)I(ehH00Y7PdiI(ZY(ieJO5!yh;}*Z)L$OX`+CYa&m=28}n#tje2nf?P2-47qv%Cd^-5me*~UKk)DpAV66C7p28 zo@4-joDp)7mdh|_-YjY%-LTCg+1a`N#mJ653rKSkvorqXX-)$8j2dAvQ82rtdga7K zCyv?di!QrjOg0-y!xE6rjOHK#)0XvySuH?eFQ{fniw5F0QZVNRO~3muG^RFx7VCV&Hdh1OY=Dz>&FiCy^n*O^f%w}V zye*O1?$y%iv*BlBd07IdN?f-z4mF-Tr4$EAuJ+FpZ{ zWO6;{0u#(;4&5Bhylo=5uZ@eOs#~VM5gv+qMLK>*`(H2zh4<3pnU+c1I&c-wSw<@hl1M=cDa$#{Zu-_- zNn5Z_lUyf0r{tfiCk$t3Z{_k0yJblmYajg-bI;kl?j{jaV1+$D z-O;LhRi+q=w9ksIc_h`QwO|+c7Iz>==7d-CrBwayt;7wcn@QxRKg&6KA zM{z5VilLu4t@Mn%T9Ri^yJH7~Z2qSoToV52rb);@y$k-%gTGh7-|OIS72a2;e?INb z+aFxYXhof^b#!Gh!k)Um`A4f#UEDC!UU0~0H=;C#db4~&gzv?yo?xQ_7pdK=Gj<=) zVfHoD2VrdOh{3uM=`|yg>qUH)wIcSe6EV7+)k3oXfWVmm^BTjx=oJ{IJ zFcvF^`VQ=L6^K9`>NhadO(14akMtS%gk1y<%f%!_t$M=Qw@M}c^{g><9g=cM;%S#x ze9a5g%_-4zIf49Lkf8s17VKURWE_IRom$r#JXKqxZE`z}o_I}{m`jF9o_$ul1$@6F z^>aAUvu~(;$&`*|fDYz+l3_WgGoyBQdu>+jmOwbL%{>!)863dT*|q&v)bR`NGHiQb zKIE!bG~;5KY$v%6pKs;p;17fgZv?qs^%k+kEb>;)s!jUd&X%>wr*)ahHP*X;EYv=@ zf>1KA7=>b-cl6O@k}`!ylgXpWB)GE@{@#xar~w&x9<4xY{1}tgRvFJYtg5{v_}*?zFlI0xWkUpG<96yg%F-%f%agvoR=@Jk%d9K?plALK)t-+$Ul6Uw! z_^qN7EiD9B+s=hOfD5fzzE)rLyMcBTw34;REt&K6doQ*I{5GiF8x_Rx!a_a}0HCQk zBhO~fo*umXqlzFv9TQ-i&;r2dN_~3p_S$K`dS6Bxj6q zpw~cP;YxCavjX!%P>Bdi8-dlpx4=;3f1G@O_U<2V&n{lY%3xsmYvyKND<8ZW%%D~D zXuwS3$J()}Zv19ffY!PBfY-m)H7y3#fW6O(TJhUM&;M{~56=~QpLnR^AF)znvT&uz z2dZM7FG5oxh|)4YRw0DlCF~VrcC_45=2*V-+QS1yUTd1IP~x&*GSZFc(LN}LqCnIV zdMn`SikAf=fAn3ZCTy`_nf_CNo$!3$_WDjwe{_HSsS90(WF!1)l2rE!9(6@o-8^7= z2*oT2jCn?0t;33Q^l1Bdag zeD$7XweCGzgXbz@DQPX&H%;?OoL{SXWrL!wEoHxEEbV&stlJISDBmp5w# zk325B1p5X4Pumr!iQRN1TxIjsUZAeSn8;ypFsywSd8m(OI}jnUZ{|?PBW~?< zyL|O?O^Y^b$6LK{?aQrpaptuYjsrL1}lYr_%79!;-UCPkj6|6OmtwOtIgWEGb=*gRr|tG5oyi7h;5aaJxI-SLMy z#a{#{?iB!Q<30^2xj(Tp@)uvMbV7K4NO{;<#E#J(8@ojf5m?BI-Li_DHoT7pm0$|o{|%It$&>BHOo2Gtk|5o+IQG8tq(z5k#R2c zzyB_ZL@_^Yw3b@iasVEQ7TH86#TF`9RvM+59j#jNBnE9z1A%pjWcs$||1|hVNyV0M z#_;s)dBPIPv3AK%T9&eq%gt43w6YgcC|v;s*jr#ictim}453AcdkauKLkw)6D|rWg z1N>%G+9_qO2wh7n8DXhNH|l-l9_S~1HPDZ}9p4?k;rj#`9hQq`^f1yHxs-VXIlVB5 z#AvHuxZyXM)x))HOfga3CF=E@RnT0*EEVc>yds&bDyB+_%>(^D=WLZy(dReN<+5YP0r-YSTWVLAz}* zo&a?3*6bseF0~!|Lo{MfZ^542?Kr9Vdec-r0b)Yi^{sa5Dd(WK;rhOf)RSANk8gUB zu=w{?fgrI-dSao1l+h=)L?7SfEpeS6q5=BDv_2o-^n9ol^yqfyp3g?X-bbVJ1Pn`f zWPFBlvpwIiO`48=(R#(nF4{_v4Z~JVXirp62eA|Kgi!-VsUD(|tfuN(Wuk`;Hau?3 za{_iDE%ly7E^LUDA1`QjGNSw6u+0?pK778}RScPEX$3({Q%v$+Rqn44Jv49HRGEH|#Fy^c-K{!&_l1Ia^i?qA_vj0Tb2{ zJ~C&d7VMT8+LLoL=XnhW1Z?0x-L07TPAICmT7d8C|9L6jO6@G#KlO`)d?&A{G+HD? zZRdR9ddSb2zGJr8BC3%629QNt$@QTnSZ&6umW~213puBSi$L$En%6te`jOjzHo9$V z0R3|p$a46S&kJUCU4z-buu8h_whhp&2P1G5DVS1NZvKLcm-sw%Nha()U6%!$U`Xay z(_f@uBr9sAV{!2gjhY5V&ya_~3Ik07U5Tz)`)u_>wAK`=$d^yPeDdtMk&V%+PZgVV z?2P&Lvl;o&3(?Hv8TkPJ|I+^dEAoMSAmox>CfNt=jgM(H(2 zVa|xMI1U^DKM6t3nT!~W&6L0VpYB}<&(%Z<3*p%bOyQK=fz5ACZ_uuI2F+c~U{Ai9 zKI?ZROVI{%%cY^=Zl)*CPG_yDn<;)dNh8ZH*xb(=Moq;AKG{zX94R;xn>)H0BnXbB zF8M7ZE52N@Dy;JX9!)sM7@J5n!H@M*Fs^6hTpu4RFo~i(k8+&%kyul!*>J$^r`BNJ z&-`sRR9~EHV3q#r%e}4AYkz{1Nd8fC;sYNj?VwVQp!Ci!cZB1IRg3m^Q6nS*QR65H zEW#+VwJ23H&k0llah6$vrrCaw6{A{_ACKDf;>RQKHF#0mM#nXM(7QWi;pL7+$ttsn zAb5xz+v@B|m0RnxwGPmV+-U4%32u^VXGP*}B1GI3RnR*folQ~%^kqiQ5b+iGh|D~q zQgURUd-u%IgnpWT2eP1hj;v+HEEr3OE)6#U@2AnUJo-i2eEK6RcrkIYGR}lQw;??g z61c*1VV69DriRyi-}IzZwp|n)Q`O$4r}lurgG0_Y80Qb$^wAqyT5=OK5_=tCqETkv z!L+}e1gj7mseS(6l6ZW>%i|31xn;_UOd9_mmB$)1?ZnQ1J85Qn3O;K`B>L3^Pdtop z7x*k^UO-V?*FZofM6f&Hijf?D(zj?C6;K2zV5SozH$ba0oBh~N^px3HryPBC@)enq zr>6$X1s)^V#85ObEIL}=zzozV1!hMSWcvX4E#aZn+m2#fDsa($CK^aLR%&%DXGkPhu+3Pl2*}=ZG9`bNLRQaF z>NTHI4J+#%M$~A+MOM@~%bj`_FlJ@M({oVxpYIFZn+uA#+U^fI`%bcAHh`f{naj4@Q_1kMK)I~v; zy@zDbv|*O?8vO2Ku|Iq(hWqe|egc4TL{U8N#OD~+1bB`p_WI3U^#d3Tw zY8-3gv|Bo)5!PrBDl%yyXulbveKOoaB;b5M$REYN_}zbAU@UtfcrVN}#J?v5cS!V2 zVie_d^Sm<)+2m8$CbVYGU=FCT_D<7?tb|&_ww1cL^3HOeGog7#i%>2#GB3|rfA+a! znU?N>IO4-n$PWz!!f3X_a9{kupZZUX+0UbSo!v0)JQ@|WuPC8sZ9zq5PRMq|M{g5X zf*(hSFe*&OB-Gi);FCezAcqQi8H=fMG??gFq2$ES83%aR{05)Ezx(wVP){ryWX$za z4ZJXR-?vyj75)3OO;KmqoAZ3JjaoT~50#Hj6?1g;d1YV+9Q6S;*ia02oS~eOt2gq$(T>pM_01Say zZ zAMzHr<_n;2F4K(NvT8$0sT7b7An{Zwz7&zB&(2pyHR_BoJEvY9S^eq=>^8DRp6}BF z5SZ=bmZ~9DpM3S?*^_5GDt#}|P}RSL{~9~I`~Uf(!!i=_vfDYePD-MwKI%pI5-jIm zoi>(y^3{_spFBG;bN=$w%s#Nie>r`2N_vyuw;PI zm$Y|0wP*ZtZ+U>u(g1R?RopOI>L#*Y@j?I(Sp?va0swv0ACr^+%Z|=@lRjbDh`mcD zAa5Z@sST~$X2Q05=2giuyXkw!hu>F?fPwk|L-DPHbpiIT31D*zV!r?V>H!Q2WvO)l z_N)OA`|#ZZp_gXH%goZCN~ z*FTWc|3LZt_npf>xP1vZ{Cno_M~yyv=I+NNygl>wqg6%r$=SyZ>POvu^7V&v@(&Z! zB6;||Ir#g^UNHSM`6o&Au>iY8^<=4MTi^-Bn=ZIc;BjBatl5t$ytETQ%r52bEP>XD z^#dT+tJ@l2rkG_+DKyd=Z9gdND4Zo=AGwC!saFM6nrDA0WOifbo@kosdb2fCLC$`- zA}_hR@ugzpwx_L{36$K_b5<}=gFrl3(`I0LoZUWAIej*PAt0sKh%eB44_^s{lmq0649NcSLgpicir^(jvj1vE_yX^f zwSh+7O3~ZP5wCZQ=ER)b3I6#*#Wjmkqa+q&P(0pDb8ABG1vi;gJv@uGfRsAw%OxYxK(PQT|;Tc&Ck8zriTO(Zj;EtN*vPg0d5w! z{}DIFeGhTQ%GDw*d(J%i`l@X)eZk5Vd41(2?7&|bvm?N;71~#i6)E_FMXCs<6^9pG z%PPHw)vHye$lW;cHT<;|*TPjCaCynA)_ z^5WHOo&RPf@1|N#YsIEq$2em`;g`fxBq`-m$}z>ckHNFWl?^SYg>9!#ZC1E*VM?jv z+Dk=k#}aDIZ*@`NTI-6jNoZ+`wDEcBQw*wW0P()r0QopbIfmQBWQX0g zZr*9H_EqctkoE|OsU^gWL*1o?tPE~eZCqkHgGG2!D^|_$)1S-Y4lz0y>^l~1&DgcH z@7SViW$kV6`Hd4Cl)z}MR~U;AmRBndbS^cP&-(Ohn_5M0s6)i=>V7_*s_?q!epPsl z;xvQxyT+GKGM724=}2#$hDMkni>jQgSJ7NTJW>Ta9_La4I@5g3#b-keQrZWg$g;Qm z$VJ}2>RTpFc(yG-+I!Xs<9*DrbPy*!Na4KDZ69>!+1g(I^+x?%O!4c}GH=%Z^Kl%v zjdF1V5qmJBob$wuGR+MQ7J;R>d=K6?@)4xJ&r(R) zD2gGqa&`yEcAynqGtH`|dQK*Vu>CH+Et$B|G`opXB)O-zWC4F0%T{49Ybi^pv)H=?n4KL9uSz+)mWGdW(grWJ zN!HrjnRD$4PTU>HG}iXB<+0LLO(B%_ zo1iuNbG#H6@xyTV*S4Mn3cA%HJe z+9r=1>@9evF-H(dryQuPKwQ)xwcVGj7zh)dkb>VZvQS@FvM!stxEKx_g=tIC;UF4Z zR7`)wIHW=QWa})!(_0du&6I9RzLu0%B$9njY%B+aeqaVko3UmrCoepiaY4#uCI1O1 z5O*XC*K^$}E6NJpF6ljR6}XZFMhF=yuWVnjt&4X1dlXz5?1XA)dycq?%Ql#=5SMDP z0~xC>#SLgkuVJ9n^#=&zz9DVF7WA^urtFyClvYrlV!F251YwwF%uFe)gcyanrqzwn z5iLcYL4JwywZyQ<|1El#)^ro*Z4Hj~cs4t>9iJ71o6}LM(J% zkWl2I$r_<{r9O)e5q5+YH}ADs!Fqr#cMR-!ZMpKddvayGI(V~fKUJ`h8HWrv0qni9 zWV{}LG2=CcJ)bY4WkG{=Z4)_r;6s7R(K<*9On^9sBM*l;<8{w`8LvILGhV4Y8m}}? zjn|Ne53kl4b8bGIz9p$)FZ@M+lls;$CA1`Hh{%jwSfY%Fi_VoSYK_2Ckl~_D(;KPJ zwJ`uawo&dI~|A0h!h~(Spr4qXP>!wlmcZ3bcbT6GWMm5d;tjrzvZt6(dX|E=x zu9(Fxd5Vb zp;@({89NOv46IzSHLD^ePF*mh%XwQ-k=ycX8LNPhu=ko(W-ne`os+!cx2#g+wl$Th>pS>hCjR8((anHhY8bW9+H8(CM@=jFOxK)bq3 z$J$QgMHQ*o(x@O+P<9eayBr**yI*K@<8tYbMvl%2#UHbyw>)>U4R+)3^#e<)tTho_ zKu&iACX7BXj-^}OI}bv4@P-}N#EUwM^% zsG=hA-i8j_zhXjhkV_1AF|rdkBD{oc7+Rn010x})uVA(xtoi9}Gyd;~Q#`SMb!~%iwsH~OSd8rx`9%K?}#?qiK2pnP7O zA8p0#0f?d9qqF5#?`+vWin{48OM@aOt?~uY`8!7B7}oybcw_Pi&$R9Q^BHuL+a3|f zHAZy@GB&jfoCU`$!9wj1?k-_$gaVKU@Mfe!3k+ChwNb>79-BRD;uVG)NhY9fLV^U0 zb6>GGmv`chR{7awZ~j#1b^^Sc?FIbm>O-n<{c05^8 zl<4$w#msj^4REnNoN8z(i$Z;Lsy#Z@ejQG=b9AbO-ML%z#uc23Tj|F7EnYU14#nH; zw68!CAChFGJ+#d(2C}^?vm5WSK}LB%0M>v4(=q4cCWQnG!lH99@IaPwz^sQC2s_aK z1bZ(`d|yoAvY|k-ENRA~b+?+!Gu8z2a{59M=<4mn)0C(CYlFH<%4-ULzBcRDQ%kGs z;))nyPBScK!m70*axrPe5^rr*K&K z*cLESpeBhRZ80MOY>N9Za1N$8j}qL6fn<^7oK0~b2J%G`nMjgAT2e*=*k85#FkqUO z)|0sFN=#UU(7p}pjKv|WxIb$u^O~b(J&7r2wQ9K!kB?`wV+{Og6oy(arhgzzWHQ=E zrn#J+`+Qz&XgAz$&xPS_VoOqPip1VswI z=1@EUyQeXw6*Joss1={DlG)&49F)yV*0IH7>u?LH{wny;b z5_W3*%EyaN83Y6_vzI(Eh0#6TPF&DzUCN498tZVKvA7f0mM^E-4Qfe#IcFd@1$nkdN7fI%W^~9Fh^A-= zgpfHi>+JyHm|EsvbfG4k)uz3atjGn32@k$~B)JFx|AoCVGEUSPHN+*;d1Eq1+)DU! z!*r19XeexO4Ma3ORd< zHp5)ag}SA6pY@No`C3pCI*5kroz8qis0-loWZ!v+uf3JV^1E!4*^(_hsZ2Z$15pJs zK&HMCccbf*Ia^?)O8;>&ut2f4c`!LEn1<99=Iixi*vwkq8m#Z_w1Yo(Q0!-vG26rQiov>W8lt+!Mn*y#bw&vy z09(&lo(C;vz=m=su#vL)$L04gCWOz}%>640(t&YoK+7ct*ZW)#KzNu!U}%oQH0$Kx z{`5yPar5S%rhg3D9Q^4g!*MF}KtgX2`$ibh?fbL+bSd+zl4Ud7hUX{LhzS$A8Yr!G zDIwa5=Xg|L%oY2m3!AY0T`d*NsAgg?QrV@!3aJq^c9VrQIRP`F>&vR81iAyXFbeZb z5cZxcD1m}&3tK}m%G?Vm=92~!2K%KGD+a)PzYtP?;Zfg34`RD~LT@E^t*ZH`rXRuS0%qw}rgwIJZ+~7JU1QHhx zfIH(tS0QFYl59oAk~KNpYnlU&eB(8MAv1>C4Z1KAHrKE%b0d&=SJ#ii7c)ioa{R2? zCFDgf;~^s)6ldQCu<9P&wm1%rKtHG(2adPd&L zL>DVYVuPp*jnuF>ng1FROO}(o25Gz@rbEuaWRt9s?nbTXb(8U*HQ}|e7G1LDu-DQ6 zWJ2?bxq=4RKO7di7KqsGB_Fm(8=AdjS~XFb$j=O6w9<<;*#NVHZ5V_~t(Br%F~Ch! z(()z4eTODL#y5wJ;4WmI=E&RX+19+=WL;oZj{>_EK9J;ZSOXNj-Nu%z=<`*1`QzWUq}~Q>_>iE4zEtjF+>oH> z|x^RS}(S(swT>pHYQnBwauPn7v#$qWfl=$FLcf z&$2ob8(hkl0m`7cCkf`#Ov)%cg3b%fCG^>0FmGDi(T##roxDjvO#NJKJ?>_J@7Q9X zmwkAR3tWJ3Q3!fUdwT2Son4;8%T5Pc!ap2mOZJ4iYt9VnH)Vlf+rn>*jzhz%jj!K8 za|-KS@*T%3X1MAt~ssUq&$WRJEghQfEL~eZXay>E-1*(6!CiCWQA}Bg2alA;r{Yqh_{fa>uu7_rL_!7{ zIyNR=AmAc5TmiVN94grBE>&k4W8P2}HvMfFfnA<8)!8a3PMQ!Up}45hv?!Xk)D{gS z$XM2U5{h1Fg#3C-l&Dx)P-~P~)FQ*EKCU-o;?Z7ii^ibGE3`}T1=`LVeL9V>Iz$3n z)CPH}lXy^?v4!(q%e3IVIn8eFXq79-<)@lsbFxWmcqh$V(vFqL+fvl2w-6}i8&fsI zO#@nMxu%+DV4dQN!1J^AtrMJsKkZO~=m5fvT~<)BE65}w`c4wQHj+7oCZ>y|jDB}H z-&lYJJF8+;ogY?dHKAJggmyG+WN(J)JzmZgLJ+nl$jVV^$G6mrtF0I3C zoomAnecG`8VltQ69%_r=&ahz4Dfv#N3OE7j_BpNhrevY<+v-`5n$M%=(>c9I&8O>n z)O_04uUzx-nFQmto+d#j2qFt38vO&whQBWJ_4$IlW0|~V6%zhe?@KC-?E9MXB9-i0 z77>`&pw~~^&ok1R>yg)@%i1o0EuLav+5nrfVz*q@Y3pF)Kx>WERB0<$uK3co#&aQG z;KHo(#B*S&veH!7s-3K!fn(?%T?KycAA-ALG;1mq8efB_^>%t{5!o}ni0qeQL_XIb z*Y{+$F#1Te%2qwWuGmu#cG^T& zN84A1*GP6id@keT`S{m8AMaeA?A8fE&YZrvJ~3nWgtV#*y6=q<$mN=A&0M&cKN;ZX z%M%L}g_orXV3x!9F!tUzD+`Q&pjF)QNHpvAr6q*Ku1l;X4QG35lNg*is-Y9fV1E_# zWYpuBB$<}AyVYA9U$nK0=$sYg=bBX;Vgx@IKAR>-LbET$a~tGqjl3yp-&i>Wth=w? z8{rq~f$XIhZ3AhSZ-S*r#wfpxp-75RH7y}E)f#KAv$rqPmF5xg9%RStVfVdj63Gw= zE}w_dYDwQ%yT}dOq+2?OjZm}A6dhO$&1^M*83A&*=wbUBi=22RHTVdxzGfe|H&CrV zt(8J}VG&xpC6HrhmQAKlp6p|)z{~B~0n6aGy_GF>!!`poFLy7j`yqu|@zOOogmZ4O zfqYL3J_d`z8KO3OE+*uy)aHM$Ak=mwUYhHrWa_Qd@cu{HB%J;MTV(Cy9BkJZ4iCA= zBgfH<#fSCy&F6}o3nDAKx_cY}J4hU!vz0cknUN4X#%DKNR;?Ypx1o9u(m+h~?uBr` z3JZ=$A3qE2j;LnK1Y1@6TC-{i^>ebM1|iKgl1JgSCti<>Q*_rD`y*b%EkfE)JETb$ zfyZj0<7WM&d|2V4a9(%egx#56O>R8j#P@Vr{gbO%8|96C<$}2S^XGC$aELaG` z+3kYQC4)61GK$d8HNT|=!^96Nd{-Q~*pWZesRg#dGa`YHu{gS6o1@7#T}p;{bS{pX z3TZ7G={p$Z}I97KpZgc!+%tZd||07O8$ zzyHxT1TZ`Qz2%Ly&?*|5mDa2Um;Z+!-liAPKW^UX+s!*!75N>{hjaK9IWd4vV>3xm zNJn+}iK|?9cRU}l81Q&|5nM33Sp;HMf$*FOZ4P%8m-SBo+9`R(vEl;2hO)csITua& zj%EXfY**%7IrD^K8gs%S%~|RdQG8t&nhGXsRcxZ@^f5PKn7J10JsyNKChH$>Gg}Dg z?u47jzzQr+uDCU(caz)sCUG+B( zB~b2py*7fZhqoY9#7S3@>TqAeK>jNh2Grvo!k>2g9wHLe_jSW=~Y zr4=B?_GGDcZdcGKG77Rix^@hItC{3>!^kMqp=Oe*quU(LCO%j@(`XgQ$Igh3Mb3 zB+-m&s+b-!$F=3N<{d*l!ZTuj!rqiQp#vntmKGw@KUpzPw4f|xefQn@%dyL3-f@B;{M}xv1WzlJ)8#+cp$Y~S9R1>0SclDW@DwdG z)|gCRa&?ndUVZ1a_R;$K#Vca(CTcF-cETY;VLw>e6a+l5)b}!DjdLRB>gJG=?faV^ zxAg^AShT798^yPctjV3NA7}NHYhFGluY{^A))af*`pggxCu%L^6_yfI&z*h}1w7J( zxoxJ5!oF{8h<+@|Ub<})9nvx$#$H;|N`tzUj-37#M+$=*;!x~7U${8qiLI{XtC+io z;!3tJn_#$5h=H$t*s4)zxxsZ?zy;0 zTvmP{;EI2GL{4M{Bg&{C1#?;SmcndMD;Ut(J{H*{_ip|!^HHM9F+Lx#)X=M6&$|HKpao8M&DeK_Y4^vkx^1SFhvRm}kJ-(%(`r9^BE*`o7Y7Nk4dPWyPkx zxG?l%mQ^Dd^!si_W9o61IcLYiw~NhIC|@9p^@0TxI#;%uRZ(o3J`dJ9LYG+hx3Zp+ zvf{VAV9S^xaK*(kDDDxK46#jU+J^iK^RFg5w-+#+kGJ1mLtiHF7ay$Xn1MtaZce9D zFe#n<)5y$uaY}w*_WKeVO;ykZeI1vJa{l)1CkBF%KaoAQ;BW>7i5u_n zLGtw;6xvzay;K-r!mpmMhOXmG1z?%7Pw0})LAW9fV8d%Pa-s&)r;nBMWzJ|IEh6lCs zrHfe;iZGnha+pscTF=8nezQXn&}=SN4yyZg?&a*!nX6wttG>iK&%0JS8X!;M^JURs zMm19LV<88t+uQbPbYjC0@NuSH*Ac|NdAG7DW6g$mEf$uWSzE4XzPs_0Y;t+TOHRQ} zQ}om0GZ#c*0h=IyvQ5l54y$Z7War?WwusC%5#$jLA(Yn;+rQ)Hv9&b= zFWYJv2a&0HjbO3m5cJW z9hYEUpC6US6`S|t&pL{&d~Oky^2Q5YAb(76;U+K(O}`5rs9-%fKnbW&ml#j8ulOlXbC2)w zv^=h*Pp9k2nf{~HJ*_hG_(uPZ!L2H=@i6$L!;lcMW2~mOLLa#OtRmap8st&*KPEhY zMjRE3_|RxgN$>&KND{5RFP5Rzxr$82M=Gp!-W{_d)rONl*`n&cT zW$ineJ51ImYe(}j+jQ#N@?10QM`83TIrJ6m2N`}fdXQmh{ks>X%2BDfR_5M|{h=vY z-Yt{*oiEZpAwgcB9@kXjXD-%2p~V@tIZw zh5fndj%?M({@|6{L&NE#V;A@_Yp6^7B7Hedg83Ta*|)iZ^wk}|^P}rrKpkR{y|8|z zmbP#B4XWf1Hsi#gDH6Wk9)6(6s10s$URV){GqaOgoP5&uqtF}`>4P2jy2evPIbAD> zn&!56df*X%(xbfIKjJ;>13XCpB>M!30v?% zv?vqN zre}Fs{;@K22qb;m+p3IE~zU$OSEH zBn;y}G%hUkI_60qRYk1JWG4C(3&TSFr5K_YEf#~gUB|*)xU9b^V*01aX28%VxLwe> z7#uW$XkcRX5A+;RiJJU;qu<*F={f0cqr7b04A~f8${ylrgJj9DY-FFb8C=J@(cD~Z+T;|A+>0^Bs~aC-1QVl;hK0ueOrT- z3h~loZZ)e5vX*^d)wUL?@Ux5Epbs2;>s34Y0LN%G#S*{iu=D|+G46|nTouj0yB_)YjM4>Gdn^Ca_)XnE*-sCLAD zi$YytQf~Oi2i&^X#v*HJ)0jRT_e#|BCQpPPoU>>ZV@^7)ct5EhGd(wz-7>&QPX0i+ z7Q<9xKo;3sLg)$PArO%l#!}S~A@o2=5ES^sw_%ydj~95DX2}N^gDIn)Oe!H5OL7}D z`I)kwiON^zA3it-aa!363n|I)Q?AwZa*XTvbFW_u+Z!Rr5(`y!eDf~A65385`+$oC z1F#<_if?UdNEW00$-3ObMWN5gv}y}c1<#oX&wUH z5-AdTcRFab3+FN35Ha6>S&WJdki;Of0OIT7C9sl%+PP|=08{v~rh_@PRD#N0?6oa@ z{>Hqg&??$B@Y=xIuoAYxErW0}Ri;LZ%NLbSLZw>jHhP&t@mvIN0WUkPaESe{j#6}ot7{g>UFd3k6fA+E->F(os zz0K;fU+O&nPai_+$Fd0wtqZg2E0NH6N@pIfp>L$;Q*w&Am!>?4>03D-JynA3lN9om z(RI}bJrhADwy8FDqa=gO{9rh*VNz`x`LLSi&~PW@(+Sc%na3G*oJa=lYMC-}2#_(~ z$qeH7nimhi0$C5_A7;I+xfcR7btO1>_ z9JWhr$A5RFYOJHZmm50mv+r~o%vtw~Zb}XB?n?`V8IT0%V#ys#k0$j4`nU~&m{|lasJP=_|~01X{t8rQKv_J^8R1f5$EUm3i3ZMt{(&e zztyS*K6A0H3PyYJ@_eNouvqPNQ!*=o%*&5MZ)_#Lom*$5)kK2%fU1xZ4!V zfWvLN{1{j>Fh{rw3r`0jo~5~-4<@#xH;-xXc76Ye(jWjzB z%q#Ta{5UKd;Z*F=d@>NZIAu@_59SL2@d|i@1SRYLDHB6}@e02GTP3>x^ZJTd5-zPH zQLTM&WD7q+u6W(2l9Ofk=nF7wME}sLCJQ^9PPoNm- zoQhMmPVNJXh&xGk*4oyFxsUuC_WotfQ4+y$5(0R^2k4l8Dx89)6t+vN;sm$b|XamxU*P|;8 zkA`x8vZer7_t4{FJx3sEAc3~U!^R@i6&3QM1>02Nn#cTB{K-&AV(fZ$1)+fEA?TGL zNJygXX406gG=rZZm7-tsmu4J%P;jCaQZetXz2ppxt_t zZS}oyGrdD4G0S%#R2sceh19haaKJERgxI@~punng?}F8F>Q&PuzlPMQr#~7wCQj?B&>`*D%C$>W^FQ) zhr_SK2|>_tHEYyEkuus!eP5ZnbrsxV6mW&>Kp2hBH1G!1mJ8Lwh@Z}s&er7RkstD# zX|wxI#J8cBXd&PJo0?L{vf=@S>+mRCFK!Vd6CA;{c23T_z}d$-^^nA73#@TsuDW_G zs8tAh720u>7dH6#e+6LVe$o17G^1&)4e`07HZ8q+a;@60;pD;RJ#Jsau=ko7P04B# zIxduSBAIu$B&yC*Uq(G$y$K?d7lrtBW)>^s@H8H~<%V_^bg3WGh0QsR$UqCfRap6J zPu{+iwb&9a*ZWG*O>H`HgW~VnIk@}Rhjj!?ZmK3+FPp#r_ePT!`xC&r zkJB0?*ndQ|iJ{$@4=_9m`xIaEd`(Z^tsv@BB2*VJ;`T{yaDE;Z-p$9{etf7d+0#}h z;co2WNQ5287p9Hg(BTrCrV^BcntbFn+p~te*$K4R=cf>uehm7s)$9u z`r0I|ndaYApRT={DG2`w#QwKBwvUs*D>~l2t8}+ER9J%qu9;AB{KsrVrg9xm?sWa- zy`E)JU4T4-Y=J6IFE6DIX=%TQl)6Ic%a>|3wX7{^+uy0{T>iq8o)6a;7+fwI!&eKZ zXZa!}zuxyZUh?UZYBxY~cd?)TR2eSori^M6R(4E+8F3`^U=Iw=SW7T9_{}t*AkbKC zi}_QWzQb`zNSom`_`#BILxSr}g8`CCtsB`@f0(bvF7%}?aTQz-8RFM7*~RQfp|*A> zkfH>Ah|bIIpsS>hzX&yIaNhzAdb|7tZh$8=W=l4xLj9ZbbJ(z+`}v#$)0WlZ$a|zI zd#c}yn8rl`tSvc?lVnF`kcFpaiYYu~4{xj?$I#tO1KPhxdLnkQ5QwNByxy73mKKuT z{q0I{O)4LAQVCCY?Yh@5cvMQF->t*Z#^ufwu}=K?(MEs$`t+37kmO60es$Wu6@|xc zXxY3gf@C5jFQD~zD8whbdT;Hs%VOGfngZcF{Z^5>#q! zDaF*{0at-T6FfOGS`a?)?{MsCS6>Kh0eT~hppt4A5B*V&CY{>$ptI9!311gY<%60G zyN2v7F2i(&u!bJAwthcOSj~%4HFYIt+GQ>ZmZ$T>bGiWp&$X@lB6LcaV_7c1wz8gp z`R}59{#6BDQ_0JwHN}~p<%D40oq0jm)DMg29TJP~ zJouY1tQ4x=Tu^-8+S;!>!IQT9AGW5!*Tpo`Ree%F9>Fk{P7;=u!xZknjg7BqG20|p z?DV(X+?$;eWYP}tZW24LU~4_hCM@_tWZ4T5Oxd=l8Gi7~(Y7a?5`tGlF+scH{aY*& z?1w^H)8EeNpQOOUWI&=86t7gJ=QC0}vn=s}Yspc#)AaE`I@loI$EXr0o$Xt}jw_C$ zCj&dl&O1=%W*K#80`2>)D^Yur9lbRctxDN z@SBL!8eV1P3+81$E3MSm9 zK+klyyX(i`(qKW;MmZDE))f{eZJV&m(lPR-L()lRwO^XdocH|R6g&F3ATh>K>zccU zj)%+bl^&4nnd4yVUDld3nm>dkGtG4gRLS8qwcQd1(EXc60kN+2PM0Wo64&u&)hqFe zH)&maZ1+X>ThK}fB$fukFXrHdS~ZiD)|^vsxp4ALv*XVlm(T024^%=j)LzNa*K$>91*`cR-+PEW%>=TRqV zHa85n*R+e^q#UVTR64WO!{JaQE|a-N_Odq>sycgaSG;w^uBgav&bxzI=dOBiaHLa?fiOf74`$pw^ZJS94NlAFqu-N_vz!!mxFtn;T;@*NWEyV>jkjoF=fA z2CFc+K|HC&Y88J*$%>lR5AD1^qr3A-s_!5|tn8 zU;RFQc}||yAo-uQNAUC-vVFSim;avvk6X2cG|()*)@O2p=Zo| z2a#h!=b33E(?6hAO&XSi<<3RjVSx|Yr}ZNpd&=9FPjb5+joxo%paM}~TNY~d#i#o3 zKHkfdDra8N^jU5tp!T(NKm zg!O=bwU$}zRJesQR1xwhXtvXr3>wRSA@q#JRdrQ>T*!Jf^^7i#pgI25=b9%dC-c?f z`XuDWGYswgqiDcFQ7LStD#vfg(UYznOV`=eXjgUxU(%au8FTgCL^SI6x}hKMfgW|Uz(TAl$aY}*Vwc?Ok~X0=aq#5%wPB(9C;Z`9U!j# z&CJ&X2-n0_?)`rK#ffyOdvB69aj5XP`Odj7OFf-h7U#nF zrzqm&G$vM#Svtczdryz_^0A3-V57h4ccV;&tx7-53=g8Cn;)XX*D#T&AVimc{5V9H zKQZ@)Y5bS~ekLkuN@$v!W@(V1Yqc9;l~Aj2YZzo{xTTbkJ*Zkg|7v2viwhXVPAn>? z03*=?TAx153Syb4^3-!p2!luVe^f5>S--6mXz3DdZQ$0qD8siOsdyetoOdJ`L8Y!ixz(in8O^H9QS1Gduap|DP`}wSvqSM%m zaJK`r>$VawbMj*I{HJkK!nlQ}eqC~qz`Txt*~TyQD0SQ~#su(mJwHr{qAmO^I6 zhatG0hjI5Ff^Mjpl7v6|sRK60r4np`69yDVYfM>i4Gv!Icce3$EMrzi!VVWBQ7*JE z;e2S6UsDb-stSY#UZET&9f%ggFO2zN3`6|QbOJyDl~_IcSNbAClGN$+ESX=k>xk7m z`a{DDQKdVUFKxd0@fxXk2@r7q5~8k-hTpP!0BC&dfidy1l+}M%_{}urXy2J6qbXZ? zolmj_Zf*^s*RqybE6Gwo8?0b1HPm!htc)$jDkP+9g05UiLvz!iV*CSp!j<7BFiD~< zCyt^9;~M3{qsXagMxDvbpJXj#U~@==R`0^m;EkRx#Zlp>i>MOVkn7!2vb0ossc+x) zMHIw3^iNc22-m^V)PLi}hJGJxV{t?6yv(*=g+q_o{PQCT^`Bgq3a4A-U!ep(c-D|B zE2-kACn0Ovk1lgSt>1E5CAbQnWi7MiIYWaR=?r;=Geeg;u5J{eru!6BT$t`a(euZV zPre*Z=1qoN3Tt}G`upOf$)aU)ur^7{LzAWSlN5|`cLLAsV19D5pN}>H3(n#P#rHdV zXP1uhVz_CpQiWi5FWDu4At_s#c3NI4v7|Rx~8f-(syeQ3*VZ1q>wn2s$JC;ox0mYw49TT zPCN4_`d(_Uwe>!>^)@A5!<)LSKS5Ktg6pXS2f5$=KNB&~LiZD4@XbFbOM9Xzp&J(j z9wBkI>HL;Kz>TYPoB&DFY8=>Z=MgH>Fnt;>DSVfAfBZhTj~WH&T>`p~)}I>io{fB) zy!ko`7FRlQecQ-Rr9WLVY~~x4`+VUi*xe$x3YM0A60_lJ{`u~Do}bsy+u03h_#FQ0 z-f@HJ!vTES45Kg2WA#k=uDxb(iV6eNmi|T;vP%IfvNE87ptfj~PXfQ66SqI$5{>=B z6NjS3cL9ggr;?~xNhxgl=6}9NCnhE)2!7wves0eR`nW58HGVyvKbF(`c)lMl=YHRw zCol|r{U5qg<`nvPzdtzg;WUJ8RW96_2=(PjdxHEm9%O)x=kJnzNIg!pAu5%>UeUlx9+LsKxHp9L0 zW5*5^sM~f`LS3Y9(2NkBbzh=+zmEWoPgL9t#RpR~K4C;L@T{K-0MUI^8mNSkSf@kD zHVOOj>MH? z|G!abKR>#6!~xM81k8lnybZ)ImUKoxeByLCZ~;tRO80=O!4o*^-XFgy0t4l$w<9eg z$JQ1I5j+fC!G-nF*RT*es%#7_^lXNJVA+cHGPP^Q8Ft;R_HedVo;Gf_#AI+SK;bkm z#nsir!5Bj=PTvjd)5Wu9M5Smjscv!V43Q3w#c--nkQi1=o!(Lb;)Y(;$$V|zisIrS z)Iw8RH&2b3`Uc)SGIRCx8IP(u#6|0hk#>|BpiVCF{zPgO6w((1q?_J%_Dn!#&8)2l zj;68X$ht$2QN!xH+*XY+@&|@m6FzSu*dY!nL*=;E^f-8Z;ad4&Unz#RzT^gY&_`bI zu=_mTzRD{5YTvm?FJof)@iO~g>@Imzre9qLVkOBu4g}hS?Xv$sI#5bI>uG95mW%76 zt%_{As#Ky?WTY7}`(5kf zJaS+BBXc1dePQ67G*Y`$(2N=LEP4Bs-bh@hoQ$Ba$luIS7nX!Ch(SN>TRwtucL z7auso)4IKJ;vkek*)>QDo-i6(_B8m%CMEy!6h)l&~@I>OyJ zi)@`*hND4wS@GGkxH6}3(nj*D&+ty9D)NXQNEXd_wl(P?cJ@MzsI-Gc3wY?Ztirkl z3s4FjibV^gM=xrOV@$^{6sHd*)8JHeF0)T^d@G-SnRqyKnBCQgHb5eUS0oEEs%|_B z3$!LN%F%q+Yxin&26U_v+$2)_1_lfNIz zju}SpV0UGoK(~z?oEj499Ru2ROs}>8g4^$g*1A5NU9q!~kqDj%u_xrO9c`yQ`m-N# zZRDMg-6L*1SYr%@M!`k0aUGc1tJdt%FHDC)YGU$(Zx#GsCamdZe`MmL(w8_F?S2px z)(A*@RFMlFQlPG}>wdQOQ;NtiCq(yh1gk}(>;wm~kGFKx63CQ- z$!nIOgZJITv>k993zxs6n zDxjCk6jP|c?c;e8CYOAosy?30+JAV|ydzf*BKWQTZ1$~fz$c_6V&^7f8?+FWnv0Yh z4;8g0Jd)8A)HW_=r`4@^(dDWbO!pzuM&70WzQF=mnR?T}Fp#qlMPNVn)OoA1R#XEt zP_8L2_&;KNwj2dB%&5NZKGYE{S?0NX-fm7-T6CY#v%S|vt%m?VRug$$_J(j$vkj>p zz*WYO4o=X2)mt1ZC6=XU9Ll$hrR=H2ovZ-Qn3xF`aIvRGpB#*NvkrX7hg8KYvp!F9 zKD?EOrOb+WED)6}in*<5eov|tv^c?A-a`)1K0;j|YPLBM{))TuLXOtJge62kT);FC zng?HKC?T9G*;#$Hsyb>^tDCI6bM|5bvwiXrSVR@gdXiQ(&>G9>n4h129F!d+Po5D67{^DLW=pk%6k;O(1h>sL)66mZDL|& z-JEx7*)DD;ZgG` zrC#2}%$GS8J8j6E%a*q>*xDu854`F93vQv?{WMC#Ru!x|S7bWS(VWL=0v3^seaA7Y zIM@{XQMHo&K|lIxsPDTcE$!fxu&jE^N#W+kJk!b2v2$skXLdQDskzcqqLs$Ai)xK? z+uN_mYRfXP$sb!U>;aoao0C2G&Q(^Nvm)kUAvB6c&EvJ|ZW4Nem47f@Seva=&a4ri zMjXs1_Uq>OFf}lsKyfJW{S`6vyw7nw=h2eCxG@*AF*mg_*ZAY(^)|C_(KX3y9#RWM zNNo1ViR!HuFr|tBNd$iCWnMWck`XSd!hD!#U!FP>^+Xl)eGqc_$kZwt^^a{xCW}z$ z28^v?ut_k=KQNe7Num`1^(4c#?VAl-MF*t!hMD@<(ev4~e%CfQ{aiy7`yj9rm#b5i z1iJ^kUa!&$gT)hy;D9NxN%6V&wpXqnYk1)gTWA#<~uKX%4Li%}P>0F3Xjinrzq=lNZ=X97fL zoCHK~Tmr;kdS{FAL2M%|1Z4h$STQg6^o^4B6k`Gu`|>`CQSBF zslrUSSbnb&o8J&Ly?btjn$-8(Z20fc!na_hclCR;U}_ExS0xo;p2=i!N+@CRnLc#R z8%+pFx>U(Iexh`;Ja+S);lt-pkE(bpn2nTX{tcGKvX1-TnJ{CS6O(bGf3VatfkYRq zuQffaI7AfQ`4fIX=D{v%P{t}iM4&aa$_rbQbMmi^Ny$f;w(NYZ_ZvMShS|-w)i+QJ z*l;rS#HDMJzdp*X+y5>_0d`3 zQs9k5)827$*VM%^P}LtukU;4c7DW{MgK3o8V8@IsK#k#TwLe8Ya1#A%as{O)I`Y={ z`O5Wq0vlUioYairLU+Xtlgp;~AxwDClru)poZakFB`Z~-v$b;!^orum=41)BEx2Bx z+Lr`&f}~pE^{ipwW`knwULGX_-=Sl_yg0uB4gswKIkCS_FQ>a66$bWC(@GC)e1au~ znCu;6Zh@z4D%F_5gpK2>Xlv4B|Fn~df7q!MTd;YG8zqa`2!~iSEAQ8rdTsbowk!$j z{6Rh(gCF*~sY5yGmxyQaUr%HOw0DLHYV|jwQ%}?t)%!3901^2L{J)1sVDyY69@~D@ ze}nOf%HNF3%2DZQ^|aZu_s_|)(0t}5TGP(QkKzqLq8*N2IcvU%v)`Z3m(bR$F8Ara zy`O!fuj`(PxDr95?MIroFBUv)Uflzn;D7P! zj(0yv$Ud#U+FUOCf_T4M#~0~qfIc}uztT(Pl=2xHHOgJI+s;<0ueqY+-CrY_OrXwj zgg@|G14JYZPTTOQ64#Ceb&Z3*0R=zcy2J@RT1iTCA(X^9i<$LHWYpQ7T?DoI{H{%e zUA+7_^T|Sw0ILVBXg9_ZM`wf3Tsr4A<7r98Dgsg8;+41t{j<0{q*dF&a1NMq%6T?G zY7ePxExN*jmE+3JNLW!*juOdTU|BjAs(NGgA=@7>M9SBlRR6XkzZMm{zfEL~Gicy1MHe7R|4Q}{}6TZPacsd2Q z+8ZTHtH)KDV_j3ZY%6M~hxGIseOt07?6mle?>wtdm5O0hho1SSVZ}7ce`van{w{u@ z=*R87+WgU5tB&*OTUir2Aq>qZPx)*)k9>Q2TomvN%Uj13bV1G+6X3QabBItL^Y>Er~2qUIxe|# z1MML_&uOHVzb5eI#+K^q3Y*jh%|;suEV|w4Rh#No=l{GGczdb|f1GW1j78jlcb(gp zoZ&2{4|0ygy2KTk*1X;)GPjMy!Z99g-GgT`5t&{?WI4h4udxru!ZVrtXoP1p#NF#8 zV1-0yCE1NT*wbG>K4S(Y_7H6~;=qA$xJr2qt^L^4nDJKyCs-+3Di|op7qKQarMmzJBZd{pMN*T)8 zPv(&zs-WpDxMQw7lv2*dY}JNEt}e1rd!X{d3DZqx<rrfkHj-(=`7p`7j;%vo;~%aL;0H`9y6JqXQiMIndq@7`iO zXYwd>{#V}%myCI_+sNf^svdnqDsLeol$se!v+Q!o6lOkub8{9l3W*`iUXjP|-l7Kr zF9wu_RvP2eD?6^w0C*#OSpMIb%oF8Xc5k}L9MTl$9?Q{p9;nbtK5i#GS_<|Zn^7+u zYS{))NPjzZ`PbjC?+MhHJ^`o2QsjMBw($n0?-%cpc?9&oj=FV{1uTewhUsPOo7?Kv zTPlj=xYQ`a%X;7QvQt+v%|#r8T>}{hJ-6fLmGtUd_(trsTC?LuAB{u&OJTI?3;%8o}+beI1Q4ma#r^$)d1_`!} zfH*)6oit>fS)MsTOWN-^uDZ3xmFm317(eTx==WG?V{dzJG2oc25Fv4#{9=&UuNla< zu-=;D%h}sh&8sleT&0>)vZOsWuBIj!ek zPlnXI^u_DObTei&WcxcV$8wiN)uF^8z)hj$HhJhL{_93 zbK>n}4Q|dsd}1wYvGS!%IE6JYWy#A4ScdiNnGK@}6HlQzM#{9#)4wz2KX1@Vo)xuM zNhVu2M{AAR29I_=+}sS@{aouvkf#f*Wc#+vg=~vZtq_b&mHUad{DfSJ+dXbRmVh<% zex?%$RXFb;e(Becyl2vW=3MBmKifp|k6Gl8Y-;z;Asyl~wVzhjp0MNoO)DxA+TF!D zXJQo6UR}tt7Q>c46LdkARFcwpT0~jo#0hCV;51CP;oQx%UN|gy)c#QY0ys*IN(1ei zku83NOQu34m9+cC4SPs5lf3$m+enS&6>HCP9Mo=`_NGNXtmCHac4uahvIrQ#3@SDA zgXsqFHc!OV8-cxhhdV~Ueix3g63+sZ;wutST*FDt9;;!0FC8>}eDK=CWyt=O)emAd z)U;p`V=<;66!^A|&?U@o2g?}8?XGOx9RaHIaoOx8)7NMhQea<(>+@=4v>vD=x4c0D ze3;jb>+$MAZ8-+FmhZ>-ONG!FHa?Cw{)wxn+t!R64JF%OCzBL!W?xJwcaEefR#^~` ztZ84KnZUMQv)bsI8;{5B@&F&D*xlhow!nMAVbSNi_JX^gGy^<{(J(yauN?ee_v1{1 z!ji|LB=>(ut%I=A|Bgo{rm*Ro`@H=>r0ezQX=;(7@8h6qH`o9D`843`<0yuKU*LUv z;*=tQ_xr-*%=FJLLj3<9cI9U%{k)Q3o*D>?2o$Vbxe1n*!q zqUgpgl@SJ}ORz#%a;zy2!!saeu;#qfX|>+t=*m2$N!hp_yu?vxZKlU)A@$T;#Lozl zn`Eld6_T9bI##dpXPETZ0y<{&M%_3|9xX4Qa0KU0ih&tMCcg&1yv!!4g-0DT55`;u zXXzRIzYaR@FKgBUAJ(jKJKi^W4{hi?)N)$%8tS&oAz*F2uby+JRgNKr5Lo+2D)hw9 z|52xzt>weN%vc!q0r7JI2P6XuY`f+{o6wM8pp);oihgO0cofi>!E4wJK`t0=r&Ms) ziD7L>{|_!xk<$S|I8o)2PQE*CG*qog2%?;~DqMa#Os$r5r#<(L6>BVrCapv~!aQ|_ zQ?Vfb^~?09roCP-FIpz58&G@B3GA(3@m5u-uZnXwqSu(Kb4dJT#T`Al8R}h~##%-? zgDj>nHV$aDU(gj_DkMFb3;DB@bBu91aL20d~@k1U*XC;oIw_N zXvaT~j^YXBve@!>mzZ)kZoBD|3x<%dq4}t=ZLpEScK_{`?cFJ!UzThXrY7!uu~w1| zqR^<6x7#em%ExDD@DEs}TdKk}0mPRKR<5m$+RG5-0e9IkA+!7Z&a%hA9r5BxwclAK zS*6NzrbruZSWi4>Q)4MiXq}3-l0Qy!W<-W!Opz^76m?|5fyEzpi=@DBs<}jlA(Q$+ z0E#XjhA>=4rwj!8D5Ulos@wGN#sY~4DfLf8N)@Z#L-JamvKgOiHLvB2#Y*CWsa~U& zVjtBGewT2LRKx{v3#|$f)dYiOyX-?}g!(ZwQgJ@nEx^270stR*Yc^`x$qf>!Koz1g z^lB}q{3+Q+?jMV#hL^zwTxXZcL|U}Zr#Dc!45_@ruEOMj#9$OL{q9*6v{C9YPU_*7 zOH8#lg`AG2+n_7Rdqv7R+ruR*ppPRSC~2|?35tX1UWHE;XJ`tmk<@gS*sX`weaIGB z$&ntf7qtZ4_vXbF;g7?%2Pk7_&k?qya904j+2b%7#lbK_=2JLBd+ouW9xxBWKU0TK zfx7eWgSjl|!)s^Q8H;u`@Eg|I{v_t0fzsn{vWlX#B{qWJ6^R*F4}nsK$;zuXjX*685K ztoYTy5H{rERd}3zd%Jl43`23#mozM$;Gi)Q9C%c&UAumPP28U3!0E{QZT2y9A*p=O zwlQxaSIVo|Yhb6Td!MDJtG^CtduD$U8H)b7TnB#;h_7jE;bvFYZW?Tz60&!9;q4sq zmv8M`>^zaZ0%0AyA*_IG;o|V0p=-RhO)^k9#HZ7tFz0LSp+WicYnX4=5AH39gK@4| zI}4>kgC&r+jBj|IXcHiCZusIQ8W&{OvP=wu!1}M0yTLKf6N17sBVwBt+#NDi`7)$g#)4O+GhNq> z&bC|exqN%J)-(_3Ed(dXh_(YS=EMO|GiIQ<@$&O>LhD=56<REjKzofZBb-db8` zd2Tn>5~6VWF2Lo!P=7Ip20sJ*M#XcOTa=?wF8gLm?86G-S{-weH4ojy8(4Y^r`5MC z;tBkm*XZfx2N#eXkVm47mvCc-HWvSA+r9UpA!J`TPKf+1L5Tb`znHq#=s!;|Oc z8dm)@Bw#r}2)Le0!9zND!s}uph-vgZG=U|ry?GQ+aqd$MtmP0lEMwv&ApCn2iz{Rw zM2=!e#?j*8l7~M&6t8(l6jADz5P_RT6hTfOVZLvd1CM#d?UWrGf1R3zEw%u5y9-(y zGNYUQnxRy*uuP0bil=e5|51md!6wC!ZS&xaP=u6izpVJqL1ApGm{f$-Lgt)BTi8Anq$BTC$z?zV5Uw z7Z{-wg-i=yk$^<vV}vy4(?0FO471!tksl zPEyJF(hT#&Bwm;cr(9mA1S6&qas`guvJ!t~RwUeIZPT?@shCii%*SC|1Ys6&Ur8}9 zs5Y7BSs`JYJ9*u3Nc7xdRLH+f=R@DC1>3Y(*)oLYDd{$Sjby;!VxUr^Hh%^EF!0RY zHYQI_+#Hh$bsfL@JtJ1$HUag_j8phiO0u-;8|tpI-U#)|E{3v#e&LIG<1bl8x<}-S^|jzLSyrwk97|H3hqxpk)`=5IbeO9X+1-#OA-|sd2Fq^f@yP^T;P`U4@mnH zh!-b<3ei#Y8a0KFa;zk}2YkDZLjve_;Kv(z zoFB|!{c8U9@F^v(00w8t47OE43JntQ7ZgKz57w<50wb|~3#cCzFI}-xFeInUNphu} zWgd!v@^F4~z7mG#0P1!-IC9jx!gXd52_T6Yd(PluPEto4WG$_K34Q&0&mgp4w+&?IIY8NTWd+{<74Dg=frtk zYqB(N5TT3h>RBCJi~M~bmo^d|(}I6yoW{H`T3e`&0ItKLK4*yC57JDVj--nX`spO> z<)SwH^HC2FE49p18^|&D&TK%Owk37EnptHTo-ls==uam+6b1orOKRerlD&uhBWj__ zXf*B`z?42W>Cd{-3W0kl$g&|z5@GHZ|KTMy$NtKG(uL-fe3nxSRi*hB#QOw`r%oD= zV$dpv!f?gja7?MJdj~=<76rw$KLAERxxdGj_e!8&jsyZ?t4u{q8Nm1eFx>NEUrF7e zd3a>{%kLxM+q>#b@%fhJs=nu& zsq53&J8!k<%@BX!_rbC^-Lfy{zDyJ+hs2M5;pB5WtPNlDTVJ`wXN;I%9@n@IQ?#lG zB4~7!K)p9P(ra`CpZyRf#4yAuVzW2Nb52WVdSB!K8d7dLP6!1yRtN<)K?nthwc)=4 zV6rsEF_|A`A1#RbzO*v_wmyz1+QP2X%-N_v&8ZjU*VH-EPMusrUq6Y`h8?uk=n+P; z5vuPJYZhxP#t)?fFuq0_`sAFo~looLv zOp1zD+KCl{5Ad@}(;dGUs6t<5@aQ{b{q0t4{nAR1 z4#GnW7r${TC*@Q{-ZHANEGNaZ z0>-W+Sz}U_z^o)02F0o*dDGzhZv(fC6l_u+&!{R$l z`Z9#iGV0S!brt(sTWJBLB};#;hMdr!rN)gtMHR<#D$XTTxY)wdJ;jN?*dk;xg~=m| z6UODHOsb7{35CgGd-sLOBP*EHT5xH!on!xZ0+*GE147Pre)#{$v;c%0?cG~JlwI6^ z|NDk%2^H5GX61hesz=AvROTNga*zu1pz}N?_~dki9<62=EL`>1s0|=^;AYq*tZ($E zLd(EPf=@dNn>k>o+q;9geHjb?VJ2I!4xr!hfG6~zeRQhJ^mGW&FuI1aS$5*fJ_2^d4iC>ry|CjMeGjoZ}*1Av3>#9Qv-)-A+{|SoideJ zMZ9BdR8FRgI$?i8n=LW%FS^I{=L6O`Z%15*%I)j642lw)DtWwu3G5+uG(HW)W3UuA zp0~a%$$|nNMym=d`g4~CScd@>S>%eyGISAS$?BM|5w;G_aAHqqZDE6LqnM}8Y-K>8 z@|7W-1ecPO*8GQ@v1~IWobI^9ym>(`4cgF)HiNSw4m@{J$J#b4o=A5$sr!Jn%B;!P zo79^5D2SakV=3$uP7iK?+(H;8N~HipM_|$f2vhFx_4dt4SCFL?koO|C>Enjz%ICJ9 zwmCG&AFSNnAnRreuuTb{Wp1~o@2M~G9#-% z16LT$qMLh58k*`4$387lNAj4j*l=`m*C7wRJymwXvmY*wI$1(cI&J{W@44o~OtytX z-rpwHmJ)MVDRVLNDo%OC+4BTZOT9;^ZDjy{VQ}fLErIcH@7lYwUh^m*wO#*H(z^;Qx2rD=y4xJR@us5R$I02 zZb5joXCnkzT0Pez?sOn+B(P1!%zN(sPznVDEg41T^h=BSct0hh%F1+*A79=$VAb_ zS_njkVyv?e`rMw6eTTdf1O#pwS+%M+4D%XwU*`}JS-|@(C>PF@9JAPZPWyt!&tpX-)wz`C07jnkA;QW>WzT$HnCS$uCR3Th^WAVp^mS(x;z zMYajblubewd9)E-EbekhMlg>krJ+wL9sX_MmfgrG6u1E4Jqp#kG0z`>0+Ea!R^kn( z&d22}CP&zp>zJn!B;#wYaBe<)a7L}+Tb{tA~n0f*xF*GTeyB#w)Vq2dZe5j^`Jo~nc6g+E|?-2 zoX2VU*z6qE{X9FtWk(>_5@^y)XS2nufUl|ClG>HSn}1qkSNHGsaenwDlYr$C1g{W* zVLcx$k)uK^1IrcY{;xLLjtWr}w#$4=ko2%pko?@1Y9SuW#a=8{>bVFk7s20KC8$9a zCDg;R_o=m7>9$w(H0Q36jK&=rOna<(3y$UVJ*U=hc;;MamCMis|Gr}m3Wo9Hg`b^#L%&f#38E23;jpN;@-xP z5-?)n&^)#gMBo)gh}Ihtz;*fzGxLCm8EZM2oVLr@_qo%99t8f`#|n#J;0W4$LsUwP z4U9iznKj6Egk3rC-=saFD$5b@b!_MX>TEyx6*UtyI2tQDY?~9dCL9{B%Pl|{vB+mB zph!7DOV)?x>umB54@D zz6D6f@Z{QZ3txqZdk~cAT9W47-Mp)#$5 zCcPG$8wzg8H@WOdWY6AW6Zc(*2VPo7e3TH>xhPW`(q$>>Sa{$B?V4~P4uxB9-`PM=?9 z-tQA~H5&Vwem~ClNS;4Vr166T_Iy#bCA z20V~4`)!Z;2$>k2Edx%EbMrxK>>*Q_cYt2~$a}y~n5kMn0rE@gIHx>xOf}ryB)j;h zvgUcjX)4A>C~k&rcg%Y)+xPBy%{0PEm;sP~vyMa&i3^Bqe1oZTm>%ZRTx-PGBuP-v zprZGL_T%?-S0QCJG~e5httnfREKjvu4`kMIRP8<)y@TyL0@j*h=iWpjfqY z`kO}BIDhkeNE~54g<)3enkMrElm}B*6A5j&(MiZ@TvP9Wy8&EVHJq!-RlBBtiwR>;cJn!O+An~BO)Fye zFiaaD7Sz*z4d{OGN)#f{Vx!e~?ZYyFCf~%?s2C403S5j&WK)qvOhqeo@}gt~gfKNz zFBZm+-#xy_sc&uEv@Qh4)6HO_?*T~Lcft{SeR59Y_t(%ArA#|tp zoLwzT`s^Nc;OiIx5ONF4$)Is4x2MOs7QMB>w;|Ebr*Ttj4{w4KuapByDvLheX2la0 zFeO(dwO2yYX85IL!W1+~DhH*vh{sCDJztc_VoDG2!ml;49DGa6v=@_J$iRNLJrmO5> zM-&hnOK8?9N8fV=qL;lak{k}az7*fc}JzP3W_am>@`HC?aOD0OQQ?UsynC4a+P8GdEDxzpdDk+mFGPVG-$DBe) z#HIF`%2EBGB?pTbLYSdG2hP(M5Q(TbMci{@QEbjYb{uG1)$p~SfEbai30K6k%~=Ym zw1IJjdlbk>;i?1b`sIiiwnE+bTA~4aXd>7L}(+4u`=F+51lcHeN z&9%@~dk$pc$bdQJ7e-D%eqzzRPL+9PR!#M)NuYQa^1YGbCfcEusQ%*iQJBJQcl}uK zM2ci@uzw((2~qWnLUWCnK}YjkbL{1gVxqgzN!Znw27RYDZ$G=uUQEflmy;BI4HM%C zmbMpGH5i~5DG)Bauk^incljeXuj$Qf)yki`)Y-=BNPFK{H#&l1xaZpJG0)C>vEMR2 zh|wHpiHES-gRxrM@<|eijOsWDI!TO9q~W)6Opg_hx9cA}5OHsp`PVDmE0m?dvIS~1 zOGydl>NNL zoCJ}I@Aq=_u^_Ze{C-Z$qHbVr;a6jxOO?I)o5_G__4bl6@7WS z5V*aLuJ3anZAxrZc7n2au#vl;u!(QeDzZ5a13TPf^nn+q&{$km6^(Ax{Mc`ozB43^r@!0=zgUgR>q^+0IwkH?Rs{?i}9|&UX`wz^o@d| zUzBRM_Cq!Ih+GsHG90Ctde{u*nI|2@{v<y}Vi_@Z!H2xBO~8k|$Bu#zcG$s&ygB(uwx$g{>f;CQ z`Ti4g^64D%VeIl74SC-qn)miC1NrFK{E%wES{lLb_^d|Eu;^)`mtgWVuzw}~~BB-}*PY-R>499<{OW*fba3$sngejUMJoG|x~X{Y_ws?9GR zZ+iLKp#e#R&hxy}IP&W)ilK`x6;rRQN77A|6VaSZ5QzCsVqlLg1tct-M=QgWw1t}l zsxL5?O(Y2sAV(>O3kmRJq%_x1JK%lEO`&824VpyH2X|*BnVlBJlNavq?c%9#_co6U zQEEHHyar%{w=}yJZ^zzZ1S+Of>>#tzj-y93^nlWpWgSks(FK#Y5^JCW5p=G36FoyHW2Rz9(3y?n7B!Gg~5>!l43NKTPezrL+ayMjj;Q*&w99cK#~>>Nz}O6ER%_+0ETz2s=yXw7i>I0XhyB9){76mh|BF7E)}zmezY8Uy1|>n`Gd>puxsdP#!fC*^8AP~o4td} z{*%MTt>(0~qq(&7Z(&HXnI3$mSk*LUvsfcs2HsqzhL~&=vRNNo=2>kFLPpW8(~qrF z@3mt$4hreSqQ|WR&j`mW1NK(%wVh*3O20hHF#{xhXRYvYKvFnGQPlA@#>B%{ z7HL+AK6Q`cNcpgcrCv+68FRSfpC8^4O#oZKawa0JH_%ylrm2lOD~jfVGrgEVi@WQZ zh}kP|%-uWgA+sn3SHX1dn4v67tqp`9Nh?L|9uKc$fS)cfKn6|8cD(KgkavdR2v8=f z=2C|d4BbOo>Qa@?I^}fvu-~+jN%15*+aC1O51vfQRV$@fYTar=?CI$?eOUN3dwiLl zXa}M64E1!JcPGJL=twq%`jNdtkEef-0Il%8b|A7-Co;N|I&f*|zgMaB!9ZabF$t`a z7}YYcW9sQB8tq~t_TIN@9BgDRxcGN^k<43wu==zO)go!Sjspa}h_-&KS~H#qsw9R` z2jT;v#m-$M*#%*>3@sEfUm^g{QBk4t4J=v*wbkp9^VKzNZTP1T$S$%r&R)X*>*BdhWcU_L8n z$CF!=+TTWNpxCmjM~K4M6m*JZPC84#Bb(`QoL~#X$dUocC)lHXBT5xMxQPKCdy=l& zii^_{`E|ktz@rFZ2c>}ut#Fw|Cq;^sM{YNci-Vc_Ads_B*(*cV9W38cwB|QImWtLV zx(pIMzyh*MpP!QSw=6+yR2oyN#=>cKW&B$M4ee2M#&3mrWlS!vabHYBUc~1ToO!KV6NxMM21=qv^;x zdyJHx0ziZMqObd~SiJotft7t)$Y}Q|6f@{Gu{>~1xIxVPUQa(*!?ezTWf^2s0RM1$ znaFss4K)#Dm(kTzQs>kP5zN#m0Ohtc;j?`&C#JwZSMF+uEGSx|v`FI>POeqs5CyIo zE9`-Yp!I0Ej(QI(cYqAUbs5;RG${koQx-vKqEnC)4N~ZqVe1qJ!l-THs*6;^Q1Ww7 zbc>7I&OR-z?G8J4IyLm3U*gmzg2(sg<2}B=Nn&t*DZt-J{W|%^U9p{;YEFIMM4sV6 zx0JYZAbj1B{w%e4p)J%>cdO`Lww_g02;GnzX*6WoemYmUt#I=C?^p42kRQyQV*X&H zr&fKdNcWbpOYgEmv-qEn>b{dtS{Fx1G*l}PBavHFM6Hg-ueNlhi%(&1J^^H5e@18= zgP{pL&k(?b(|}fwAKdiI)?S>MuA!Cow5%i$0=C;cG_;Z`1pT5E;humxLf<7G-+%SgPcA>$p+zt zWwaS|vKIR;H$bXDdI_r%7VyXhmNOM;^cRUIxr$$iXTn9F2o`*znQ)1x z|1N*FSVXem6HkRpJjXEpN8+*7$P4_H>=yIx0onD&DfdGosy8`5-PYRmU~4YbFM)_ zyr%rD5AOdvNPjqB2M6cRjrn_i!iW3i3&i*Hf1f#^zs2|G;NjtTzxYz$+tuOw%f$PA zM5az_Kh^KY`4-9h!+|`$zt5iQS@H_lUmKoDchF+}>Mz5V_JL2wqxT$0iChLkd5xwe z^$LbRQWsP}p-#~%pAIO;hs@Ige?u$NdTpUD;}U`aGW&p~Ley6(6;-2+Tul=GRw7h| z9joZoM(~eFwTw}7`>n(|^;^9@R@@qiqkDqEiXOnl_VL$HLEgciQop07I9+b}{@7^RO#A^yyNj6>}YkrU?yL3JQz^(IAdwih_vfMflhD z@F2G?=gOe9=#TCa%3OP_nm{KgPz&sBA!!nNV(Z#vQX^XD6gy^Ty{~?($18yZz~(@C z9%W`Cmp27s9w7TbbgKyt!ht%#T*<2CEBvjpQJF|F05ijAN)}1w9#FcV`eHiMI8VO+ zEJv@S;9Ztu&ep-At^U*`I2O}ydVE{^n4`fwxXms8>0tn~k&>%+f2$ z(LH3Xi3wzN5&cNX#S5}Mij;c)gP4(zUj&+V{(b#;^RidBm&fbv`heAY8ikRhex;d^ za#&5`N6#<~%fd!SZqQt;C2daGpRC7SR!H(G5J6r}G{70=1_Nlmue?W zzoQNelSBp$tJ#+PEd{xtj|Nhn1KPywRD(9k+%x=`R&t8I+N7=JC|hsNJ(4~D;!LKE z;2o2XJ^fE>>+!w>NkS9es`^&jJz!g{W*8%jvx>%JtoG19Fo8 z9J|Spoa&3uD|8LQJZWsaB&D^=NbPUQKhLQ(LhUKYPW)hm;(_<0t?gEeYZ% z4h594{Kc1%G;>!l%e2lC8~2Di6HGfJlb3@f&5%x z?g7C8+N+n+PRN(dwKo$Ozs4RQ9n{efVh+&=XM#v`IeUo{z6ur`+BI&g!#&Y`3fgID zbLq$~wC;!Oytca)-|8cnqzJw7+i-5UiEhyn$6g6PxJJ6jAu9E+3TTrzn zc=|JL3z}KRj9bD|02V%RLq9)7>M^P5a2I6dnK@X8KU`h$&<#Qk?aWh5R>*-_VX|8l z9ae|)62UyMZ=dEUD-Qg;=M0B8Nj}d2riuVGKG`PE{7iav zfzT0W@Sdi)GeTjZ1X_#Imf@li2q^fVtK6DW6t*JLHHo|}*MW6A-0bQ5G z{w(#}f}Hc}Qi|-UF1{dOxVJ9NK3PIX4NbJb9kfkgmp(j$TXl@Cl%Rq=cXZX)UgsW} zDXFi0_ovd~Ga-6m#T?zUUnvWV>ZG~qtLD?If?0t^qu$KVJBV^=DpQI&p?Gd4T?fBt zxE@uHv836PM$y+sJ;4qzbW=^tR*)(h<;4wmgCxmLC=LkaIwl#~Rnm}CwhW}y&x*HE z%eMGSR}tLse~!IvSm$D#x%-3KZ_^O^T?9FC$0kiW!X5ePc#7*yt9 z;q-vk0RXiG5_Rz}NK_(gKF_K*8WGCN8M0=~Nfr|<5KVCcVYoOdCm^;GZDMZP*=~|| z#%7w~TJdBJrmvfSJ{*GQL^2$@0WXQ8k6a|^wPnv}Yf=Pbl+do$&1h$$h@RcTX zDP54otT?n!GSJeonJOFkszG%;PRMWPsC7IIwZKjxzktVuF|)3uh;-5oqZ|i@VUT8~ zt<)pAwNRxlY{Ya<9#RT%pn6thbcs8Ml$pvA2sa?QO(BUQrbyd~hRiC3q`CAl2=NRh z>qKX)vmCd(pdAA`1>7`gT1o#VfVNjN3R@5q+LRG<>c*4|OH*-;gnqjpa7)5WC{lX1 zY~~dxJSkFewigN&oYs_u3y%vG9u+7$`ND)IW*AYVb}{gEEdLcMIK9Dy3rz?Y9@jFW zNZ~<|w)1kKNQns-ny`ocXZ)Y74@EZN>H}RidSg16IE!FVy?qeAD##)m53?w)fI<3gBYXDn z#=zE%DP^VFNF=}CQ_)DO{q%0K)8lOoJr%;)Yp4s?7B_o4ZP4hAYog;>y`BL(k%qx| zm{MOw5MUz(ArX`O2(12&z&c zN%0W!l7*SY6!hRrTu#N%ZDf%yvy>q6Op3^!G|URm%iHGq*qr zk;8yO^OVzYk_|aYe5Qi-c@nTbzzT0b30}pxD-%k}%tDNCA4(`APKFfIW*S}z3Gk`i zDpWf>++rP+O0fZ?S}H0;gr7^;86vg`=rx_({}W4gEvws2Xj!QE5X$_7%q(Q)1rxAx z!KVTbj+|EYw>!gAVa8riF>k>HH?*YA21)Cl zTlupNbZD?Mg>x=?S|e89Vmb_$nq*_v!m3xk57)JTy&^s~&RO}Gt!rBgnicknOjIu4 zH?Y|zDtQb0H)-w!`&9>mF3WkYq3kj`XoqN5x1%geZS+jbYa03_D1M7=xkO)^?67ob zMl{)5tFo9)aHK-u<<1S+F#4aDHb4Kz%H?8Y)W@ z6&(0p*gE%7Yn1STMdIV>slArnby<{Dl8u{*6P}c;Er^VTNhLN(rp?;LR%U;VhZ@ZU zb5l{Htr(U)2eq^qDN0C|8BeEZHza@#1 z2X-8yMGZn~+j{Amnv%nqVT=cH)mA5nq^+W(S?e+e?Vfz<;A_*=FJ)d&OGpK8tWQQr`0dV z#ddbTp3tmea- zWWs#~Vyn%J!%AUtrZ#S}R7EmwAKrEl3ho zXsm(0`~Zm((y6UBi$iTjhlo?K<(HF2pZ~V(>pBHbFCW(`>PKHaw0WwN3f^<%3&g+m`g7Xk0yt1wGt|D#Tbmmf!ZiLu>A%27=IM86r-{DdFg5 zw-#C!A{bIracCoz%|k&$wSs^I+Q`se`MKR6ZJ@qK2xsb$X|B}uC(xDl_eE`v5dq66 zj3IWjo<9L*ov)UdH`1sC`cih7C47-kljz&$bSs&1&~AR0zd(?%k( zP;xg3I))6FC}>4Xf~y>@S9^Pxm8gVFhtyP=v()R?9!6xSg7K-L^3uzFO)PJf(d$z3V+#jQ_A>TXbzJT-eRR83x`spqE z>8<*uz3fd5*~7JOL43Z7BKM_~(1Yk}^(>HZ1{l)U(w2XWkR0fug_%)Rf!*Df>ULF8 zBdfCM)p{1sWqs7S(%o_C5!Rev<1pQ}))1h-UEtDM!t7`ly-?dg;boe|?}S;e(Z0S%LCi$=|1nc{sV-oCGzdxsAR4;-@cN-r9GTT$6f)=M^pKs^7hG zSUuGFKd&yj3-$H%3>J-Z!Fyga+0)j!@uxqre8>hAi}Mi7i_00a#p(XmLC*9V5E zVpw_1FM1L>ze=rD6#t|7d3b6@XCW3p&w()YTU0uIGlBh;+nPWvQi6`aWC3k)+Q4J0 zAdC|`YTT`8)Lhgsb^88VfXMCd5zv_4@-q1Bvp46v8>782CS7X(N~@7&a$Y3_S3Po^ zjwp7k>+{d^_@?|OH!=2ox`u7JYD4aQEo2%*!hHwgA1opJ<(1-zG(z&ScXJeDFAe+`3TnDQ_~O?6d}h}Ag!vChOh zlFoABA)O@h&J>rm1r6^%uij3>1jI~B&*Xr7)dq>vce-ANQ4#DU4V0P!CUnn;02JMe ziW=aJHyY;eNreEAy^o$7OQ8h~w>$!C4O?3S!#ZSP-X+Sy3Q3TVH!us79NQMED!{Tv zz|=ZzhmhGeCN572o@j}X36^L{01`ajQax@WE5RFDMw>W9%HhHr!nYFh`91xadR9C)QFc zFd6r|9eb6lSh*j!m*hJ2+wSL$9q_S@WX}iP=qs4d_)f(R1hfFJPVeohb5J`)GzG&5 zxAVZPhV;wiUV}+*y58WDF4-myr#7I#n#)SxG(Aq<+AdyZxz_wpSnQpe!VtxZdU zc;qRl>}0e1w;Xa`!)hM7cV53P8OMozZcqR(Ho8$Af=s{#sJ0is;J$=1SyA=5 z77Bz*hb71;Nh6bWzH;E#=PQKVyYz7CSx*lL@x#-_RyIKJE$q@XfO&=bO{Zu9OX)?j&lc`4l@ z(2@b;`;NDoAR%v{L;8i^?sE;k;l$=e&8Q8$BHhsP!C00SzG7%_$K7HftB!(7qD`z< z5k*466YlMG_9&d5!3K@E4v>sipyiQOr#M;o0l{u-DTA4)6|KG%RDfU+xn@BH(I={L z(b}R*SdnCgO~q7aA*6MvbU1#ksYyyU$84o#wuZ~+73MWs2h&HfN&3eW&Tv-y5J6ZJ zIl?2Chn*Yg*%-12`l_W8`$g8~s2FS;&(kS%R-dT5Is)XYgD@gP+H+B|0f8dM@` zl+DSl^D%9zi6OKBiLL}-66DysBPuiQ-{KbO4@9aBnc`SdsfM19HShg z(6QETFPoIt#+y~v(N{N><)?RL?JCEZsq%`nnSy#%baa286Y}|pTF;C8boXt65Qc%O zUPDP=nff|A!SYm|g0p)2x?i`C)590p1T2>zIE6?ItNB>TTooc2Sgt@f#s9SEJ1Rs` zST1vI!BWG@!E*CkDnjeqmI$fd(M4;&2J3H(}=wZH^h$%aoEK%Vc z^1)_{w*kFEP+Eg%L*1()+SXFiIi*6_L)ts5jsgG9tErc02>+Kh#}Q1Uey!8e5w#1Y zet5SQOM{`e%r%sU7-aW+Ez)LO-cqo@0%f9eaMl}&+PSbcj|Itw6LKsz#~cMMXVp<>O)Gf-|1-QPH4;M=%WCF#MDl5KNKAVRbye3`5 zU~+pA?Xdh!{;u#JsF;vP|9^yvQlu;a8HXP+lV9*8c;+o0>Dch;0Pw+&kBH*X#$t3hfb)CLml4&|M@!R~8 zeFW0AP%)*4Ky!r}*&hNl`;VT$U5pd9?}WPGn*1V4Wyv7LD7(8jE9a7NN?k=s;)RALON`8)t$5ab$h;qFg)^)l zZLQU^5uv^AiCglw&gY5>yPdGR4p^7jc4D>f3wwX}Qyo+~F86Pc-}hfb_Hh1=fA2#F z2Ll80d;OoUr;i_V&-dy18jU@yf1fA&UZ-qY0)q&6Cr{hDat?G?A!Ri; z3aXKUu2K75T1pT}2q_IAGc&}JR`lf7<|(3Tb&Tg) z7;dc(dznwzu^aHGe5)7cxoEBQboTQ~ovK2C_gYW>p9ta*OnBWWalxV;=1}IBh?P&9|#cMnD)fD$77+d;EVy57K_#x;$nxNC5j0rm&n%Nr3dQ#E#Q3bi!qlIKf)Ez6-5szXnaU zT=jUnC`q?v~hI%F^@!33UKwZ^Vnd= zr!ZDC_=RgGqQi(?8NH{MixR#&$DMdO#sCCd!t$+PM5sz-M^?43tmv@+avi4B=;0wq z@5Jil0hKbx*DR^v@dP(8xvie&2reYma@VMOM9NCjthEnUs40Yl#&eh@vvA`lH!yIe zUKL=CIJ9OH9@zEtntkmkj_q@Os})^dLjj-?xm;ycMFAyDsHK!_wIDPgc8le4X%-U^Sm zY>q|YNQ|B59Cv!ox$3tV6gX2#jF5>EPhP4`4bryn5!|bNVc5^je^$&vb{RYWPh`k+ zP%I{XnvLZGWt;M718a?!SqGNaSN~G@zZ1h0tvI;;P_L8!i!sc&zJD~f)*}!zqAmHf zc#0uYF|{r*jzBLn3?n-sg9qpBFP{_}77;vQb}`DD2bLAqD2k~Ye+b%_p^ahAab)`? zN91v{hc?@ba})u&7JOM)!oowMXU~s82xV>kT2)`0jWOel`TrM(d2VTD&xQ2RbivF! zd9>-1EZnR*xq|liyAEXH$$>fL=Z8)~zoXC=VFBeEUDMX7B8B9i&hmqdA8vwDCwquF zNQ4JcNI2Ji5ug8W>Tu_&baPO(`BTh&sidz#LOj8e(nM$|Q!}9*U(EUWX|zc1S900R zpV7LRD|fB8iPLM$U!-(;SRO5ach}}f*OuZs%Zd++|4Hw+lP?7yNUvxROWXR~aLFt` zn;K5S?$Y2Um+s{MY7S+)VVe^jk*3np?^^*cWK84sjo9Lc>Xdnd>-#@1qwC4cvfz-+ z&JV9&Hny(t=LknAPF3)63PA)16_*Pwg+taZK_h71+NexyNHWu~vOc7+YTsh!8bwTF zxU76^{MmyzDoFZICv4?EZZIc6C2hurpQM2Wqo?clwpAK)2Yv>!68l`F?%Y>R3P!H9 zld74Bt)r5+xM`FWMjDHhvlbNaP?|k6Mk2PV-VnrHaWnIYl56zE{1*%`?ml zo)~KdYX8d~?hL+GLpK()&%*6d&WTNmtaMRr9J<3)Iny|a$%EgN?d ztVMZu(b{aN%s}I!tx#q<%ElhbR;5Owl!9)nP7}-xRgU-AUBZN%5m-Rv(c~}&BT~i> zcxWAqK>~WIK{%DD%cV6`6?Y(L^^y!uLyM4(5~&sk#Z)^Edy1i>^HWMI!%#urzsn3=HzeqRMK2^eb5i87E^11 z)sXjaJ-Xq;dC_2RKc#(hW%DpxZHK@`ficTTf~l8XUy5PMS@1_9gvZ~A zv-|v`-qh8G)YO(-LkxyYH8|(`1jt`9@TedC-vr{oeIMh0RAPMYwSmW-hYv%~*QkL9 z{aCn3HYUv7%YT64*MA1m+u%cA;x-=ee1|n_=~@Bt(Y5^`)r7J&h1qjhjT>OwB~`{- zodHygtD+>@&Tggx_OUmw;Ba@_`9x=yrkypJG_h7oq8LUHdgE%H{M@EAAF(a+d||dt z<+@B#9g%I4t2eQXKI|s(+YPK?q@fntCNr}bAy~%YHk%kj+*oabcAMzN!zB4XjJs_Z zmOvK+FNZCY6iqfie(cbK$3f=1+^Ze>^%TWZMwJVyLRWdJ7a}#E+klLZ(t`#$3j!8W z1^-_W;)2374rHm#2=h9i4c_APdb}Nbn=y!(a*>1VdK->D(cnGGXM<$~=~g#P-g3Pjn(T2k~OcJoZT(CC$(Y5Ak@ z^*12r@vbIvy{w|!d|7bTm8!$)8Si&IFOMgTdu<|exx~WaM`xsDIfqxC*kW>5`ttM9 ztvTXP&?(VDC&V)mRcDFTN~EH}0ghD5BSRrx*-JM7OEFaQQkiWd$y)TQ+y80xY_Q|;$iY6}O+_L`pa z{Xsnef1DTPL}X;uKGu%`HvZc|l1{xZqZH@XQ5zQa`avU)X`d&POSK^EMN7ShBabBL z$NmP7jL-ObEOp|`3QsPvc9TtO!%TGqawnVbce!sDf!6Lb4KbI z^w0x$5G}n)uA^up;nfz0jNX*(Fe~dTXnlV2wP}>{G2)&Cqfpd?2h(fe5vCnGH3q6p zs$F?sJzEp)Ex*~^&c^DMvFK(Ii5~($H&TjJKE3|7b<2YQ_Yvh}DS3&RYOMUp;XTpc3tM!#9RnT5)ks% zAolDGNv+zgu;WL)oLO=Y<{;T1=OZ(R#!tK0N+sx7-wc=@r`=iw8VUf9qS-sYF9*f& zbeXo9;tL6lpB8A(`J39>f z>L-4EA5;n+x#rgR8FC2FJfI#U**lEF64dXpRgZKzooqmJIi@~CmB%2UfMkKU14v|E z0#xATeOyTCxG5BlQ0$a5!L(q9(KUU29pMa9RbeYKG0cB=_hF#W4+P3l7IL#AVEBWz z!>E)URv0*wqfQpMI^uI@_j?`9QKz1@qB7|mtF^_vUJ4K=(Sz9#16UEg7`AV<(=#yt z3_;kGc{){ZZ)A<mn@)q-k08E`e^XR#XJHgdn>=)6(;rx+ZZ)z3<8o9tt5~W6=g|YC9E%c3! zg;j{8Q9hilwAvV#5i>ZQS@>kKD!<}>m)T;tg_4}3S!S=VpLH<~DDeHEx|P0A?d|t; zMXi*}X8BBcl<0jj{7&%W5c)|wk&tE&NbsLxD+brldNZ?2OxBOdrM+m-Eihv_i^q_* zBd*iw23#-^SE$ycywiwN%Ce1&>`Bs*M)xcEu&6$Ocor`CAYA-Lu=I;y;TOrmCz^pz zJUy`F`tOo>?LR4AA6(+8@&Bj|d3pYm&A9$&A%4#}f(4%SGa9ylLnP}?FI?hDxa4Ej zHG&0Rr=REH6v2|u1@}Lf{ww+KzwkTA9jZ1K`79S!#_CNpTVaXsZj5j5MSuUPN-kzRlVIz@4C3TD(;EH!BL zko;X8f9e16aQ`e#Ozh>&{XX6G^L_Zi;rqFMmM`Vs;P>F@b@I=eo&`+Yv` ziD>U0>htG+$LfCbHYP4D@&9-?Xu~6Xf3|#})})#Lt@#v};2SJ0DYX?1NYD}-G^`(v zWBXv_hX_bkDOlw&fph(Y&R04*+$mIX=Dn7xOhpL(bGh4(8c$-^GFeHgNXJ1+gYsN* zj>+l;eF$$E7__K+ls{vD$DdKdV^zKO&_q?xfO#FC{+vGW@LNv)D83mvr;ReToyne| zro?(RTz;P~Prv#VChHK$EP(!w9fvAD6zWPwFYZdSO%(oTNi84_RHaxWC<-#H2Vt<~ z^fU2O?N-R71;APw-El@t`>$%bNkOZ4g_~qLwH8Z~iE2E`DNs@+)mEdhu8Q4S)Vdc) z?GC05+G|A5qAHwYq{)bF74OBO<3)9ARfWX55Tc}q#bc~F&4BXOB z+QLzuy^0yc<0AT!)YB2FMHbcZCY+d|pM%u8YP|ZX+F$2y?#e%>-_QFat|u8*IPGkO zW<2M;N@E=w#uqqNwu*E8x?^=wl90vu7WBPwQ10Q^D+>$$Y1}C!)n_#%{Rpwo`vw=n+c}dZuYPW}T}xO(5AF?QA~PX25$9!#K))poQ!tg7$(;?RWGq{j(`@Ro zvkBSDbPtP@z_V?Yw%RalkCO0nhPAj8CGk2Yip}~NCS7Hx38$YSP1k&6L~E`rn!WMH zT(*tiwcx*iBUFwg<>10l1!)$t#8bU!qbI%T)2QhyiSn#FMWWrti-1hg{Hshom`&CF zv*(GEzIzJNx*fhPknvWn-ALzWQ<|+8tVb_?Uyo1ig@|-P0YYTit^nK~oQ8NtT5Z91 z>@K0UcTw|Qa=VMLMWIQQ%;ZP`mDw}@&yx__mB~na#`Mt++Z+$QQn6$ro>=~m36a14`HW?E)#+3arV^O09k%A z^3IrY!T%2;K;6GBsP)!J4E5R8^_I5zD82z~79wk8JZD5mqS|cj7pJoxR1*#ss~oqr z?-KV`u*q#}&xE}d+xFW{@9eF|;g;VhR!?G<-#jw2^hG*N<9+Uf#9Fe3=Cld#6lAcj zh~R|fj;f*~Y&Nvq!8YS;c7&Bi)bJRa{q(Ffs( ztBL(FffD^Wjj4^Cy*p#lhjo8UaAJNwV@fhoDVAljtgdRQRn19ffO)iYpkcntglHB* z4DakWL~Ad2jyA|tZ3Nx_l_=8yC)hqr>>GMMH{GiFA>7qO_PNa>!)BaEWS=sl1t6fs z7n5u+==Q^1P$7EG;5*}JjuJG0WWf{0U?^nHJl82kS>9l9n!YdEqK${tr%$4bG= zH2XDIKZ6~*P#5V%yO!s@Hb?p9q(Pq85{HU-A5gJZ9 zhYTOwM;G}HAGLyVrngqZ#Y*}52#q-0eY5i6A8 zmHp@^UZG{B$2jj+rJ}EDJ&cCa;<^eWp^+UVpSRcN{l`I_0kC2{nfiA^^O7qSi>uJr zJzK1$yd@_=6yBNQEA`9}`%I4@j={x6;i8s6l@*P$_8sVC=0*VvfpC>^Zn1GK^rrzK zD)mx7+DWs!RPPdS93sp0&S;=pa5~X}56&41uaAczf|69m)HDo{zjV)$oMEy8gFqs> z6NHv>9$2$BIfh(9)AX@#0lDdRc4X%g2dh|tW2|(LYjbfEeS}WNj0wi zX#RpuL0C(}&%LR5)j6<@KQY?qvY;!-NopJnH;(b1H)}wjTeF^XGEH2mnF9ZjAgr!s zZDOdb864gOM*{?CMVpj*S+4*g+VjzO*s&0&s41H*mn_qgX|ze7fbcjt>EHxSNVX<+ zgH2oV4=KI$TlJ#}49wo}&!=-jF5`^T23VhID)(Em(Q1vJVh;j2Os#|vG`POCKW5QSX>rJQ`T)}@d0O)+ z88V@MG|NmpAR3eRCqVCBdrx>uob!lY$s7U{I`Dx&yodqz;6GV$wiB0y#DH}lVZ%VU z=0s~s@Xg!cwm%UG`^}psUVo#{+tWg$Fh`AZO&Cz#d$5u7yOna?PS~B5)XH zR5-M&Mp|1H$@y~0Y~=8Qk&4!|)Q7ktW{MYV#UVm;v}t!F94q3I;Lh-v;SDMIYK=lX zp^#!rtk<}ZbbJ1kKjqwx)vQZ4I!U4Ws#HrW)F4WrtzN|2Vya)Q}y3 zlK1NkIRGUefRdv#dH_luyXydyd;m&5044j^0VuhD9e|RzuFnn{@BoxN-opb>@&PD$ z=Q;o-AApjFZan}clLJt)y$(Rh2cYBwQ1SsN`AKHv0F-5EK(=5^5Lq$fo}xr?u__p$O+&NwIIH*$QVDQ^yVy_j zjcRH}OB;n-QjV`6G^?c=U$n9<8eX}V6U88$AJ0fPaS&V51&W^GFQFwf!;?N<7pRG6 z0P$K@%?4xL3@T`?8j=;91%f)UhX^YO??1DmiZCNqX2>E3NxW(l2_i9c9l%$UoGYUp zsWUIt=y>Rc1=be#nnNoDct^$_!G36DE?cHJkxfGbtXula9Wfxa< zta`F;iT!c%+I`KNhQR^@WLaQ^x!6ufKJG_tk+-(-u2FqPh&<3Lnr;>~XG^l=5O4({ zOlOEBL(%-86sw{CMy;^p2rvyZz@~0RMw?EGghW5SBr6DwG6ME*pa19Y$YP^U*U=WW z(Ada^7H$#?CRTclfYs=8$Pa?IGYpXT;R$&0#qFXmIy}qS-7|09W`$hM2T=aU4)ds* zKR}|g;MpdLNG)s9W@Cmr~>)#9cRLZfc3O~@%*>H$8ADnO@99zB23j8 zG9=)s`oJt{QD`kIYGiFBEc_5*?@iZD8VM(hls7m3Q_GFxT(MECSe{|dgVWTG;Sf4D z+zpaUc5(XlK40!+(U$BryW^>4MtVVd>k~T(Cu|{G_Dpo75G%NoWR`^vxc^MgIXR>?v zaN-u)nYTuPn}Wd{$(cLjK^eFK06jHrPhZL0JOrB-5oZU4w6w_jWEMPSQ-9lyslTW7 zH*QnoU4f;3{H+Wfn7ET+51X#^PuQN2eis`J0lS|?cra4;oS>souYqZp9KSq%l>Ll@ ztz|`5(Gm+=W8Qw$Pd?VdAN&sH@5q2F(+0~ss00A-K*yY&NF%(+XxRor?&*fmTDBse zqlLhkYu;{k`kfTaS|($@pkCH98e+L1FXk_ve+t$Lz1d=|?;LBzee)>&varSz`t}wH z*F!Dl7WWjs(n=ufO4LN={RBiJEmeBiq^Cn*;t9izr|PP9@xa}W+Z>6x+x#BajSSd4 zKfnn5As-!)6TI^n)2AX~<%!E}hJW#)ieQ^+&eUgyteD#`Mpp@wxajThluR z30}OQbu>Gwk>nc3Miy;Dn7HG$6eSZ4xuZ1)EP zt87ZYKML<aDEFZt->!9Kkr*&&e3?`jE-|Sh@TS-#KcOWz5(eZ4slHES}kpzyUegLgS(Hm9^YOz2o$}!X6W=(}u@7rQnwmf%h zS+lC(88y%U(7<#5sP%*YFu(oKL71WEheA~vG8kuRTLSY;=lga6Huuh}{jH_Jv%%L2qVcBywfkp5Pwz=%9-#wEGasy6rT_L_1Ib zVn&X3Q?F*aTg!5Hzs$%#)j;%AfY=MU`f3;#cV~Er%JVTsm$OB?yDMkl<{tR^p4Pi# zo}q=Wk4L!VE(uoHnG zP7?!9t{d90Wm{ac=52o&B1yF2Sc6c6fsi@7W(AznALAf-4Jnm|GTuoWq;w^74E$2r zn&oW)Zr@4#7Qei1WKCD>73{NX^<%VU7rNF;|Gtl0!~7UYdX;K8(uK< zH5c+8fIuKFUqTv~%*b>;INkAcw>Kdbx2Uz?z!?uRgZg$ZKW2qJsZ~v!Bt1tz@nL zJW-jf%$k54aq3Mm-Jo@Sq)lZcO^8G>8h+!InJz5ir9mN438B47Yl&;I*D_9$THaw$6F^X4ZL4vEAm$c@^#zm@OzA=nM z3c_<%RC zwxG++1SLr1WTgdINVstjMI%^=KI+LmTTGH2jPCKOph~55+T2ev{Q#j#@s;QbQE3fs z+UsSh<=7TP5_Rz*E83jpZ)#ayh^4HDBCY!y?kd^>nYp-}?&cwzt$yBCPBqP-2ZljI z*1MiV9SgUY4Tm4v7T?MGRuSvT!&X@S&OQkON)I-dyrs8{KvBYxB7C}&t^qVDm~Tuj zmREeMqP!-rnU*T`K6$C>Y@kWDJ7p*n8W5Y^%`qCVuth5V%9w>Q8a9GZg990@#ZKiO z+=*^E`fcJOSRXg9yG60<$X8-iw7Fkv$-7(ItrdlNM3XO8>eV7kHn`5Q2noAHNj{YK zy%nChz#lgUy0v|UgdX|q-5vb=IDHCxN^F0FKSFmN&z}MkXOiv6n6b7L#Q|^)P$ZyT z*i}?y!(q{0s*KUxF_-Xv=tnO^4p`M4zB=k%hQ>AXsD}8OVa-3l`{&r%AIPh(E(xty zEjm}f);!ADI<4yjx_$S2{^G^_`JjH2lQmzhha5Zy%hkbh^{-~R`mwi5BfMj4OXF~r zb@=I~II9Wk_Ixdef;>&2j5OTE9#X&y?9QLf@JGn~0ov>g8@rof^l(oQhlkjyQSf5( z@yJmp`w6&VRoDd2{;1K+`8BFA8o0L+rmL7i|Nf?`^F}PB)6&Gj^-U zhS6FL#O8V%`1#Lv-<@u{J$QQWea^klYwvUEeOCVW>hw>iyIVEDSZdie?5bu<{vjem zXmtkeCKcS?TdA6@xPk&Efc?3qA}@xKhL_To5;KszMqc{n(%3wV14If!iE0k&knv8Kkt8251YdJ^K81^s5dB}vknk@I)2$D&Y!c|*Of zo(H;v-FCld);!WA>3pl`M>-=^YgEWAs4&%ROe0w1*PKN|dY8RAztq^oq+}qCniK}6 zL+hC~V-w~@f8jC#-BhGh6%Qu}CdGo=SxfG|o93ENex`ACV=L7r# zafwi*A-5%{sGbqpu+@$UqkomGUO~`I2umgj<9$-Owh+vAEOBDk#AAQ6(;hTc2NEX2 z{euJHt54$Z_g;?Fb&P0yp!7U%D%WWQ(F45@)d;%htGg;?ctPT>@L82O+ zj)O!6*FmBhy$%u;IY?Af`*)D24ieR)NmQy|k~$((Z56O4Ygy!q01zxB|Jl|EZJL@d zz=$~<@0`i9?4ws=t$bye%2v%Xmlr!w#CxI$wp3r%vaJ#%ucLD`z|(@Lwz7uxf~@rO zc#xa38>n*>f7?UrgNWBk=5ul_ONVHn*T+~cVX%K~7p!0-xqHy!8ldZefbVs~`WTkR zg|-RX_XY8m5hWfp;m^tawfqJb>w@kNp$D z?hzQsB{}xd9PB$>N6m}IkP?1_@2^;0ax}LuxRGJMet&H6=j!`eeLtJcW~TTdfJBcu zWc(RN*iPA)Ao;Bs)=Z##u^LC+qhe#mH`gaRvr?Y}ed$V9>q&yBW`MnxY92r@;ELR1 z!TynYi*c;$n9tDcj#ofnA*{x}w{74t7ZhO2N`kXZ-!OhSk4?1=9x^mzN(3#KCRFyJ z3LwTB0JJMufjdy+zcU_A8B=LC-^kKS1eZDit}WSuQ`ime2s*X-hkvF zntB7_Mms2auHeDQn7&a=Ib>^lfH}x|$Or*AM>BRVw$InOp6vj0>|Ko_Q4-(0>X&&dg3{{V?B zDK9WDjgA(M*#d-F+J@)uSk)S(0nQc@d6en@cxYFYwz9A(fxSKvE2}o z;m2{pfC9f!Y{V$D!<)W&G~VO`J!o?5?|q~>n|x?^mc3!D$q6eswpM{-EO7+~-a1Cv z3Hc^AXzKlbXQ0D%#JE4OrL6+q+nmcAS^&!vDq zCmH){IfRig*!TAmVb7`Syngr-_8o3X>-){7+UmQ0_!IWM9X--^`mRsEe&69Z`c!?7 z*k%>WGFetvwOkIuZ|n>YZ6)xKNei(@?hd@ULd(dgIR(r{dvbwMIqYG)L1wgAOVtGD zV`ombAL|OE5vtz`--S|L@B&*J+-smUpyWmemW8V54VKn|`;7M@(5B|h9aNd$ZX*G_ zu`AR{VNguuVP8WHN1ki=<>rAUvyDiw688!rxq`cRllM6Ek_F|Hr zu^cQfDYEFf3uyrJy7EQPu7;J-YV-_`>uP-=@eEemxWvc+Y&~K#Y{-|FuS2q8PrMb< zcGFQRfNz4AYO}3DjOx~pE6fEQq$y<(;|V2U&!pO@tQqr4_8m!hCpeJ2XciaIoZGe> z3`D>_0S6<%3aI3-Raxf|bFjN3=1{ji>W&-#QAF`Tj=6tb_V+OE?_HKENW0h#D>6iTt+mOZ-_W%Ql+mLqceUHr8 z3$6O>D~GR2MPJnn0DNe1U9l`tT>J9v^?Cn6;7e;}V664;gyzPT6_VH3VlCw@Ik9Kb@C9lNM{H1%sujRs*z2cfd;5vfF zROeGU)@>5g=&>(i^^2(hx-iD01X|WiqH_W-7+FgI4AEM-)={JYa)X_WxI8fThE;mQ zbLjq|@Hq=nye#MnDzGC8itmp`NN13Ay_;hyQdjI3(5w|Q7wdm9)g!?9eXhs49K{16 zn1+9>e0v1w%WHbax|}m~Qfp1tn}a}`$XZVj3R*KkleDeL5?ZFDaP9H|41EAWbV`H-P=*wWTV6y4eAP{U|77Uzltn&tR*q&p`faR>^y#pl1Z0=E$ zN?3Txxzem1YD`^+YBlwvoqYH>Q{#xVy?V!{VK`3mp#2C)p7s=l749bQ&+vyKcFEl2n41hlBbQ#g9?#1tgMQ`x&e4iL`*P5 z`OMTyeD-fP_-c4I}^&Fcs{hf&3??iBlqL@|Km*!-uU|WXVXH z#H{#F43Is>mo5nIk_zvV2=KB^D6V_ii$*IVJ>?3}=fSNo z9UnM-e|&auLVE%VMQm{{N;m1Svo}BPY3HiyU=2~D1PgNedqrN%zj$#5TL|l_+#$N5 z8nc8!7@*b}!HSM8y!g*E!kgU+J!ptvg4I0R8rqou!LQP@FOPOpK65AR3Og~|`Dx!H zcR)Hwb$)*5nJEFk7cw3o4~~j_Mhhtzx%&21IFtbZ@F42F3wbwwYzd2N>ws!*J49At z7Qo5{17DS&-2u_-RvWnb_SG5Tb2hhs!L4C44HUkBQLhll(iNHm76#q-HIB4afB1tQ zvVQK5vp*R56#n|b7%6|)`WBC~K%>wHR~q;SttVD2o8$DUA53BwD9>{&eI1KsC7mq> z5O*58P)@hjnu37T7i;YmvcZN!HY0!&)AMu!k#8SpSruA0kdhQqtWM|TdqK_xVIR1H zeT#D%D2-_>)6ffxHm19>og{?g`IV5(uPix7I%*$mB%ybb+uEg2VFMGibCBVypwfF3qXS6$_ne_5vc_C zlG7Z|pk^(1^-2s5F4hL20;;|O!hxxc5*EQJS=92D36d49YFG_a*|*4crJK=YXVTX~ z!jI)mCC!0i{^||~2de+=D?*nIgCci&$`@?Z@k-Dvj&%{f7l7WUrPebv=^5XQ;6)&s zBC+86XlWk*>emVE5=jkezQH-@YSqAAjc`k1%lcJlz4E51Uo?X^nt}`lGJ+pRnx$H- zHD+69H-~D3|3ox1lkeTJ%9LWj!4ka02-Xwk5%dh>!R$cCVGgw5qZ%ffZ@aTSUZEXq zsJwR0Cr!#`dRu`c+@I)3i?E7NXvy*$a}H_NX6+C-4CjxEE{#bk9UP)rI3*3*w>~k$ z!%ib|cyXq=$y=GnZ`~G!y+^wYYRvIfpiANn(N+m5U6I6oY40_?+h4}4+*Ddg^53vS zGkD$X#kk^QuLPt5yb|#W{F_6l1@_V$2t~oEYtud1xvqVQ)>vQ@hQ_#NFfYlkdLoJu zQw6QmS_>WsQh1`w7jv@+^V{FU3iWS~6;IdKuXZG&lfk`lngY*(my^e#HYHf6Y-z8q zHWN&gi91SJ__gnA05=K3s-C&x%T9R^>^`AID_vnSsu=W|_urV%DE7%hnUHg_!6|(0 zk%_0*CA7$m4vE1b{d`n62JObWCUDp9ia-;h{ZzJK*;<*(eL-Y!^IBfe((J=pu$S!%C0 zuzfa0_Gs3;&Sw>csH{!J)S2JmgobJG2j_$`6B3>N{bXl(w;Q^GfOW)I?6YKmOS%?CO<$cg{07yh8|mphiXNJa8|r= zb&tA3K4s}$sd;KpH24x@M^MQ!byW>vq@0q?+E(T1hesKpeSOd^Xm)!~>s&!R8EyE2 z>j9*+?sx6PleGZ$&fmMIDOLf}WhEP3wd-oXilv?wDcCr!&DIo@XiwIg$-2GpXa~W= z0@Z1icBF(||I(|_+)KhsEk#q&ken~g8KInCZqmx{%$Qb4zJnTf-n^o13W6k327WKQ zP-#SFkxN_X^=wQ0Om0eEde5|EtcF=lLc6X2?ZCLPNE%?!Tl-D|5+GhD zc`d8TSZK@B?Qz?h!vJe#aXb?j%+y{5>@bXL#Pegjn`u&_7kPH_XV%bD@{QVxzI5)b zG=(sx2HA3%sOKDXkAv<(4!TGGI_Mtcm#cdO>ceeceZ9jkrsP7@{0N*_#joCQN6fxZ zoe4^+Ad|lMP*I^J(KnPAW3_bqmu2RivC7os^*teztf9k2@etmiW+p(RSyQ18&O0tg z!$17X#4EK8a!Az}zYLZi#pW_v(M-$D2w9#)?}pE+hBmD|B5kBNxK19?wC+~?u}{fR zy}#{i3SjQev*9ql{*@ZkH~dsGcO{=^5LW-%D1jqmR`i z+`bNb;lIXSNUDGMklip=4>7gPk*bNq^$`;rO|PODD_YQ96RIjEw)olej0rcZ(P}K) zSC2eh@e3?w0$@*Ak68&AQP*@BsNL5tH^|^#iX|xsVb9DJ!bowMqqeFs>^I&>D6W?G)i_V z(?qJzY+rjE&}r3awy)G`HIoLvRoP~wdd;LJBb9C@RkPW?_L}T?wVUlLx%$ndiRo9$ z8Lfshsm+ASIpY{l<4q=1(s`V2?o`&9)NHs4Pim!|omQvtd^5g%=yFPoo4wMlZFq$$ z_w3#qDG#%KP1MY$jyGEQXZw1HGB4XU&cV99;RRD0)j%LZN|PnRnL$E? zkPYD?E7}|(0-$lk<$C;y+=CRm2Q3ZSKp^M5N;6^%r9eP*it~bJDG3o1MxC_V;|KQ) zR%7(%M5w&c8fsQ>dV3|^*rduS~xHB6sJ|fUI}~n`|E1m`m|SMv|A*bDf#~5`X;!j#tn%8GZ_EZ zDa_r%>Z#{qiIEaPWHXye%S4WqNRUx;Ek&tz0S+1Kfgp{XlUJeY83be1Ozz}dtAVd* z$%Zyj!-m;DIY`@0f3^{R;*V~LaS+A%* zwxE@aHf$wp{pX3wWM$S8?5(G6Ng^%3r5jXEouO7?92CIjz-is(r780Uz;}x`J&_i0 z?H+{;;t^r}bppZI8qHe4e{5NYFzj%|V*|eQl#G6wv^(R^2rZhmY*%ZHkWYW~M;3&!&==2H zS;+>}3m{qF5s5-2$I=t8EUIyAC)mS$Ds^$dv0ZXjtAB$PO(C?%hEJfzaNc>J3IQ_N zm$YpTbGD{6%^Fs7)$lA(Dwcpp%LK9+>Oh48(4X`ouVww1=7R1k{06kYz}zNE3aJU< zPmv7%IVOT98O~Thq)R-0s>D;INlZzRn2`K1!C0D>^e{2u zVM?(7gubSwFicEec#^)RxkwX|5fTy+k{vz!I(a4}K0ig$^OGbzPfT`B>1TSP^H>^l zlKo&O$vK$99GlyL1m|e_a#RZ&N^VY0U5>PVvvmwUUfS}+lx0Xrj!nf=q$^KMRSwso zvR87Naz6<njA?yjs3<-q!T2C0AEe}il>9FyB`2onK2dt^6Q$-(Ov_DW zJ;^D#UBYdOq?MMA+fTMl5(+G_!ZT!qY~o3-saXrjX-renR=sr8gqBG(WpyVNHL2By zrK?U&Rh^Kgnx3LMZ5oL!6gDY6b#hMA)U?#pl+<3DY4U(a(@}R*Oq1JuoK)1w6FV&h zH97rsVvaV6rJi~@SLtLqo_4zZuH-gHQcj1{NRwL~N;e%$Ax&=aL8+z_(@ghBF`brf zIVH6;HLY|~O6eXcb}3_EoeY}LXe3>9 zFcCDN+2K^t2Z>XUEluxl?VOhMIXU68m(H1HO#H;jo{5_;HOX_21kd>4HYvH&rfx=~ zGaiuGIcd@+l37|ZXF|l^gA+L)ElL^;*2!_wh$Xq@SIFAB1}#b~WnH3%A2o!_C84C- z7?$1)`MkHa=Kl2}rfJr+v@p$LNOZcNrXwR)#PDA!aGDl{go6I>(;R2}NJUAe+-YXq zNhaLS)_nUQ(`|2a>@K5ax3>0uqEFCd5}oh1*VHt-HSEhe^W1A=L@TkP>zrJ={KA6P zE9)KfcCl?c<-l|4Bh30!johd|an)s&K%`fsMLSVHwz8pT##RGeMDHh-QQ4W7DD6Xa z)j6EPwZAy6GIsFEZQ6hk)|?Hsjn$UT8^BBoLyFl|X_2*Pn!#S}&aLOeg?Wv46MV@m zKO3P>5XsFwCgd^d3&-%kp%sUUkX0>LS_^@RH&K*GS&??_X==xaIuR|1D{YCe>6U7quJxbA+OMk4p!Zue))fQ@|_)iS% z705EWzInmG`dB=E54S?qZp%hCSs&Fq!P8&AUFMMnx zp&r>ytr@-U(%zHPZ=yFtsX(c2UbHl58nfk`wa{TICxCqr+~=Rg{FO{8x|%*|<~ci4 z>$ap~Rx_Fd7=`&YY2LWV0ii&wwsw+U7jG_8rfXtJ7OFxZN+To3$Iu9k5Cdnun)}hz zcf6>+zi!S*NwYN<<{VKC6*z^)b%pi8c|45WOaO>r&_PJ=w={XIcBV5G@RS1>kgOX; zj(QCF$UvenYHSVN>5$a$_z7aqh(}-Yn0;!`X5r?kbQC^*ZZ=io_R(~RrQGkVl{DP+ zJhANwQ;_Nuo27-cqQIJD>`C@C#Ed=FnyM+rTeyGBc8NcZt;C~PoNK!kqXfOhy}HK` zk4V{*2_x$HPpnfLV;k_j-&&`7lf}lyok1lcFi1G9G_D$FvaAY*(Q&=@jbMlwjD|xl zHs-dN#@S`-%bI9iqF_yu`Wb21DyNU9em>L?AG{&(V|#)=%|R^Yx)Ko!!z~9wFhg|G&EadJimDpq-hQB>xLro9PO&vf>2lUs$k>~&h>l7 zmP?j3e(|9GSOUw&{TCO&nXKTCgJ?8w!L$JpbLe>SVd#0y}5q zD%rhtv{CE!7ddney0z~l$Aa5>GyQd9Fk8%xX`+CK!q&VPw2L2GTFl97FQOU08g17+ zyaVce&x<^xHCliTxTumQ&qs5g(H1i*9P8;QK^X@mts9=T1+6Wp+h*@|=*aY%X)%8E zknc0k?WeXSl#10{V!BXBo7~zcCx~!hX;v;R-*KDT3keT2>H`8`+HZd9adNj~=j6{$ zf;c05T)y+o{ics*+n6Qln7~rj?2gssB$p(FHOlUI)|}4C|Hob6EK8Pi+OT4CYGWHPj$w>Itme-}^Sj>$i6~?QhAt;j&bOv`vNvO> zPi&EbIZA^(b0B^accK@Y=z&$nSfkoR>YNuoEPuly&R^RAI(3Ax**_t4WhxsrK+IGr$v%xEi!U!OObY(<03=d59M z$wk-(DB5zNb&4fU6^0S=nq;!BnW`j4^tg4-Wl8y{VwGYNo|1|2G&vZLk$sV1uYa11 zi^;hbHjg6O?zAk6bnAaoX2sLwR7}aHm|!nT&!Bj!+=&TU6X^w~_Rg1>QaU0rN8%~6 zBhvFC(yJGxWJGkC4hbzL zePW)%^w5jMd47tlgr~_zn4XCci_e~%gW%5DV6%^reULz53AaqoHRzV-uWXa1qrdGv zVr7@cNRZuujU@x%tPbkwAa+8cFlxXk)obQtHB}qGh8Tjd;9+B)6|f5_Z`;Z)sVnQj zMJ7ulavP|AO%*br``@z74D~*|zgtxVNbG3^-N1vAH)2Z3Wry}VWhzFSwIEggDCY#$ z3Ph+}isJ^~X*a38b+pZHqFhgm{bNqfiD(Omr9;042;^wS&^sf{2?<^4{nhj*aBQbU z4xI!R7-!MFJ*3ByvWY%i-?ICp-E;gFZ{EUS$l0o9kS>8M50p>_&JidO!0wplJvmoc z8b&V%*ua0dUo-KIP_*G{$?2kC|MzS8PBzw}{lmC8$QvtqTlAAkYmrdbyu$``(14Iu z1S;gX0c6=%axK+0v|fQj%&3-@1Fs6XpoN8{80VTdJNLStn?F0>`1cF*?{CP@3>b{{kiYov_F3I5Z-^A8g%>9AW{|J;X20(%7=WweLm^y{2LVuN>zGmKB_f%{|>j8G=1hSNzVQe^oWh zSnl$?a36D9Q3hW&&cS#yCzs9fu>zB*$(uunbn7EArdGY+K-GiH&9Ds$vB=> z`Zr(fZIv$F4o+hEN8OJ1e4uoIOIe08I=5hqr|faqsZ3dRNXkw z$d$~UWtO06cD%`&(MFN)kNW)L`y=o*I8obxRd&enYu@Z!uy+*%XS=nO4AxoJ+m7r}g^;9)*+Xn%3$%|fWB z=KlXkI^(;uJbd82G>3BHA&vhA<*|-T2dVQvPP);ag3sDAiE%Z-F%K==c|Oaz6HpXy zT3{e&P;x|{ih6Q_`)Od(GF$tpJ50~C+(28!Ea2HV(^Gn3t#b66li!dTd48&?T;Mi> zMf61zz}pQMUvAb~DbOpTy7DSp!UK?`ivrVfIgQ{l^iq@bTy$%4DXwJRcfj>PnnT;O z18*iL_Cps)cRD|M(lc4eoSiSBkk{r)7CcLfwK;mX^PG1^ujM_tXE^OZdZXjN0&l3_ ziD&u`M_X8j%^r0|YPCmBOe9yZ%wJcS8bfRLlFZ0oq>%MXlzIguPP?OQv>QRAi4QNZ zbucR}o?e2&|8n2fow=YWOn>7q{eJvRyj=PRt$*CW8N<=E*AJV~zrwTy1ZqtIUQ{Ip zRn7`_#|pO-JnGg>Y7)BC<4{}*q@jU*G;Xgktu70?8r>vO(}teX8}PeNrGEdZ5beV& z#u)&{5e4adke)+SijArYyZOq%wARl&RPD1?^%kjpbamsla;W@`-h z#TWc>e8=eZJX*BbEo-btquxMapSB?jT2PVc9kLzq5dh|dmt%|2D@4a3)Y``2m0sK+ zRHG3Y3#oF{8|ZnVk`_pPIz24L^EIqICS z`3l^-TcmZ=#}8Rc)d*U$qj41xW>r%jZb55jG`P0j?1&tyE_p8u=?F*e$$pQt0C{wV zZ~e&t2zYM|h@ZXQTsA|tntk{4jzucXe7>uw^^Yuwd!lBajsqiThR>G(Ff5r8$H&fI zW$J5eMYCxEsHX{zwtQj*Ued6lFhCtv5eJ2{k)6iPq2<0UZhSok8uS}8-ZV60V|gt+ z$U^A;D}JX>z$B9%nm^a9s7TH7Hh?$+orGil+}HmhM%;2na@1SMBgB6m-{ig1nlFLB zf%wapgx#@vLn^5hunu7HR4Kj^WysIGv@uekGs5VsdUa&iKhCX7vT%Aod4#uv*eTC zJp1C=ixWNOFHZI7JzM-2vlpi%kf_yb(Tt3#o&kL~g2*-gQkYse_p;7S+^<>0(Pp8D zsx6SgOMBkJ_Vae^y##gxP6XbNUzM_E<0Z7Zc1Eq`W5(_X3l7E5%&Y-$@V6#-9L;uF z`@miM#xK`2<6`v+0MsKf%QybLuI9IaI|3JvAUA6&r1I0~2KvfARIKIz5AS4Tu*Z~i zL$Wm$D`t6aTNzTXS;45n?+2P()Qkc)**zVILJr4Pj>c2&k zoD=fam^sIeN=(f)Pv5eUgpdfSjTcNBsDE!dY%m0y$%t+8j@5UZ-9NjRwVvXc=AJWC z!R;B;Tz`h8*GGnY^Y!bmUy}1YH!Pr4Y}pp3AmE(Pihs*$1qbUb*AiOG^FIx5!pOYH z3S+zy=H}Ye|G8f?Q=^NKYg`qr3O1;ZSfh2QfC2VtA!We^sC}F(3`f%i_wkf?q~6Yc z!-wR-!0zssKCNi8wsd4Sd#~ogz{D(hVZEi`>jcw+mQ?Wc?Y-`-|EKQl5R0R?OnDO3 z?$!yAhXEOkOv1OOw1O#p*c&6{bh$7>SA}74@DLK$Hd;-*ZfK+F{hBpMMh0%dzN#xd zXN}#os+hD`rG+-W&2lfivY@kgu(;L~6dx|zYsJSyIH@tdDE`(~) z0o!c7Cfug!nJM7k8Kz;?3>bn~*@QiEff`CAp7@A;XnuQiMvgu#mHtOGOLa6Sm!+$x z!^M)-HB!rhnFoSx@+E17S61{1h$8rYJlWUsp5zjtqu3o24VFoDacSP_MvGEsvaD)Z zaxe8BHc1E#HYrjYY-&J!$u#O7v1ICfA$(liXa$x50=-^u-x;!PGpAY$$)6RFR_>M{{z`OJ2~r*qr&}cJ~EaT*}&Y z1gvV$kI0Fvp+{Q9E0`@eZ5Yi417A;rHEUD6oFvK9Nf7%f|5_BmysGUwnD~UvimhAY z+gZxm6!*5b1ZPLFrZKwHw+1x5g*3X4$lys-inSNEe)&3mp6nA4^9Jx(^TaZ1W^PV{R);{E0{yA2nU+hE1RZzx zws51rhmzSS;{XQ=8GYP=sKu8BWZuFfo{;hYc_IV0zlxO|=+m0_kY(9_(LH>H-viwF zOY%+%7S7&#&uC8c&OO7A-_^WfL2i_E1ZfhFl?3Pw$Ua~P7b*)B?fK(2$0y@iUCx#6w^3sB}j+`t@uzr%G4>#2W>+}W~^ z-D+Uhb5Aj8l1P6;naV$U2V(6c(26XF@5=Yf`UK zghthREIUC8BajI}!yP|pb^(#BXH+Mp#ucTvW*5T*JJig_et1*+a7(JJx)(*I!#QR$ z&Qu$PJ#7v)BeR}jDs}izi=yxu`vn|J;gawQtGNRs0??yF#N*^ea#HTA8JAaSzm{6B z-O(m7!YR=}A=H1esSPVl8m|@+Y*^9*jjHvYv9&+iHC{zDm&A^r-z(3IL8AJ#4pC8 zT$d49#+gwr=r~di?|kmtV`A|?d{W)~=Ud)=&L!jSpOZv*dNriRGFa+^Wt5>?6B~Gh zP3bF5#&DI~Ws%a*27Sl9HR)}(=B_wTk+?@edR^XkKw-6HI8&$05rjH5VRx-8m~x|L}s^-_wsvH@u5~zY=rU+s-f4 zq`{ra99PhO(f4=;Bf3AC^0qGBPtg zVRG0ok)x}WL>JN{Vm=B>mLiZq4f@QA_FAK9m(!Z9E8&BY2MvO3u(v3#gvFvF*$9~6 zsVsA^uF2ZGq=;E9BJg?)MLc9c%5#Q>9RHm`4*0$xtVAClr5LfWi?>%C^9O(p#2On^ z;izgTh@Q1zmE=_Ro%~u7k%pRB;xoAjy7oW8)UdjMJk90s-W)Wr9n%yQt6nulBT^WE zVn0|Z(QNoTcscCG^V`iiO?6YzoS+wDB$ui$W|{;tN^hH$pbo@>Z_--$RJsTcCY&Bg zB5WQ1BP@8q^GuWM2)MzHVA@zceE(W2LRu1LYhw4(_4`e)yR)+>Mhw<~sRweRCo1gF z&@vTGx_Iy<<#b&X63U8;gw+*|Jr0P)OFL2i@tjq{stssdU-G0%_rv^zv=wZO^CUfd z27_eLRf;{GG(ilS&06J4V!_$LD5I?n3Ves6ljTY{G?;PceQ1wf%_(KkEjMG;Rgx?p z?c$+iB(;W89i9}xkVUohk0^!j+vJwIDdrBt zLYiLNg3{;AS*KaYNOO%mOy4PDZ66A(Ue=Shg%W_b@R>fmc~9fT*RZ_<|2Pk@ zog{*C=q8;N`cCR)M@-=m(TK9LnL&n?XOOeBehAL`%E+ln{&7CL@=BjnP_?3nWy=D! z__aY7QS3m_pz|wC%#Akv;rv)XQ~q0_+EeVQYKy>`7vzeHf=N}x9<`dJDbJscS_Zpy z`C*XrGPU8!b|NJuP_pEI(0_N6PBg~=I?YjmmCw({g}-xynBK?AFPPp~4m+X1o4tne zs6X&>7->=&K~{Np$FPpjhKO+6CJ4k{`qu=p9At5KT}&qId_fB`o4gIfcL_1mQ($I0>{4~| zoiUx0nB4h5N5EeXoLlDfjp*`D{sfn7mj`GEDx1jS97F;jAMLg3S(9E4a+hCUmaKkN z42dqUpMvGznP<1>AVAVgVMKd(8_%&mk_>C{0o4haDc9}|fuu}l$IuzK;gJACFV5vT zMRe=uJ!NjrWnV1WW+~^2_hNgmuMu?}64j1!->6l*QlWcS?(4~GMMyUEeW*ub1pjJYW`mF$;UQmhQm8Q+i<_d6?{`UYA?GJFE}OR&nI z!qp~^RHT)K?>6M{`mG2!426I0Orndfx7!hB0c0(=d{+_ZRGp0U-qtXfRMq4Wgg#6p z`Rk~!`KP$j4)|nk2)yIO<6Kz>d{0MFr0fZ#0Z&EI!KQxmH$t~_UCcQ{I%1JSIj0ZglH`Ofj?#N0nmfKDQ_yn`3qnK~=RIoOUfbSf}gmL}((3_3`zLg5#`U)12 z_nF0o7q>($%B zXZHtdWq+MN8ov)$OG_yz{5~%?8SHB4e`e(TezMFbwZ1M-lX~l;dMVf)JwHnQ|peb zt2|60^E=ye+UDvSe$G*v+72$-PEDH~+qt#7%9+-Vg@XhS>6 z)u}WZX+zexO@UY}sruer-ZEwEZA=@RK?g=FR_a!uC&RITB>M%?O^~&{X1A5&5`WB? zqIQYz#-R6Y&4BVD0c$8$MevKX`BSywipNctx__%1wo~7bJN*Ha|}@A)&5C z24mpO>WP*h?73R(c!xM>0$E#XAd{IFuOMIe7&jGQ*?|-fSY`_NVA-z-yv~Tb zS{`J6n_Qm0uwH7mR66v>bd)v*Qge3(U1LA1CwP;X^O#|PUZAFpT!VfSCTnv|nK-j{ zQ&TdGM;JzHt0#`=)eJf^XA)id#-|6cu#_KsWX0l)0;()MuzgBXg`-~VZC!!Muu`wM!y@z_}ePyo65HjS&TrmfNH zRcEd9f~L{qpWxNjXCG}Qlpl6V_d_jWf)Zrz=fB}@1L@~L1z!!P3bR@YB?zGc4f439 zj5nb|t=M-rQ7hv*xJAp4R15ua(>v4%N8@V-VJxXD{d41n_q|ouKy99cKjiR~ZASmP zRpTkAlrU?aKYnwusaB5&+^Xcoq|Tx9ETCneCS{)VLMZ^Ee;jz+u4V{fP5BB^nR>xr z)8peJ`UaT3*6Sy?VR^wcKY|Xr)Tx0)UoUmz6s%xDD)*x$uK)_GYQ$r1t z+fn47XxPYvqfk&tAo|?>$c!Ip#6H+P5v`O^JSjY6H7WP=HT;33e2k%|s2SLcjwFa%emszIuFpvhB=T3Cl?K$HRj3Sj=o?IB1VhxOj(q8v@maEyMelgt{f5|1R===eL>=h_Cl$h^_`Npllq^V z>ZwIs4=uLMjGv+v4+rJCt7xwHx!&j?{R8NB<1U7LG~- zP$~~*rvugJ%6_h?ax5F31>>heo0XH8l21gM5_?mJtm=w4wIyWcnrWd`jfjkweip~_ zAoW0sRp(A!PNw)}Rgi+%BHS2gauq@t6WIxYr*zqI=X+SDjnXrjk@4|T36M6GrG#RUgtPKM6*T3APV}%AQc$P3y$e*)T->V8-8U?(DGA?}_2Yqez4}_9H2@IbdzbadI7+qHAjJ+df zfDtOwcxG%@J6e1SmN>Z(XT@o-3yrS z)2WmSjTLz#l#SLRp$a}->(UWoUfp!qLHMcRPWmt*=cpXh#o=p+*i3jk>S09rZ;vGzqW8WlCm5@>6If7adtE=Ks@^0KG)j z^UTRzNRh3B|7nr|F97|+VX7%GTtk!;&ia~orzmxT8ktU?#Qbq>;$wu5b~P2L>e8cC zY9RQ026b(&KBnuo=+RY0Gm;u4hqCkmQfr|$sp_fwk8ki~hV^}?k=R-&Uhm{N_W><9fuQ?y&w3)p-4X1~Zqv_cJ~_Bm(RoSnt^0 zsKX$L&Tx8l0bgs4ZX(UQ#{EVMM^e;JMIH`;o{_Pvf)|+-p}r2UdjXhUf5?V3-KK4E zf*z<6;*5eV)(>JT(dDP$C1tCso|)ZlS+z`xA+LD6#xRZ7d;!u{ar+N#obJS@?B=Mg z2APiHp!7d%O49S}QTZ9&AdD!P(LwkxwdF$*YKj2}zt}V{0al>q-xkHFUoZmZ(~19L zvvb%p_}m-f%0U=9O%c)|TCcjT%e7h}^mT7^!#{~aG#@9>IfH#{%L2TmVE95JzmN*p zy{Kl!xYy78=n0oxiFGb6n~g8Bt@qyJ8pJujAZZioJ>rR(<~uQ3KvmW4c{<)SDBXl{ zS|saKo@*1AY5Hcz6QP5hSBF@jYkAK-H30?PC2Iutk!gRWss2M=Od+&`i7hw4&4NgMQ~iMN-!;0{|$)!xj3zI4icN-Sq7m+a%S0 zzSkbOH5sM9@&;6f6DINAS+ceCv3Tf^+c2a4rya0)A~jBjuM=4aPgFe`1b~IC*(Z6U zL)uG7kAJa2a?dpXO!iz9s(u?m{H(~6EFKWHA!4y^W|WsCwaBUIcKsU-k!09pWFl+% z5RhsHV7bPfGCDKd<|FjUV){KaLVRMJbT?nn%T{ zb26b_|98P&+4ApEUru73%S-xP7i^2yl4=#gjmylS;#r^40iqzB@hXspWiXEWL)bTg z7<)?J1%!jNi^g-(kyWf*&z$0tM#k=~!1QQ26S^N7K_7tRJb$!}t z2^|X8x3otD>*BDt!#E#{AwZl~%u?)FZ|oAY*r|UKpx^mT-RqK|jt5`dgMN zN_Zk(Oi1Ka81`pLI#q>Es>XVvIzIvulm^WP@uH)5K!oN4Tzt2S7_M5aYW^B=J{1Oi zsLDh8>TR4%BUVQyb%*kWp-gbhYm-DwF{_jo z`|GybEd@Exf0KiyHSMK=@ygc<;i8lviI^swr1?e z0V^;IxC2Cp+THx6z0Cb*ReBUiasbi#G`l%+N&j;N zIw?)BqtZmfm-+ssaEa?mwy6oeXR}}YM2*Y$6uwsbw8x+X+lEEUE$*nwge0+=8rXGy zhDveN?__tk&hGr*;!bNo z_zMmLYU$WKVRXD&c4eCX>XI31;Uq|oRukQI$k_N)bhuY`@;)b2J}V9>)Vm}*vz z*=icC&N!9BR$4AXUB|$f2Mv6qy6b==C3m43elH@0JZ&z7W@UKN*5kdxTi71fD>fyk zp2bYTI=~FugqZO?AMR1GI-$5?Ye*aIoc_C!N9f*~bC_v@WM4^o!NL1+k~O)iR4T!5 z1TOHlm0JNON7HxyK%k`u4hkm5!BtWn7Q@^TrHXYaT~gcLArqvw>rvI)+~e&cFx487 zIa!y@XiAA|r|RnOj}ZK!Yf|MTd!)V12iD-@w|LmnqaPr`eY8v1ZDTBT1Lq5vSF&|~ zPSpOhyix_~wc{xh9g5P7?ie5+cu~A87YHZEYG9j16=cKD&IYEJYRs_R5;4*gU0R=} z+GP80TarTdHQXd*2jd`r$)ml=VZ;nVXEHy1GiyMyB}itjG+r*i4)==QBek*9jJlLz zCJ%;6LipQ3Q6yFucY2(S!9(B@q;VGy%)gt#8ab``@=x1AEqp z7&2QE5o(6S3Oef%hv4NcpsT0N)9?P`>*U~B>Y>;J^fly^t34sRoip#QO&B}~7d`d6K@b`xvRaF6J4vKYR3=RF#L5F~i zmDED1rAekEanBNFeaqgN&t>;80hJ9E=KP9PctstiKN%4)8+VE*;mO0_5~$ukZGdIH zq`K#veZSxS8O%X(WR>NFlDhV`Ah&c5OyE*10kMO4NT-fyvhTK{?^FARhr*eFaGKk~ zJuY=+BU=g4gRpEVa|L-b0@cm<=cWW9gH{R*N0)mkg4JDklnb9dl+!GR-UrZ0Jof15 zJZXEc=JlBxdeOt4;kZ8sE^p zgD!ohjm277u;%Q7AMFxcgZS3Uo|E`LE&IxNBKzs#3DDO&A|Yv{%pRiO>YvKV89W=z zm(k8zyF7Z7wz|Pz3kwebCV*FK1J83wQZk;g|Vy~Cz~GDC3* z;uPTMU}eE>+MF}X?bF#E5u7LelOw5WN?(5E^NJx=CS4yvFN6PY;JgUwiyY4b*zylB zPgdT>q}cTVcKeK4a28z0`Fld3P@QH^h3Iy@fBmpJWp&@X0?UO4*TV*HZZ$# zhkwIkQCd(yNM1BYpgr%rfZ+9p`*YeOji45i&VxV>0DD8u5yu(tu{341 zrEvr|>{L2Wrp+gOv`5L|nQQ8)Z+5?$Euz-V{V>#RBYIH7DN+4#Vj7x5E1HSQHc80^%v-eP_15frKIGwMs?D!ko#YZXRiW#yt#uFMY@L@W zvzfZQz$H`)n-RiG(2LqdECH?jt*OW$a=@1fj$~p-Q9?u$yFjt8xJ4Ap2Up7S?Lu+y zwSvtCQkR#S<#n7sAe+BIzAk>L%BDE@z{V z>GKyLT7`YlApVo5o}N+K*S%+v6s|r|*yiCYjK9vaK=kK=*qdAekAn7NJ4zm|1I^9f z`H3Acl9KGi8lgXEqYSDfnNZr;Lv(#C&kMZV+@QOM>#e)KF!IA(Bk=cTcl+oS5 zw(UcH&T7e+bb_(=6_01_>2{%g%QkfY?pcV}?3h@<&8u;U=Y!USn5^XC5~Ps2h@7xU z0zEr+x*8RAupI%?qwe++XvQrR{iyU8iP$VQbNg&}cQN!Ypn+kknHW<_!G*LIk0EBnM4^0bp>B=<)Le;lIjXsW z)dN|AKH~9NL0GZb=z7766k+9{tvY*GJVS2zGoG+IqB5A++FlzVTp*GVGE?iL@bAwG z>a=WKln-u}`>GBgyd_{2JpH_d9_qAA{!+=nRZJR`RePjnAqBv*34`pfni{`I?UAkp zhP63`TMin%A30NTUjM=X;P56gmv*17^Uf$VK6aaWj(XJdLpO04b zOrq2T=gI@4KzgR0id6&%FDc|P*g}M#pTi>3uih{z3f}|=kY@`o&Ub)jx*%)hqVvr) zC7pRTJ*e{tCFhmY!dW4sTTnjnb$y_BDdAX#B-4v!wvYx=r>Dhw1+}bnTns**nbf0+**ChQy5Alp z6weXn^)`0vZ8MJ&3-WBTGD_Gk@bL)3FVJZ;>6ud7U#uUzl=n9DSu64b#b#cPBWFeT zx*0V^Czx8(x($Z9%$3G%pw(&*S7!dmol^ACA0nfGKwO+a_mc8J@+030dWjxXmd5vNNdvH#B)5BPlhiRlUu}P9o zZmM3vOEHEfmtw?SZNdD8@T*Ofg&H?QnE5C|!74?V=_kQhV5-6r@v?QfPxN%1}^HQc_Uz za{aeS)A~we*1B8x64taq$BUxrlh(w)fMdYIKp4hb<(0lXv9ksUwJ0YXR^k7^aLyUs;b*6O(iY7Z*5N*3P4 zt8<=~7e!_X3q9V)(##q?IP}1}5AcuO!QI{;M(&TFySwl8tKHwXkGoft=UhLokDs%k z_ruG=!ok5;e!ZWkn~PEY$U=nM%a)FyUOwC&3jQA?iKDZ#Pd{1j`5YbjB}375*wiO* zWZJ(7;Pfopo7tDLPi}Ng3$S2C(O}~@1m}CQEWd$cR9QQ>h3bCi`>Aqd!HNHH6`fFg zrm$UXwz{i&r|R#FU@p|Ymv|-#NYvmqtc4Y~DJppPg`2J5IL@k4C-|j2R>Nu8#`vR1 zv9pRx1h?k3{Bn!YdtNjwKWUqNQZ=EceNs30q-=7FTV@rb+W+^CXIvgI&z@L`&c(Sc zR4=9cNpoCok$#${A-bvARp>@a3~aI|rCyk~0R-;@!_)(I?4wTcT1REwHVtDjBDZFA zqRFh@zp}%x2SEhf-w3i&K z`jINOk_|h1qG-Tio<6k;oVlU{BTYm%JZ1XNV%-hqjM~OP*U@$+$rfw$fwe%}1_!G{aLgAx#g zc&k}pV!OfM=SF_LE1l~~Wuz=5E}(#w%RyMY81t#xo%^dB_b|GO; zh4rbKm#9xa*T|xb=0c;AMj048{xJO*Y=mi7HIzk#1D6beSl*NZqD`@aRQiHLxB`dF z(R3!cOWmA@&{u!p@+BU_t7#C?#Z)x`3H9wKbxMP@Dl4(Od0Y6D`!U#zUW>Eqg%@3a zdDpR@kEwHfKCD^>mFJ2=d?$0X* zAU-~FcFd+<1i6ZaXKokJRb~f_Q)?-uz*?M*cT2c~ZKx^PF^_cBw!`H78`!c5NB=eT zI@beQ4*sv6i7FgtsxJE{S{3a?3^7=M6Goh9yH{~SBy`di9k7*V!%<4eR?}U-=v(UK z!d>v%+#dej6`3k#@R{Y0^fFH>G+@$L$eLoBGymx{>L3p1yf^TUSRZ*8KBvG zE#1tMUtZhAHRQ+H_M=fhB_2&rZDGqDUfkNfv%xs!__tK>-VNtfg}2XvYCUonpJlrc z0>`9h16P z=!MYO4;Yb|{dIF)4l1x}pcQIUVcRwv{=#}dEK=$k+PK4QqepY@Gb67tHA!QC+9wM8 z4o{<8xbKYU64J9JH*aFRk}qvyWK-?*cUZ`mM68;RjF@{`ypeNxsg$J_xiK7pV#Hfv1BMuh(AU{m(1Ck9Q2^*Bm^4dZ#%7IisJit3m?!Db+6eA zhNsq2D7QS`aZi6pE5Wrcg6D%L;^}c5TTa@KxRsyKrr6vi@-A_~59c%t{jbRCYHyji zq*fS_hs%EFNpDL5YWl#785*VzGH1{uEcJype88RVFQ<|HSPyT-LQ>h+L>JH2J@xq(hGCp^F`dWbo4y*Y>M8SeyO3Jy>9r@T9_OFJ8LG6i$f z*I9Bsf&lYOfkbhws;39^QzMMTC2C`8dL<~pCnflFmA4`G)Qngk%7IwPssv-REF0Ut zS?k80yglf>^qv5pXQK#hG-g*Tp!*$O&zacz%N@pWbc-&i@2_{spj2clYm$TkE<&Lq*w%Iw%_@ z!`<;^`tektD4Qr#Hk~dw!Bw=Pe86GsPf>RAMR~-sVrR$(lqiNM^&CSN+F3~p4`Z%^ z%V|XO8cyZ(pv#eOhi%NeY~?75<;b?BJ;QXmTR8B;vSz@gLJN z*Vv3z`Pwnx7$>k{RE?@58?b3SDdV=53PagS9a$=na2lZS8d+J3lJjZ_-CCB|JWj<3 z+$#E5LMuFA3AW++X!@EKLTxr&1&_K$!@xGl#i5Ud8poyD?V>?~BF@DmTr9|KmckpX zMGopR-81j0n&OVUL`c3mm*|5Ri?uMIUP+D$Y$M+ej@2N-if?yf_*XvHi}sv9Ki_6h z1Kl#Haj%|M*k%Ajd`X$YN6cW@VV~=FFAtXhW>Mv1M*4Rs>~pii(oHE9t0>XZLBaS6 zq#vCIgah7p!W~%iXpCl^F4~oS$HA7t<;Q5_8JRb_}(&V z@sloThl|HYPJtDHOcTzmqj;fdB@;XeuEYlV*i6~E_Ak@j#QqC|QD60jVL0f>B)n-VzOCE;1|)#Dwd9QI zdi2uEn=#)9q@0rE%m;Fj0RH(N+yO-v?- zJyq7V77O>lz9qq_FRHAa(Bg=jEx?MP7cr|pd}3k+5OB-caetn0h8d}pT#=l^v72#h zVd~9KDX^+>Q4W+TawC?{yNVtcp(hQ*g9Pggcc3TUEM!xQc#em!VH>R#{v<}ECMa#8 zHb$>VwM$q`%T;{Qj5C_G&6qJN_*m&TS`SERCo9JJN|8>EkPK9|a^9BpkyUJ3%BD|D zFwGW*p5Ve-ozd!hZ7PeE1r&pjI3r5@7X1 zHu|eOEe76)d|>Q0yjGgCUDc zVQyr3R8em|NM&qo0POwyb{jXAI110dKRpFLDc|uWsv;#>iOrs#Gj7{SG;KQ?$xio7 zpB@FQ0!c*0LIprcW-R~Kd6x5N=Sj}O-nbTv6n%}8$ZK^+MB$E&y*Kt%(z=*2naxGb zwUE4?4j#>@)cJzWtAF`0|Axci@T+_G;J?G+u=C%&;n(*L|7GvqSNmUoH9R=j-}{&0 z-r;a>?_bF91C77-U#m2g|1x~<;^y2x;!p81LNR)#&eU{Y~WkI1cm`!m_g_rA)9*^vAz*~|Cun?G;$HZ}dL zh8Ao@urOY&@9*$i;}< z8xDuhcyC?3Xv|($tzl4W>_D%XtI(;ykMUU$o)#-?7zHoIvnnD5X8is(OM~}V8?{S3uja_dvZV zUJdfe*9`_(0qpXeZFKtYaQN4s{i>ti?Bsh7{MIX@!!z$c^0g=a76-oK zyr-2I)5`Hamgig=%9d|P(z~@wrV7a$9Pp8WfI=&x*aefcB9c{9vyyzrsyV?PcFDKM z?U0hH*;r6ns$D|s(!6WJ!$h_fQ=|~{F|S!kF1enOQWWi+)%YHRCOEbtS`f=N6Gqj- z{KlsYBdjhnEizV{6&vnH)=ZnOk*kc@MI0o|g0#w%>KTg*zqe1Md}}6vEqb@77cYSE zvW+FZ%&~)3Fb4Ze1{`=1@ zEejzV^LokKa}Rs?d3w$+Lz9A)k{AD8GG%(4n-^-GPo!^?dG6vIh){Y;1y|wbGs6Y+^An`5&kn38lykGcqq)k)t z0`SFFc<&AO^I?AYW!u~v=6m`6)*AUwkNod~RxMK>Pyo1A{&#Qh;BbFQ{Y2}r3oH8nF!gB1@f&O@?ypn;fhf5Uk{%>%_c(5sW$Y_P*GFiF)J!68M&a6 z8!8NxYs^Sv@OMdgtp!<#RuawTjUk;X-`d(CCkTa+Xp1}&^@LAbiLkx3vy+yVFlc_wLuY4aYq@l^MTq4(Mnub&ZINhhBM@ZnkVi9vi-?9joS1De z90L|H;$?5&Uoa^Rp%)Sg!|w^{K@3}oj1)-ilDsk^tgGaDxw6YovH(Hn@-7PPTYL zD`ocBiYbm^%6r58!>`E>DJ?u}K@=-m$zVx@Dzky-_Z0z&=>Gl|A(2oGK)&#!W^?~A zC@RisZBG%GYbDd&BRBtejf0+^^X3O8`DBrNZWTLa@`4xaVNr;-){i7BS*2Q++b^z!8^__CJ&ibRq+FSA4W-}0f z&;z$BdP)7d;JSYUuKNcYf$QF;;Ck?Rfh#=pHv-jreW2P{aYC}106z!o;OEyk`YkJ) zxqij_2VeIsn4$P*#d_~;c1;T=^w= zb9T-aQI&!|N;@fN^i{E^yl!88-$VyMfU)_OUohoFT?CYY%G?xS{v@Nw9x4OX-TB{* zx_Pl^m|-qUHUW_cuh(%Gj&V>7G>*O>d>6@cb_T;RS~7X2-x~jz4f@sb&Q?ciw;6@F zmqH=O4zk)x30FYk;alCj^~+zm0oQE7%64yaa+gA$v!LKNoa*_3`s7TkSKDoNXu}Tb zI8d2#+QE0hmDi$_4N3boEP@=pl*`iUq^(#4D7C0h81C82lc#D#{;@r<8!#wn zdBK&C+yC@1aKYplDovU8#TlUu|5l2&flpC&$(#s={A1eyg>s#*@AKDTBoyws1wtH{ z)ny~NKjF-tanRbrfr?tlr*zCZ$0J!~3VA(^b{u!a)&9rXneFSWp?Y?97WPF`Ib|9u z`T?QRnwgl-BeHaYpZh!;DaP#A+D(IM{@RY&!^h7)JtkVASKshS{Z8-X4Ii+d2ur;U z`9OWch;09gWWTG@{|GO`4WDc`2ej^h{%CKZ$ZBSj_t5>lcYF_h0Qte^03`;gpR@g6 zLw>q!t8ybi{ZJ6K=1urXpG7w!R40?#e_KlVeo+296u15XfO}(s>*ogCpA(ny31MK% z*+YKrtj6`C!cV>gP;f3VBY<|5bvPL}{=nni8s^+liBGJ}>c%q7kD_P;c1m3vlAk`J?C zTIV2tkW{(1T;*PVkW@Kbu5y?UZ51egXqF+FGHB$Iky4OaXi_s)IzR)%J#*_gQ)bnd zj1;scC6}zw)q)g~QO$5qoIHr=nOQi<4-{;G-x)C4GbK}g!D>RA2FwDi2~%P+;OLY0 z)QKT~2?!d8S9-qLY2jDZxd^mrI#(cA@n&7nc^gE&XLi32nfI)8dwUg{?jyX*lI}H^ zf5jcX{vNNr%NyM1YueMvgx6dz;t4gPeArV=h-5#vT(a_UYu+1nEM8BKrnT_JxB@Qf zw5^#a$iccOF#ZP)a8F(}=Cr%p6nAva79+B~3^N|xZ1*-ycH*dBRVjs&zVWvMPFn6n zi$+vpy7((hGM%7-3Ilk{tvopP4xC{}_8NxP?}_e>Qkl>L+lw)AeS>% zWB(RynjXU4@*QBObA#-!<7Xl?K~~(Z1Y3|8WB8sr*BzPbz2KE-@cGT%zq$L%J8^UO zm+yYeLOu3Z`C3E}-MURNL2@y`bE4t}RU@+Z@zKM9MGuyjyFm{(=wW#$ZqUOAMi0Z^ z3VOH~p@)0*e0Cyj+cObhpNWS_$<~hFjciO6D~WTQ zM~PD@m;wQoz4R73P$-VBJECDVXh{l58|FlZ`BrDi?`G!aLwmf7RpO0}+gsbXeNr9H zjjh`s)Yh#xl+H?=*cUG310u$_`%j1#yGGkhtlM7A>CI2DDWG2LUkGJK{ z4mq0BDKjV1D)^6Vk!P|aY!Sju4cv%00)(UgYQ`$`rqFacA{XF0eTZI=Oa7@q_if8i zI$D|<&h24f{2XHpI%pt9lSS$PYQ9W(T_VS4M-Ge@^cZV@QdS$7>*wa1!?4!n(`tp> zgw(2etQgT;MROw*)5v)OyF@N|RS_*0Xh%V1F$2%s{*D|Qo6IO$IIT;<>j}g5eumC7 zS&BK~VKytq@OPIPNZcj02;>u|8{wba?Pg#qU%#>5#`Y@W4d?ZgAdl@h?!apsSm5hb zt~luz-D8e+47{ROj?g{xWfg5#X=nX-A6NxHqU%)QYhDAXZf^&*d&|A|k@B4#a*D{0 z*P6)-TIEYX62%p9_*kWF%*&!xcoKG%#D5%-?Sr8~CU!h{F1(c0h`c;`O2mXvEJCcU zW}>Z1bG)WQ*j0CSN$;VT?a=lALVWC~A}PSz99)=a(^LrHblz0VBP(7{nKIZl=^gty ziHIxt$3N}vW`q_t3K(OJ-*U;O?A3_eGVJi#?Gh9_is`EW)mM!uul(s>8RBpSq1q;G zrI@~xe|gLIwnV~BJ43o{jsNxDOOz`X*T+22v+y=PYeacw<_=T(XP7Z-sHGo&>Tb&a z+|KXb`NEebY=`N}>cU+c;1A$EO1{_&Fl~3;em5*iPsE#^m;=W)=njReXYdzv05)QM zK@tGyel%C;N0fso)64VkUpyI^yOH?(bA^~~7mDYt(xh!lsxclwPvEx5jlY$Z!8&n# zfDzeVcKt^>>_Z43U*u7x2dIxe$*u@)RTzTp&aR)|S6&2Rz_isOgjFJZ4h$Jmc*0%* z?Bc8P1;9zi@L+_YP``3s5i2Tf<18LlmAH)U7aZq@6-!o=%Neg28g#fwN)~gvAy#$X z0W5R#+Og7Hx0-nhI%%s456J+Wr(&jW^Qm+C*fHL|uR@O3}Z0GxTTxl_tbiS&>X(hT7ShmYfzgJ9mO3?gosmxxoZhst}FogEq}`FXKi0G9ugg-EG^2 zy`B?zutv9Ni&%OA_YryY5SqKUejK_RxK?Zxb@q zg+X~{pK`!4SQR)vACV3VRo)X*CokEBui#GZF;~FSuUvU)E4ULxhri>a$KtXM4^;v; z^5N0r#|+-N{-J1wV2xx21Qfiai#X_C(JC$GL^?n=uVZR~-lAb7PC>9zYZ2q(+#utf z=y=j)AS1UEqOs2xndvv1@QRsoLZ&F0#2<0RGx*}lJx=$(Gdpjni+_fXgs@kQ1Yrib z8&!yj)%C$eEjU|K78^#6iS@fzUrwd-Y+?9*n!eEvojOnc__l#4`l7u^0XCc#M zBY2&#DOXx9yoPgUhx`qrS*9`L;mZfrhS;E)vAoJZ77(s0u~3Q8nwZ2axNE2gisM9Y z?1$b6UUuv=*M)njkS7;RF8mIF{0{in8ND!k*Cnq5Zz(2ZnD6DoFTDWCU?CK7+;)f; zeM6kcVLw02@89vGy%1h<75A*nQsOzNU7{FUxAUgCi$w;pFI^neu-PiJW69)QX18J` zz}OHTD{OuZF)nmvTlY!GYz?e{LNd?@T91jQt*T>D@nT`#5zmD_mQ11iDCkfE7kToR zhmRuBjG+dpwgxS!Tg9Y`SDA2-PW&lPM5K>dW>rlU@(7UuRJBw>T;c%e5~7);gu}nJ0W|At+kVQIGsv1T|W;%RbGpCUnO6i+R63vLb%Hg%Z>J+ zQB9ZF`e~dttl*OcsCFYd9x9x~$RO2am!eRELevFov>G&Uyn{xR8EDM7USzgxhc*qB zbFUovP~9q46k^_d=w27fL$U)bl!w7+xoeQqs+rM_0Rq4cE03ix)DeZPbU9{vqaI~jYd&Y8OUKJjm}E7|Aw>=H_PLPE%w00=C@M(O zz#PEbZbNTDimtYniAc{;XsXWPKtQa*GIJD#uwirurpFAr%NofqsAie8G;H7`p78m& zttxxnth{N*)8cL0f$0mUz75|24o$Vb3jN_E;|J)H!9XO}aI14H$RKVw08S%*$~yiL zh+sO1U1YDypu=xJ>}j{+1<-Khh{Kdo?rjw#mUJ3E)9S;bu5)rMl;Y!xIk7v0^W57d z7yEwLG?567YEd_#h`X2OYzb@rApj?oTe);*M;!!leoRP4%XEVf7XsKaOZ|K4fP?|= zEV+^`+E(c9Ng? z8iM}dUw?m>Y^;T-qCTZ)&3`k))n%+SY?TL9902ztya9-{`lv40nL*#8)gy8^x7v{z zA9H;sQxgiK?bz6D3g1JNhx}uuZ zSugEFw2%LrY$rQ;^*7U^6w~dkczmz&M$8_K;=@h58n+-4VM4g^A`b56dfrryO**Cu ztxYyUHAK&(Xs0vC2XRZeVe#3$vqM-tk}@Z!j9DPdJsc4_78jPH8P*U;{w|?qNrav;Y5QUdV3w9BeM`RVnlY4N zkqn0ixx0U8i#kNUKk_H)JbqSsHj<}+ec+k85 z`1a)cm&YU{kC~>tQvVF`SHF**fV84=uFzVCD)8f-0D&?m!1eIylam)DgVp!cjDQVh zRH?r9-CF(UVY~O7Qryp}Ep(^9T=&Pf7{aAdibt>#y3t({ueGP$O({$ILVOh^OJ+(I+C(4SY9R1sM^8g8o1%du=Cv0x$-fiS(PAgt8vh6R%HraNN zxlOh$3~k@(S+2*ZUHlnoY1bpsPmDM>E5c2R#a)~4qP+^+WZMGkHrbBAXq#+D=ljl^ z8q3vxOFJ9kd~21|D|JEL3i(FJIDH)EjlK_NU)HeF3ZxRhv&iG@f>W!jXI1GcJgTTt z&x4KLX{7r~1!Ln%Ob7eJ{o!DEZ!p{+@OdM}1=#YbObd8nxKU}tGhSz*gUrk|5t*6l zg&8~(lZ@7`shvi5b|W;s-C89IOsq4o4=@LTm!InmR7!QEV)vo!!Awn9dvp{{+KJd- zXqbC{cz@6L{+&=-p~e!Qbr-xuhY1_riE!nPGSLEpmeJc~TC>ww#9N9v0)I2t5|MS9UN2ZnAXBXVRO zAU^jaD(raSH|WLF(~f{(Nxps*1Ne?rPkQ*7)4HXVk3Lm8Q~Y288Xc9cI(EI#7v7Wj zxHD^Vm+8u_$bp~#*?~R#)N;VrtA|OJg<}V~zWFs;bxnJ_^X8ef_7DOR5qKcaX=C7? z(PEYkY@Jo3`?|A}uEpJ5CQoc#x7#5FDJrpW)KKvT55Ef@UV(BSA& z|3GNpB5{CaD-m`k)<6-Mv?3U`q`+QzqVGx@bfr!sIuHTFDY|d4(xT`7YiCa>Cy!|D z!B2}R18E{K`DQ&);OLkh;m}2hNTFxkyjhD9d~^Fp<8ESnG~1g} zzCd`o-d)$AMgq}I;3;`@oL*eRi7pN0l=oVYiYv`(0uuaiiCt321J95|)*sZTPH9|4ND#IDoqI8T2`M$0eI3S<6&B(RQ8FlNqD1nK- z6^~Zi!Vx(bJ{bDJo^l2DheyYTS2qXG6fuZ##w|}9*?yrVzmL<&74eGb@D5n7?QfM~ zm0Di1f$c^Oc9<0pX1wwE(sdZUJY936)-D?!V@(2CZDhrn*}DJW4-%=;PJd0NG4VaNfWp!nGVXUPZH|{6`b8u-diUM2Kh1N9Q@sj7svKT%vSr360QlZ*N?hVh-iL96oM~n1393QCtPVR z$o$3AQ=EnyDjMB)G3t>p*lB?Ar}u%kPYr;j92od($2FH`Vu7W{jI0on3DukPeq4z$ zCggrBiq^Bq1KRL>Uhe2h;fx)V=+@#e-I!6y8Sg!TDj+Xoz@i;de(zQEnchzn6uf3{^a9oWjRZ1g7>H19=HVkefCS6mOC8o2>&Su8KMCWu#|CU5nv5Hl%1+Z|12XH^F-YJ|MP3zd zsNm?7t}!nE(97&upMO@&SUmIwD@9P5)26f7!uQg%UMmyh%8R~`(Tn7huF5+u-kWqT zDRMkJW;p(&2VN^}9t=6LeqTU5ZzW2YTd+mS&wr!cNjApW>REvIJM%X-e)04a_s8lo zl@~r5dNya^vuW5*!@!Q#T9sP-Aj4kLx4F#U*%k8819JV#!NsQBj!b69*y; zGgn8*ZdqoNWbLe*yk{o*BlMzqrH#uQ%_;gAhk`q*lMivc-QfM5k#wHde9q9-bm#r{ zIN4k&Dz7`<%64S{%gbui4!22g9 zYV%?0p0|~lV#ATGA4wvocCY%}D90>Ns&bh+Rbu+ov>m}4De$fQ7v|U-_NqZNqfDVy zkfltyjhg|yMPyt6Y%??OB2*d9={fS}QBR%=XlqbcI>T6SU-J6ucnZ0FbBFNYp8Ml- zMoPykrMiz92DUOx6@i!7L`cslf%tcCe>l5^?)sqY3|32$P1}k}h{u6g!j1TkVfNsk zU*4XpD|I!$n%yC{=e(-8Vuh$nMcjYqylyoppj)Z#bi(6ld^e7WnLs!J44LE(0T?RK z%b0zeiA$nvg+xPd!`b$-h3ccro`UX54ea#tdGcUVE)bNvqU1tU?VLTTD4)Nu&#`dJ zRCwi+sCi@oGPUC82>lcVQ|g&0F#z@nqvhWu*X;YcVB7zUiLDP=m7rhAer{P}l|)D~ z^~2$}Jf2e$3^aa-c&Yo#lc#Hr#h;+p6_U0t@j7vMR@KZbiDKYXG^5hyqDe3)#Ary% zb@`+-pt7QHd0}`4Ddt29q4DMN`1vW|w49*LEQu6D9R$0+MfH31$u}NF)2AOc|3}J4g6QFZNB)gOxBva5KIt}6q?78jwXFM$o+DUrj ziYSdiXj``H)-Z0hH@38DQl>l{;9(F-N4}+3wJ;lid+cOh%6scg1&B7#zmLg@kaJe<685TKjkdAD zDyEt1Iu$8VgGRbi3`@L_;dpLt7V1(&iNm@a$0b=M;9jrj22~W^BZEfb2_?Bz4 zQ()8md71OC_?%mZdvsAr#RP9Lw?fRD7J^OKfd(n|PH+gZ0|(#KIh%uI?Ql4J#(ing zv=G;682n{pq!TyX>uner<3gAg%`=y_1wyAc63DKNX>DWa;hk?PS{Mk4G+~aGdaGlQ zR)~@%auug91Fn{AD>2dAWX@=yg!t0G214zP2cl_vZ zF>67lEtRy^42Yq#ZBC|Ur!=FoTr(xE$X%DO=A9@vBC*SGFR?{AXjkm!gcuAwwGqJ8PX&9}GD}bL6(Sc~O8>cei1cWvxpaQHk|%sIN_n76ra{Es9R_6?D^0>vbyP2_W^)XuVM=0LMRb) z^2w$536cUAW9p;{l6KxPITA7zG1_~<3x>bU$R6!g#T7PU)MZf zp|iae<@*UiLo-D~g1|0Ql($rzYhYuXyyBn?L$5Phms#jEIjuOaL9Cp^gi+^gvF(=; zXC^SukMIRHVnnv#zoED{&%yPJNAM@Ak&Kp6wz7BF443&?4GhkL@Kvt=4ExWb{yQ6k z^-iJMqfjf^ra66edd@Cgbd{ZMHY@jIB?=$}lrt}DMKxCwj!{`2i|0bSJ@XZNvVkFZ ztq?WX0Q8A!%0HlN?5jtj*6fugGf|bw96^SUWfvQZP>mWsrmS2S;)41BLI#zBS~SIy zWh7#+MUAYqRc`4SX?Y0*9=e?JlI#zM!<^i`*vpSa*_~U$Y`Ku7)u;kAf*RsNON6FxcDwIv?i4d~X!2d2LlwBeV9ke;z`u;V3aD%}Ac-{#O&5rrMN`xW?mU zOH{IEwP0zbMIovYdGY8tBHccowSRQPrnV@y@Eo&fb<=8cYHeOlnT{kjIBnY^(k90J z{QzH|VI<49?G&3Oddj1k>kR>uCskJhX9@30CFYJb^g{jH4U=<@oX9RIxg=ULcE>Ek zsTbXLIl~=IF&=M_Ea@y*Vs3_u`!^R)8{_TN> zxR}KI6kj|*`wes0-o|esl+5^aRxKh?j*lrEGmz=<={ki3r-_(Zhmnbm7}0qiNatb! zSLl+B!z!(@<-_uVNzD~|i1#a!>sGwr+S#?kQMA*X|xo>v1oDNLmVo%OGvi-Erp zvS3f(Z^feI^FN!wD zhJqYQYIYep@|dSMBqsu?TEm?~ZXwKm)6(na3cBG}()?>c;Ipl`1 z#mjlqR%LF+bNm*Y-BN}qKroc9PSGc{v6JUDc=|?=Y48sM5U;t7)zz_u91Vaw&+RM$ zuYoY4s?vy^{)q}m&Q<3%M>v59(>{f0tHl}8x-%iCXJ{bs180|3=K+UqD_a@VStX{t zJ~NX{+&fx5%V{m@#ay&&Goh{=Cgj^?2@46y|(l1vlVW zG9R-tTA6_k7uMZkV91$t!PzBw>=z+w|MkhMrV`RyO#-uV2Hi$!Mz-So$fw#Mf~lPm zTEm|(MAR0lf#zYD;Kx)`Ck1zr%zY8@25z82cEGirDBl!=#bz zlBu}$`-!-M4>%7GWXUva%;Y6_QmzmCCA5FC6iPW}$uT){l1}Au+~-o6dymAz@+5;F zrUP4Q2UM0LEJjKDY}%`29j5!g&CmZQFpx^^CHR)iE)r&kXi2_Anrx|=JLvc9rp>VDn{$J z`M!SX`sgHupu2e<~ z;1Ajl6cf7Rv)1=vm>>a9&pk|pr zy%uajx653v_lrLkWhY%!BtQLA=mH3PtC%7)M$5|JFBi7q$0eILR{m&(CUMY@hz03M z5F6@!5OcfXgs}$)Y;V6TzAp9|-P_;0KiPXg+5RCbz8c?uaB%Od;o*bfpgEsLy$m8Z z%zP?#p6(wWW>5F;e`TkKYlu>uSyPZ6W`6~y>A>!44DtG(2i@rk&Y1gY)D(cTgB`R9Yk3B*#f48J{lQYDI20 zRDsOR2D#Twb!P*K2{4gdX)7i1r_1Z zGIW0!y88q{h7N~;8{If)2I^kn>*j8}6(36~7g_v>0O#F3@ZHU5)$rQAF_E7kwDzgs zg2uQ zgND0bzdiKisP4N`q*t}w(HQ&gW3m(PBI)dXy2Ej+3n~W%EoQ8@y`7wxGyfv#HALx{ zmh7d)*1W`&BdGph)yiNZ5^H*RWWb}AK+85u6TmD@k_N#!6StnR&X)H} zYCb~TBcw1(VUKKBAdhC!Kss2(bo;`kAv+kVo5(I+=JRVrcIo&iq#g>+&0s%rGXN)p zednCebxr7VOi0`kG*kK;N#_hw@~0Qrif!Q`F>gd|?qm`!%4{K_4Y${HYiDQ66g_PC zDHQZ6e*+96076CNGsB5kcm}$|Vs1O+JN|Veihab;VLyqng`>jly4|VR(zeIS7Ac!; zGBW34kHym-Ctr(q+AZhz9yrSj#U4x1fP zu30?bo1NLKcjjDL&#V#88TR+5+rxbC@M@UvUk&pE^FQtjS65$O-M@NpwRd>6e|U9ocy&0u<7bti zx`^$_hzs476nbm!n_yxek>vOdz;ozQpT58Xef2EPJL`Cb^i_3n{Lak7ByF(noGnWH zQ|ReWu%~`m37I=Pvxb8fsFo;Qr1!Et=jx&u-Me?t7Y`Ttq&R2ZQ2b&4&SpXx=C^oQUT`Hm$*R!sYo(cxvx1sOO%Ms$&cP9wR%kYgLNjRS zuSJ65a2~1A$2V?L)+%-{-@CuNJo&-J9wf-$BNA9*kWVhvO2#=(pK!}vZU0Vr7PtA& zuHjuk{e^z`(aAnKe(4|6o^x4AaUBb=j2s=KosS9;0#bf%Z~#M~}*1~JG=Tcm)4Kf^Cc(WT^uaATSM5jS18QR()?m)$)6 zKHer_|7W-7v}ss*X3phGUD9T=Lv5rmJov|(-fc6XGnX_@NOc9icxS+#Ug?=VANs00 z-^iP=VTH$;KvfO#tH9jY#*E0IyX>Q_!b~B*+npgMZgn$*$KciwI}zfT&5EY30oTPu#c0jCcKPQSe;^v;gMW8e>fenY{c9|LdU&RO>8~FxM##zIHm|jP;&r2mi5C*X^m!dO-OKm)mm*h;V18SqV}|JqcRCvCom3qS zvXXG}Srh@PPnXiWN55C?%8I>-H7x<@1HQ!kGfbjUIk}CyP)#xnw1VkTN>(w`7dbV2@&SQ2p>VB~(I;QnN1x8`s|~({^-Vb|O4AqOh zEXs0Eq%QXA7_eHm@mkHP`q@2b5u5D|hllsJ;&1_$M~B$y&LmY48~dInA8C6Lc!_ zG@m|_loL6W{R12AIc86M;Wbi3*!XU61C#>UAOSuLnOUNX;yPZCO3=~;3%1E#F^REb zR%nguaEAc_lwH{QG*l5d%_q5pc6~Hh>>O=f%^b3#;;hzL!K8NkWCmH{u6A}hE_=!q zu5*OKR*|Tx5(TYhLTUH0ppYRSBUHq(Mr_vL?j&si_3fF|@6-bnxL(fL;@vyR&e>v} z*Xyq*EdF?WD(Q3JTn7m$7}y@%;k%#yENy{5mK)CahEMq8amF82LJ*Q!Iaa}?P}xvV znk2*eOgM*xSMclk!>C0MK8jj20LmYXz^&4S2FU*-sX#x14)jI|dc!2#FbSVClYjxT zEQ4_D;)Pw$9QZ7|p95pC(ZhdlxPR}#5}^#G*DEG)uRc2(`4NfaS}KX1e@Y73L%%wy zY;MTH4Ow_6vhX?V=0|=GnrS(!WB(*Hg?MXv%v`o2IcJEHzU+4DLw&k#)6b)oNp}d5 zV``JcU|=|pKPAQ1dLBk{jo?b@9@sKvO+1re$vB}B-J2BIe zQQwmUM58Q|0Rw4&WvuZr=cJh1$&?VycNb`Zb-cBrge#lCLlvI_(Kt1M_xVCZc>Vb+=eiC$t+e8CWsmP z&<7aKNR}0%!skRDB8KD;1vkStPsUyqRa>$@|Igrd$u6!MDT*t#P*)dJ4k|t#l%hCi za*%)NOKcdPLH^~P+h1G_?yMhdqB6Z`STxXW+G;UTS7TmJ=K5^TRdF?LCRZ5HVgLWU z!izOgS0x*__}`00!+-R=nW(Ett5~6{t64i`x*AW^6<6ZHS6>aU=0CTrWmmPDs4EOp zvA`vn|C^|*29hCO$)-4Kc*7>@%BeTu&x+UWEA!XSEmgA#gm9UtD+rZ8QEia_21n@~vo5@b|po3qBjZz}yya&{FYTHxw<+ zAncxjDy7UW!UfJ4hp$wG-NOv>@?@otPr)K^VT(2SLzwBe{gvz z>>n?lp6)pZjF$@XCeH*Y0+E46SJMz?yC&cM^$BStCVP&OEgJ@kp`$P@OETe-T^gfli3^xXR8a!D@fqR)bfG(uxu z<7(l$ch0;atQEc-ZCZDm3=r1*H@ig$Jrl~SR}<%} z9zrT+gnb3Mt-bQ>V2IQqZ8ai$!_`vUhvfLlGs5aZm_2^{QvlR@uOnjAovip4nJm=DW{84R3$HmhWqOliFq~X~`71EhOPi0SMM< zkhe+QL8xF-+uJ2H<`U0**E#tw(E{MM?q(s#UGS2jGD2LsL@!p%OlOQjRxfkHX5fWr z$t72_RLX7Exh`{Ck}u_&j43wJpiSEV@-Mr4U`jS+weJTKsa)*k2YE=zp=Q*d8_cb( zDhKpOljlMoOQu{H?l?qyln)Z{HtdSw%(y_S!HOzONb0N--J!zr&BgpljF&kyY#wE= z+zyFG%7?xA`3y-rb?TE)OeYgwbG?X)w5S|s64Ey!pPi=pATv>w29=gs#V%MCtRbn^ zl8;;M!ZY9#FAAH-w81PY;sYT}X`F1K7ByTgNTH2T-SU#`4~K|6ruIip?q2NW$D-`c zE=hFOWpL7zo24k!0HSdCyxuG`>+(|`xV1qMd!+|U-1*uZC^p%NnQ;2yxu{Qs&u>I~>VtF*oLai1BE}MB`+9{UadGSsnn&ElJp903h&y z1so3)CCU%@jQOx#OzJ3b1X-0Aawox=05n}Y1p>v7n2Zj_1!6bXhlFg1R)BHnvl2Pt z_cpEf7JhCJoLHW)$tYqPF`djde%p>dob$RIk!_bH%6{MSrWzojfeWGv(I5>2UIp1u zXUWmWWH^!=Xw+>??o@H5@8nw~4i`LcHRezS(S+g`4DVpc>P47GPEPSEd)!ZJOtnMA z_$O9a%3)&u0NJN#aa_9$B7c1MqFypwm4;Q)(yd(?wFxo@3L#R&5GkKf((4| zCO9k!SrIG4eAmfARR*d%B0BeMB1NmTm_?;Rv|sf^~*WE8(MWsY)1gWiNV0 zYXSV~s*$!lPiOh<$B^sSGHs5(N>UQrih09)=4!Xg@vJS>{f+CHXtj+A?ielCF6j$K zo6!qrZx#2Jh15M+Qa;R+pDqvR=)v&8J)h$TJG>KS^r@#4vasjsqR8zy875YSxq1hI z^dLrL+f540`yIk?ndoRyh_*&#p1eWP(klReK!CsB`Tztnlv^oaW=9=#j5!}-aj#C% zBgXBH6GxD9A1^1Qib9?*TaEUmE3Q%nZQ)Y#MFX$T82DEC0(-Ir4)0r zXQo#*L-IH?7tye#J&2cn?Fkm5E?A@0z_wDE4N1o9sVf`oKq}|VjOvb3ZS`omk0GaB zKo3s^EyuJn=tGjD<8+t$Hfh_m_=|0^9fqi)VtXdtV$2VELhC6@bBsS`wJ>Zayl1%b zH=tw+3|_P#QnZ>8_R1!$)za|51+tta%iscmqqe<%mxgKgc2Myf#r5$5yKqkeOS?OF z&}M@)M9H4{Zpa0>OX9r?6~D;yy@MRqvIGn;^fg*L}N*Xzs5mh>Z#yz(&!`+eqS$` zeiE7o)m3P`!93C)y6-95jWX;eq2##x8iN4S{EM>)$ZOek}gx8Eb6;tvJ$ac0I zr(B8YWUf7Zx?`a7Cnak0VTzKO9rHIf8`=84TZ3D%7jnVm*gTIx6wJ)47!C1V4^Pee zZG>OC83&WC0dnVZ5Y{Cz)W;BjkdQD<-)Hq)XO=Zv_>K(P^)x36Dx;j*z&N}Qf22u0)ncMFL)yTWA@%Lb64HU{| zyTOcy_6hjt`BA7*(25u0&Mu4-r|FTR%9@JS%#So)Kz{|$ zN|3d+H9Ces?}ilJGK|8bqsJ$JvuRg@f$4rsYs(9|Eu9IapCgBuTAEv$h3tmmXB1w0 zL-wc)o@pgPXxxA|jaUre!_d~7i`u^*mPDBLSsLV({vd3^i;@vToO;R6I`%k`Q4FAq zjMilqM5BIEU1_P6M6kt(Y@f5mwoCo)lzL?Q$5zcpS)s43pd}lOON~olk-llJkU=*dDR#O7)~Ji>QgVL3uXH(qrYUG9C;);qNBR= z2~MZeyXhZ0+gnk-zXcI9IvI6$kk<-VDTe}`~mQNmD9d00GaBs~{_bJ)wH_nvw#bCm+XG`!kj&lO9>{7OFgh+(KK(aZ*FB3UR^Yf`}Txsi1IU zox2^{B~iompECz`|Jj-b)+PdK|CnW9TKng+R!Uxcc{Y_zk)P?J$-j8LUXt*}L7bmT z{>5wmxO@|{11S3c(VT&=48up)!Q2K>#~dQsoSSO5k3ro=B%8DO7?lGU{eWCMOL$i# z6C=-sCO@^x(B)=wtZ(wCj-TPtTmdM-AK(qoSMOj z+v9VHbun%)&5E8U<@*VOh+G(~EbG+Xs9@T@L!L8MsuaMjNr1EKql#QI2>y}OTIWYX zR(TY=08nt)_#T!2=$oCU_ae73@g^*0Bcd_vAfQgTHkg``e>)iB59^^!=_F`WiEV4{aR~meCm~!uJ(dn$;LcgQVf0 z9t$MYPw;1`2O)qVT)2 zPIxC)wPTn9&kfWDa5RBG_1CT6T7-1$_wb7?c0$G|)(AY+Q3TlpxF?B@spmpG615>o z5TW5wHcVUns`Bga{?1bRZ z8O<6|hOu$z$jUl>FL6cqeBNphEQz{cmj5n6b^>-cR*#1G>G5CeyRu{z(`a}*W>#Wj zox^(GOJQB{pJ0cs2KNpR9^8*ipC0?k{{<$$@`lfO{m^j`BZAm@PG7yOX(+W=i5#Ib zj=fpR%iLB!C4!U|oA$?AK#E}Alu=;CfBQ4p%lGbskCF9)c7e~(1{@uOqKuOloWiFG zh-_^IZ?|PfIc01Gq>L*u9qbSHhlAn0!Ek@T=ZzE>VCA4PEiyf0-W`cHJmYm1Q3x~F z1cpp?@Dj?zB%`%!YNyei-H3pP)7GY}twSBri|5m^X7Om>38fYKoMPy@3tnQs4qCW! z2c8DWGz~jG!}e`DrJ7yRMQ*jEg_wuMQ9Vd_MtV`?ULGXvkt+ku+t>4n*x z+de#~A~SL6R|rEzuKIIS4crMB@MFTbTqphLy0>Eh4d+76hdHGCHgfNa@4xx>;O~oZ^F?v~bokA8j~||kzyH@!_1%NKSg036-$IEw6E$O% zVz%YuXaD{8{r|rA^A}&8j{oy$vS^?FXPLj6i^1`5zF*{KzxfTu|6mwDyj@BzS)o1F zL$|H)BC`*=5+8Whvsp#Yx}B7AB=0VR%HfuJ@$Iv>N9#MwmorFsyB9OifX(nb$%v>C zAv*|}d#~y+j6r8P@VUp5c>|govhb0}Li7gwNTgwzI7C_l#E+FH5KAN?Se-?XXfd~A zF0OfU*ZxpJCIR?)P8(pe40E;o)=u4fqJftleSUlFb@En1IN~+0^cz(AZm9GdZqVgK z$hmWFoLTf2q$#l^gL|~+dU*YM`G|7U2_hZx=1uncHTlAJ@Vo`h{F^uV%68T@^Ua$C zpY9ObpK~hcn^k~HaA^E7ONaf@bw!(o)se3!t4oiFahTXwz)mQ19*3b?F%{pP_ix<3 zVctJ2mOc%Ms+_^2)8KX!5OmP@s`GqTRIV#^8X@fn6i%%O)7ek;Upjk6sS;d^DTBPM zXoAnAc3Jg1MXMy18`_f|!%YGrMb{@olDC*{CIoP;j*|P@6vdKD57T zFUW=_$_<&Af}4qWT;=_78>$pRi&8pexmM*kpVL>_xDv&A=EBnad4EQd=x@0X?LsTi zcaqY)g8i+15lot0wpWbW@<_}Zw~K++?d*_;$494NNbo2Q;{UNX%}1|UP95Msb+5kJ zK8LTzOkXmz1Xtj3bYY`mMiPt1Q5>p+q580SCTv)nY?$A>2T>C{t_;Ownoq^f)4lt{ z?CIY9gLs8F(W5?7K+6e;i6Pk#Xip1rNv(*{rkv7(NTwQ5m+qQ_%Xdio5RI}FX!g2Y zWEZq*@y(Rrj8NV?@u#5LeF_@GIX!o-giF>cUIk1lA(}BQ&Uc9t_^#lZl)?u28S}a% zbV_-xv{U7iqvK>sRT#6Ip#)94AC_}oKRiCNY+e${6Jpqy?_6mKp*4|uo9bw8!)plV ztjMIT3uLei3+0?UL1PERl+Qu1#v|Bsx76OgGIAd8hr{TKJBBNQsETT)^x)@~N%JN) zw`j0qa7G48Fi(n99h)q_yE%Vj;}=g)@p@S)q4G}xgURO%JY5aQnLRoEL`dP1jZ^x6 z$;Qd`fAR`k_oV^~hA#$70?RM zx3)&VNM9~&SFW~-LiRzpSPX($`#bC_qDBB-M(F8ceVD2QZXAX1%xeU$;F!U6;9$YC zFKYMDhAt8l@(YuK2uUg{4yduj!Xh5EBRpOzHm9#n`7aDyqoZJTc#*!r?NB7XOu_mz zX0RQ=+Z}XQ;vJyfb#>;t8bq+|W|Z(RZoeH_4Hyr6hr}eBY0B!qi)&UBKcI+rBHGUn z?)$KlsR!1^Zq9G_AluB*IXs7#oSlKm>i9cbNok+Uv;DN>^=!*mIX*k^jzX|v2GR1C zfucrKV!9xzF?BrRb-`Vr>34Y?HZ+}wWiR>l9zT>lyNpGEz5HV5mSLbXSsRsv|| zvYfL^XC$!`HT>-`jq5yXF@%k%YgT9{!Hj!L zlI1m2YLw$}*&jP`-ny=| z``nFN(1!ak+!y;1k+kMqRX4yAtrsz~LYy0|a$c1#=!D^F04sUZFm!G1HTinFEKs+U zHOns9IL9BEWOJceb}6|wKMKYTJ20~dG;Z|=AO2Cr=zFZT$H#PO=x_obd-3l% zo8GzaOX)u_NMsmnaGFr@Ht}ub+ZbHVljgYbtnOYkoB{8z<#z~2!?w zUuf0*1~fdHGAOX~8pF7*yLM%2Msar_$p!GK7Ntf_%%Y?B4fA7~jYt{}m@Z;|&bmbp za^D2)8(o?aUUSWvHF@y7I&CPzaCg0wUmphE7wG)>a`9 zaW6vi6os0-FkH#&H})}`MySZUW<71?d$bH8#S8{sy4l3W7}gNqnpj(9C0t*(1QpV| zW3(W5`(~_~N0BXe{pAn2DQKlcWP+5QeUw5ZMIU~T+(Mr7>ZM}pJ66qqz<@;^Qr88l z9>}nA!d`HD{l-I^!4RF^R?R9frH0m`s>mXmm3<}Qumh?@o*>p}V!7RL!=#D{*|rL) z4D1264OFB>*}mh1lX-gXcwJN}`_%0`;N9zKWeBO20i@!2t^b;#ljzR7?`5)yQdDf4 z*yMwqExUeqmYKX83-oP-|&z1KK8}gAj&_;^! z=gWe^?E0@uJnx>Hk{CA~V>j>3XO25|o$(wtxEVN$x4#t0Fdy|kn2|o)t6(O#{ucOb zxL{uZ@!&&Dq)0>wsRusQ73rTrPHC@y-XmvLBNAMG zp8Wmz$;r{PC(mCzd^(B^QE&(hNM|#`*_*G3sI~Z*vn9_3q(UjO%G~UkB&b!vYl0Kgf>2VhT1#4yN}zvx zz(r;SXx@pIW7aPWg5%cOUArI8&TRadv$G!^nx8XGGeC$bj}6-tL()?*C74UFV^cn6 zQRq&>lQ0k=e+926BIyg0rM_in%jq;o9rp-576e$Jpv){#+ zicQ+e@dmFrv-T&F;X%ASd76rTBL}om_LhP@hD4v_8JcfSM6Z-H^7^-G zJH2p}!mMWi?mXtM3aJLA+1P>N(~Qbu#x*N+E197nSt05R#LCK06F0s`>-M+H)&lvD zqqw7h1R5zCI;A@Jl0-aD6Umr8Q)YXORGn3a6n9nFuKfJI%2uhjk3~snt$F4nsu#5O z>387*T78X8j@wkUNX5$$+K$EUqGXAjJTbyCOvglTlR2Zc1%z}g>risHfp{`^F}{ZO z6^OnRULy!OXM~g0`_Tc>)PhV~Drv15c;`D;%w%eIO6F3onG#pzt_$e?P7~@Ib34rw z*eo4~GxV!P=L33lIvriO*13b2hqdP2&^V5?r(FeKRKpWGO5<%ak{Hr)!_;)ox`f*} z97K!V*gAjLB~5n0+s91Pm9`!`eT#Szn~S6^o%2~R+jzTl0X~c{*-Rg~dBT8TFmaPK z;F_cX@%2C=eFk<-s$s=izu8Zv;KE6nIe9G?7M(C!CiX9_tZ*Wu!2ql54NN;X(cph{ z;yVCsX5?TPx<>fvX_9CQqisWy`kCz;OscbSV48C6&NZ!?t=c*0p?~Y(Q}5n|VbkZ- z;IYNS-%2{6H6Zy<6 zi{FujTMijdS?h;DHAU{7?$ErruijOgp5rleIt%N8?Tp4zVmw6kS~Ar!%_m1N7%eZjf2Ut?c9}lZFwp@9 zIRtkOn5TPS@VXH5hH4&#!(HACG8G&po;aas0JKemy}?=L*TJ=YG`P4sG#GSA{NXZB zmPBd8_@>cvkn8CKdheY(W|g?achZDI;!Y4=K>CN>5SmBN*dhQbIREE&=St^ycfIpy ztf8<`;`R@|et@~3lu~>owJ3T(7W`MowG&s5QAf1-nZb>3>D_IiDqDZc$Xc4!3h|9yq$k} zC+ZDJ{aF>9h`l5+W(v9iHs6Un!NvtgqZjG|u#**<{>bM9uE>Y$&X>l)@ffBtuHDu8Wjn zP_|?Z(;WSa(Mo$4!pj&UcS&pQbG94IVp%{bsGa>r+kJExh z=nZ2+SS1Q0W?c@6twg6+Zs9RH-3Ci@LzSg=QG{V{((xi}`1-ZPtU+c_GWs*gQ8}DX ziv~7Wy$P^{@w9$Iw=etG-geB^n-(Lo{rWZZ_vTIV(T0HD8xG^~SzX3MtOQh&=lND5 z5DeU8FAIrGqVu13enOc3n%C1R1bg8X^M{3MeXblF_;Y^pszD{NvtX35$vKi=LsdOG z-XV`L&1)Luy8l4e(M}sXGCU-9Rs^)cw}St5&K4nXMij$67&jn!i6#A(X}8`)G|j(m zmK){D#g*rYD76Khi5nEmyK(Ey&A8x{sD{@yJ;#()PKJDe4o;TS2)nvVz-yaL7rAd| zm}XnwY``tpfm@yZTersKT^X1>@S1yuZe^V(4BSpoBz71!F-g7AeiCDE$N^D>;unJ( za?^*y2YfL6YdlMPwf-$Iy3xwJcPPAfKGc)TkE;3xR>?RY>e1rHy#DR-As;Pn)J@(l ztM}p^o-Uos&;!{CJ6?!dwN)>D$ZSA|k*$=M>>@YtHoAWpqw#(pNDNF{uv(iQN_LU( zIMcHzpvs& zDDlq1q78>#79jOWj@*Q<7a)B=LIB&^eGJ4Xxo_V;4o+TyjDxh|qu}Jtka4mLZblyg zDW^B06Bm7Tu&r?DXCcqaW}&sh+h`50Tm|BP5M-I2%Jt{A8M1T}=;R?MGf|abKa-&` zl@H=bwKc;-dz?bd=b{ez%zaRt)TE(|!eP~-CQ=BEw-uJ!@)9&_(B+huWPdmu=3No> z2iK>@cc3>HeVTgSOe!v3&OM)GpWh4BCrKCoAA9fG7q^imjOR6b`kfrZ{4gc}% z-!V?&XYGVBWM*e4d6+_~?yjJg6sa1_VD_^=zq+bQD&4x9roqyS#HdeQRCVgqxgYsm zCov9r{dpfIi70xSvau(jm+@KV;F@JjPwv9q^Gr!VTxRkmlXgUD0Z1u7Npj)NB2^YU zKcY_*T;Vtj4Cl}@4e^=V%cls%m@Tl+T^-?#41uPf#S*;+#J^!19?l6pQZeElf|UHR zDslnygcU?Fm(NNcgJDYL7#GmJyoNDDQ$|HOA zjq`s?L04YxPNrYLbQhw}@Zxutf2w-}XMPIFoeqWZn^P>FW2AQlV>Kp+>ZBcPiIWh1 zQjDIFKB*#sl8-N97~fc}~|9V~B&_MY>1rep2L5%aJ%k z;-29K4GPUOxNC*!FwZ{n>4tmGo*15s3NKZHMQb6ldE`Om6wPYxFp|n)-F&^WbgoB0 zp@hW<%sNnO1mlV3=g3U)weqW!_MkWwD<|I67}6?ZySu$r`oYp@nNK%e)_FI@5pZ{R)VMi;6Cjm{Kxhz+#ZgOq zVpbdEk71mbf~n{!4oT>S{t-BPbE4iGHg0VgmLcH;lIf9H&fFk{Hhr@(J#sE|r>LMC ziz7k7bqTUp;2`^hxCjSY>85u9CN&QlHb4sxo5vX8B%Gm4^aR+BP&~ujZEqKNI0hs} zXa|IWN;o0?FGPAvl?Ni@5Nu8*SCo_`$kF^g3zEO3)q`2B-p)0;NbmI5@dYN7V10l3 znmN3jP2+GrnVx<~?{vh{Rj_yf^X^xXTxs!T5yZ&2SCK#u${`rdR?8&xolFPK&z|(o zH^Wo}aY5ZlSEP#qB)mic?~Q*cEdGkDX#O*IfJZ-uK^P0KZ&n$Mr`d|IQHFLi3tV!Yx&zfb_ zQGi(CP;YLa6>FWvZo?gm?P&Q2RAYJg< zXJKFjUm^d7>p0aIq8qW`5G2H-GOzkQnY*d8cLI7Iy6OoS60(3+{zWX%E1s!}4MdAE z(r$a&AR#e=vy|;(xRAyg;y~I=Q$|8piGn}B0>kd$pxftlA7=DY@eqO7Pj;C@LbhT? z=q6yhMA0lfmstd3#}6kx?s@TtwzWmn{*?JcyvH+4R8}y4_i~4D$^)hGq5P)xREYzl z77^v8B1(Sw!e}f?a09e)B#e9P7^}lBRmh6FmHNSp8=gK~sEj#PW|h;m zk0K%S6=KtYzET)#-^9PACo*!*15fCPd6j05uY-={cx;ToF#cK=NY}0~dUrv}1`1Ny zH7*~mi=xdraYt~umQyLYaNA)NG4346+58x}VJXQOvCefv=N!dIPeB}o z2@#xJCoNpED{9yf94nt$wb-)ccEaNH=+g8>D5PA8F0XsctWb2BHG-EVpTq9(MVG6^ z19XjIbK%&HYduG{W%E%mAye&pnIWR)+Cjk;ur>_af(bwP#%eyIb)$UCd$uHtuOL4N z4VO@zv+NpX2*R4zP>wE_vd$VZwOPp`zNS(w6?uIaML3vz=ZEfPqLTPup_pKo=}%>W z7&e@G&odLuP&`5AWK$L-A@t*Q(}1FE28JYI#wsC<0Q1NVXO#UitO3wE@$Bb)z(JC+=3E2j2N2S=Po#rAoz@OtiWv?*(mzl6ryT)iz|sN%soe8@7{B3BS|6Dk zJTnYjP3e-vNv))PdWUEokriT?i@ z5_d{_H0G1ZnkMIBrmW6f_9PVhBP%Ewsn-=Yr+Ua#{X=;WuP85tTot=#bi}-hAzeKZ!yp;Uln6ObPTswcNjP_8cCY!Q zn=06vL$Cw`CH+UC_)1hoNiAyX zBRaxcT;j4eSgiB$=CC4#O}$YgbG&pCnNgY6LaVB3Xx}yIBvX5lOBng>u~&OYhf`&{ zRDOHm^{8EDLIAUMo|dy5^%JdIR`rUNQKzWGxf$lytYz@ZbQN)%m4wLjRb2nl)|cTQ zn)b9nAxm5DX;qCa%?c*!4@;`^<~7qc836hpz=hfE$n?SkuFP_}@-IrZ<^ z6Z4u^JVPQ^S}2>eNK?9YDl-u$tKACTL1))2%Ezcw)lp9YnfX>-(blivDk4=^GzM%G68RoCO3lHg zvrK_7aMNtSFc7K&kvofDOvm4eVfFYpJ=AHSlKe(=P!`-||~SRK~aVBo5#AVXPDNKfeNl?%*YJ z@fLioSY);kqC)cl=} zd$Mv^J_D|#JR<(oWFVxEw4aSap?z@_);7As%A6l|zi<`lyA#vlCXDlU;c2Sdk zzpt2k!T-HG0V3%JN>W_|v6l0e>KvS;x@_6s$NF{fq<lOeWp!;7Cag78#ED`_h#O zBUFYXrcfE{3995s?@m(8_Cx4>2Ynd03PrN7PnpL2BlBoAnBHT=BgpXB&^QYFF`F!- zf@yU`62tKryF4JrLmi)0PY+{>q@8Q*rRoi9@y#>d#YyO6_vQ!)13*qCP4U4%I7VOI z$g?00o=TI>2}0_%Dy6wB_iLU(0j1Jfi9B6W@6iEmY_dL!<`1-Sq;_5O`N&u-?G983fFB-{^daDYRR~Nid`40q9_@`1z z{ZmgWo5mR6gy0a&&fcH$X~?0p(Y0!^3uwx?>rdsll)8)su2^_WH;hr3EEI-gRCzZm z-?CR(MQCd7V`5q+UYLYT&%h!$q-@TkE#=qmvi!-yn;g@lYeWm)d&*EYUG0+vV$Jlg z!9ga_)0%Ba1bTi$>FN14kMc+m`J6^aevU%#v%14SQ_Au=5$XF9M0UFWkp|Rr^WXb4 zKM`$re*T-5&a-l*rH{U@<9)z2jSE&Y_o>-bOi}3B(lM2V+y39}?zcP7)!STlCL`5w zRRFHc_9O*$;nfXUHFbW173JjtZ0m}bV~koN$J_$?Oit6>p-(uUz0O=zCbU<2*cqHK zt*xngNbfN{YOXY9j<2`)pJ>1$=C1nP;mfWJaZRSsxFp)*r1xH&R67aX%&GWd%9Cb` zMYYvQ+v=oiuTCl|wZ9K1fLDGI?S4s+C;fu;#Tk9>GEgF`miv1xv6Q6Y`PKC z@LJ|HH(a)5P9c3Q^W~u)Jny_#<)bu_d^V4Koiy^bv&a`DkuS<2Z%-j_wLHJpvVd_x zM`q*9YggSc2qZfm8u`PzhC(iN>|_K{WbwG_cL#mp&pXAFDLvGz*uFT3dVfh*BqJ4V zId;H!BE4fZP)&RWO34Z%d@dY`kHFPXTOW!`mh{Jfau?V=hjTK`E+@qmZi!3689Jwb zW*{P%B-df=2?+Y9feMgd*Os-JO;ukVhX$CpH7+B(G^M4H9KTR87c~_nMGun$~EFu-*h@F zE59XNgExhNN2OeTbIB?5q@}y5itMed3HR|m&y+6oiOga9I>wALunu`aIsOTPNU2zB zD)ZC(SMN>v(KB&Fe!|6^C^uiB*kF#^%BfdgBpe>B)x5f3Ga>L2C3JsWUAS2obb>F|Kt?+oT+(a-nf6JXP?KyL(fw+4Olbo zpew7t$u7mRMxpmjlBQY1sh^Y=hf$Oo8#y)Rt6(v8zcVu-P&Sk6%alJDI|4q-63NM( z)IM=fDoR(`0zTlU17bSa~Oz>iJkmEfTa-=s=&Xf8Z zx^F&;O?|p`ZT37UBpv?EmR& z#OzE*!1*Z@yNjF}AJ4HLBoV7_>4rfGLYciNw)gEmgz-Cs_29M z@@|*)M&D1uB*767MpU!LRPjaH;f+GBYDHO_3N1hgMhfup%Lo<&jFV%*+ui(E#_t6rQWRmu(hsMGQAA z)x#7fQ{-`_D8pmm##Cr082ZW17U2Df9N+TGu;$~ t*w&ovF1tZG*p+yF$G4Dko^ z6`7MMPa@1Ojd2(rJ%>>|c>eSEKaBqS<|2CTUcT>t|KsuN%kaPd{?7mLuCp-S}qy{x4ql(=6S$h??HS;CFbHm&e3M_7sJ%=Q&?+~xV+xkScD zqDa8Sf+)g(;V@>$Dk0GK1qFQeY>Uk`l@w+52Ck_QK}?(=e)V`B;VaS+(Ns8?^kmB3 zr1#U$XKzot?%x z`gL7Mh>yCwrSRPNDB7uJQH%pJ2F}+>&snGg6nMFRZ{6PZzTL7=f+_OFf6{=({b%go zRK7t0>Fm9D@#4kH(cxh0mxOyN+yWf1#G}A*q3?GBln~@OV4HneEsq4 z{HNC+-a5cJd;8(!{p+*0&JMV{+hS?7t2>#*a0~fygQ;nBtB>R(pcY(Hd5|zYCrPJ8 zeSWQRhQ(0JER593)mo4ous*@!A1fE2wBKR=*JVEVgxT|Je5?VePssj5U^38rru9s?4ba8|5fmN{`*#Y9)AxnI03X=!1l2(M@ffk6Zx2ag7|yN3a4J6 z9a-UWiL;F?p+O_A5G!F z*%2Sos6)ASba7eK^Xv-6Z^BswV+5Q(zZ!Ih{cgY0cWkRe$IDH~+^pjH*4jTIlVv|a zI-gpxNwoQ(#5e^G%=IY&{0vC=cR2Hn^^s^I+8wJP5sD_;ei&ZPBjCU&N_tlV2RJE! zL%CMU(|0&e@2D?xb~KEz^n7=>m9cU$+m-;_0z6BWVAdIbnd_sgBF9AfC&%UNBr6Fo zbPH*h8J;x%&#hja$6BNl+q{n*9*9^i5M;~eQ6;c-tdm#&| zkk(z+mcFpsb#3y93S`yhsdkw6Vk(u@T#dJlCzxuuI$tMFrZ+xrtv)TpD5vuR-;4FkkPk_l z>yiNpZ`ob#b0m}lwv}?L3Q4*KxH)et}k2#*GW{v)X`{)k#%< zpp92zi`HzFoQ@tbTFiy8)idl7D_K7yfabBTp#BlLizv5<1!qV1pQ^ihdJuQ%5z;$f z#W521s&-+C(hH0=f{76%$e|cF+^zd%tWAu@>DP!X6TtZo2RP@Bq`#6K`#dt0?Q zcXy7ZhX}@`6ONDS8cfBJNkyQlM(HN9hD!Fy8&*GqKjtA}UphrGCekAEJhyflWR^rQ zpFu9UamiB%l{SDXN+%FpXP&L!sPc%(yrjY}sObf0W)v+{-9$MGm}%J+ykF;zyW!m4 zDu$)%)LSm&?oNY|1A(drXOfwNBmUtm+HWC8V%f0BO!~I?E@_v1<9%wdKg-_Uw#{1g zCar99^gDCO?J;TNQ*o0e2%Vj4 z(r8`DsEch(-3|+V3|K8Jk#!=~9B_9h9v&!46?!76UTp3=F8KmgtDw@Mk?mtqCpof*e}9KQ#d*3m5}A2_qI$O=~+>= zTsgSXWvha|kLBZLF#-wd$dWB!z$V=vVFH{0k?SzN#KEL{`I3vV17`

    &>b9h`qjD$V&p|y%)g-S5~V9l7l`@m?0{`{h*rGN9D9mR*~>*v z*-i?+acq`BUvF235&S{J6HuLcwPp6S-B{T85W{kM_+o4f+G-}g*RxWs|? zE&&b>SS4a7ZDa5bS^K^*>qaObmcxz*>|^wS4HPgS>5>?b=GNO$M}1rk;-BIRp)U1lIEB~{BVDkjU46BRKtp4_! zXN3FFE01qZREEr9OU+qME}cagxyEoZZh)(0TEHc5@esueV@nJCS9SxL6|jz0f2D~Swh}h9!VQ1;5#sbQ3AFv zU(&C=@zKsM2+cOaJp*jd!WhY12~*_HM1$@YpU2Ub$knY)g}#$jCeuL&*0;m0tuWrI z>@3@j-a%ddD$}uM_G57#nTVY%IMA;L$~}>r2>m=}9MKF$|KJPu8>_!DW+8%i%yicQ znw3cSL;#Z@Gbp|=x3}Dml~Q555s6Oyp`)ud|0#|C(>2+~AKe=`s~-_q9{)ES>y67 zu-MZJ8|Q}<6q4szE}et+2dhQENIl33f(_bs37j;7Po|KM$IDzW|B7^@3IkN>E@|HG zbw)}{XMnrAqlS&?XWiZX|H%{%_Yayj=HKXKwO^aRN)PtEe7PS9RyY|BX5F)POdbgU z<*c;O=j1JCjWCo|mstp53Ryk9&Oq&+Rf11`AL!|eRUZ^>F2ia29R3OY>=cCh!yv=&2&mgbkysK32z%lsIsp5 zJIxM`matsCV{m2P6D}Ov&cwE@iIa(KTNB%y*qn)-2~KR=wrx8ToSd8A|9wB)x*v8| z?Nhr}t=eZd*3;cj^S?wf>N3Ulc_^+~C>gRQ0CU!R>r?->G^o|8{R;is`u5R8F-`jq zpI^PUXzNf?t)5(`B|~wK9^RqPTL!w?DPO}vV!4TYkps#TBgZR>R7Tn19<_T*8Z43( z&73(mwf(+t9V3}s`AGFi zmj~OdzVC2dx{V&5$(YjXlRcDntfzdI4T!0p;%eW8G$@KW-jUMV!F3qF$AX?94KN%E z7){cAptAW_oWERAsJ8H@R0v>g34_{}Rvxm?I17{Rxk^Gc8`eRTu*;ifl6YR4<>Apq4H{sh46XA)!{{lNWpl*j+vNO!- zc3cb-EP{o4q`cTI+z;Mt|D58v@TdqbcpiRNDTb1%m_7WCb_A|0DO)V%9s@zQF&bK* zoe!Y-sS|X5Hl^j7go9I;IH9I~U>e+p8>jvvCo2SZ|B1Ym-rRW+a!;xKM}#ruPVQB$ zw9QE3Gu0np>0-I~BF42eQ?E|PEXHAfZ~$4xlY_~qR&zoEk9JPUO!fL;8s|8s2k$WVtpJmr5Ro$mR z3DT@??`nlgN#GJ<|O>v^oVmU{20CQv>)lvIV{3Q;Qb|VWFfxQsQb&QY1HsL`Hpr&G3#5g3q^KV zXOPqx?rH6rwEBDG-%53!SHdocPLWY^DVnBx8GwaS;%7uN}sRVoR_v2Sfyy$74wdbt2AS+J8wU!|WdvD=k%|F$3u6Zk!)f_TYh-zlt zryH`Sguw}MSttRr2E=3-~FNqm-r^vKQ#7QgxUkp!>!=ie1V{e~>k zw?%(J(JW|l??&*`2eGH~`ZMSIMnSOH8PQ1)0?{8SHkwlhf1?7c{anF9UN0erq;P5a zMcIUB7CDC5+w#s8BngV07AFz5abR{%%9d>IPUJ>N7`hnxbLcCi{cH#xQhbs2s(kp4ENkq&Jsp}K_@2WG4grfGG=OjD*20$BEn*8lf}!4&RAhe zk)qs$$?tsp9&9M9V9!3+W|4;=W21eXqIV)=gy^6#*$LVFXyG~BiPqH&ZC0O-{YMx> zF90XCYz9^^HIwFKy>U3bvy#EtmA4N?50unC$+#Fk>>R!Mx@hzp=WpJ3Qa(DOATgP) zVw*VpAEHgkR*g!F%)0X<^rg!V_69Tuzc5r^i=q<#h@SeET&Rw2G(i$aa703;ba7I_ z=F6eFScXHHJliJGhzR|GU}Vq=TCe}*{%fI3WccqOK9DYzs~4D9GRclLN|zs11tEKc zZJeZYed2mPI!-11H$`Z{BwJEzT-muL!e(D5AcdrINyMsdRzg@Nh}citu}7_!-Gcd`PZ|{( z6#vCd@Z#vto0pgRPm<6s6FJbz9cKxM+=-$)sJaShsw9X(l@x&MB=*VnnB>e<8?c)# zKY#~{8SAj=zf2#*P4Xh++NtjG^X1Os!E=Ls|5Y-tv#f}>NAkrmN+rEZvU3Zyx$|L3 z={eS0%=bd2pLr!C<=0Rxh@Uv=GZfk1K&gd0hECB1`e&I-`mEY)PWCBibe`P$DA5U> zwE#k#oG0UHg`o%bybt{Roaw@40vb&;FuxgPotEJac@oa8+vVSWgY$nxk2~X}1ZUFO zWCClH^OBVNGpTcCB&u3c)SWvL512D8H$ueR54J$Uc^Q%so-RWN8uh-sSK7V_6CyV} z2C`nZJO(n&h|dN#Ajnl5@L^vPvKY9MxPgfMb)J4jC7-&Bqi!W1znrdq8~TUvZXMD| z!5SVQcJs@sus4%n_(#|cgbQXtapefU>lTQIK4qQf9ywF;@EC98ToNndF$`UbcVTNX8dhRYJ^Ic+w3*YI1y(ZQyK~vI)Uyds8plaonw$6q+&e+9 zoR4JEf;n#(#jgTK%DuEXi&OzpvR{WyRQs%yXAo3i)Vtqa12Wu2sA?&0r=-`fE_ROz zN;!NILSJe4de^XMAQr&oF>uVM5Zc3uuNb9_b9ap)45jBTyl{p|!>j0_|MY`pcU$;l zCvYX0Cx)>=(GWcj2$CUX*U&WB(J!TK8lBcvP*-Kr0TWFST~&JUkhPG_p|uX?^May|!h2!6B9xz-;Pu+16ftI(6hW9i(C>KOyFSqn zQdnN;q=Yb!D|Gi(V6gDP-80lb$l%mi;V&3&#Zqf2ry^0x(twi^w2ru8v|1xEYiV}^ z0-1`jOq0zd;TaEby#GO0AC;B9D?`ad{_}octm(6FRyCwpbOTMwnU}K$XVeSZoJcG= z6lg<#DW0t}fNTlWY>&#stph2ReA2!6e8)4Y0xNqjC9Y7+=+Ux8zcB3N-6aPTOXk)g z^3oF`nRO8`UIjO(Syhk9&*hrAvmH0UQEJ90F-q`BQ)d}ws0Hj3yWC>LD*U(qvFgYu3e)C6ICEM_o-YFF6UXeg-!Nh#U3AKQ%6Sq`~9g&(c0I_+E1o zxgm-Tew{*(v`asBN9%2sX=V#z03u^3AhmcfL{nrsPD6?`as~3#zEytLCPavJ{rarVU6&LkhT) zw-z41>pT}m8ojj!aF4k-^fdqIOvJT=X(M;Ujf`fwA`16CD}axq*zlarm#zNIIxC!w z)nmoAm`x)|^>JaQuKbo4`B@uiI$g-12yj@L;^bs%d*f|0H&YSCRjKwK&dtiPn-0lZ z@(r?#;?=a^Inrdg94e9q!tU`{Rk=T+<`@AKBD!V$y`@(K|2G8 zbS2R(F?I&X;o)3IebTZ-m0JzMzrfeo;W{YK)x-+pm6Lv2{IKwNwclpZLB(M}W;&;p zlUESa4T7q#gK8G&#S*i^Wt%Fd`Rw%j*kDn8k_*D4Z73fbnNOvfd5DK zQH3Y;fuSI(M83m3#}N90e~q;D08Dsh2p@OT+~W1Hs=fu^gFMl@RGZDQC=v&vk{-wS zW~fZ5G0BCt*}2)TrEe5>d*$EEmp>K{L3mN&=$8D~rPIb~&v{=)aYZx`lz#+TzB8ZZ z+B#W5|8CLI$)`^m%PK9Hf{JCsthJk4@|1S~KE<^rsK`3N0*Ga!m5h@l%=`AFJnfL? zp%HUdu_PY661)EcbD)IIUoD}GJ|`rtfxcCkz+hWH2t2Bm4yJ;QdO@%%+^+9ks#8^cZ=SMB9gF6L4_5GArMtQbD zhSw-g9MoRC$Da~aKf?`OewW~LuBE@nHz4YdJOUU>6#u*s#4=L%m(;o-jpsbv(4U_X zp+HeRkQr0L5GdmMiJ)nT1c5L(Iv?sgmTvsEY-Ad1IJ8%rptSa;^Rj~pcsr`v!Uwr% z@(|`q%2Zxq-}C4O>ch|(&Kn(`Ap+GgC1Y);2sn1A`69+W@e@L8<0)k8|bo z*S6uGMSAYu?(gRqPG)#yus`2<$?`XLv(Bfx{PNu1cjE87+en|OzUDp5gLJRKupa|k z^D04ZtgDU}pz0$XHlWrkx1|aa*|KaLT{S$1D$o z!zGvM0r>^3^KjMgNiJonnf43@>Wxls?6$^FI!P#B+k+Cb=KoeJB*e{^ZU1MQa zP)|Ois4u8yuPyK0h@O&rI7r#g>t+T}>?L0rRp7$L)4N#9U6#Ub+R>&++tggwTwd&~ z^m0U2KdQCd+_!jLzQ`Q*`F9d`Hnzvq0vaspM1wwU^5-99xXa5(Bsfycr`*iQYjNKI zqZoAKz~GWmyG7Rt)0Jhl#OTu;Da<+@cv&Q5#3w6JxvLpfU*n>>EPj2a)wA)f8eL42 zgS8bNbWMj_jM9E7{}w~tgex0z(3aN6O29P8j&qfy@qUM&7?DfI^CJTX*nkklKmuNB z=$I^Og3-+&Z2}B}m4<9>gEP{h64B0B~*4k5>eh|gJ&y!&HRGBP@1Ml!WkT6#FKYI=R z3~CvCs-h0O8-VokR=dIr`Nj7GYmW9ufCbNz@IMc=YNa zQOWGBCkQ#%VZSN|CJuMhxgOt&I?0~s5BHHg5PUqW31sqqoU)vu{i+Y9QqPRG<6 z^jaS_w-#psD|i517Drge{mx2{j)F3WlX9b_!4&Zi;wr`3&Q&;QoLf9v^U z*uDQ(*r|`kpI$R7#Gb5&w5adrDq2XGtmNY zq~Wo{P5Rx<4$9|T$@Oh#6HMI(#D`~oCH(FS0Nv>3LZ5*`b%L%>P)1jSu5U0}Q7M%1 z%N?wt**hvYA&+j#!aYb`J8i|4!Tko%;@o!h>}LBPdxF9LhI}R%9-e|e^QS=FxmJ8% zuMUUQyPKyEit@y?30!C8K86FL!~dT^8JF6_WJ$Un=<4SGFd>K1P@JW9C4X{JRKWr@AJ>O20XOSu<;fXR-NPmt1F$*F327j2GV17G4d7#j`IF(!H zRBUm`9oN+Bd9x~i?kZVqNRnZ-`tGFI$t)W~r(` z-wvQL^WB5`Hhju?KEnCHX!}S<=z>0=6gsVYFk{C4iREyK0h6!m&&>_NwGV=){IMP0 z#c*ndn*2?+7vZy~CvO3CmGEW{ENc?vJ}lXod|>_-WejbNhEc14|KbVnja*nZ6%n&!5RTEdY9}N_$MkDkZiVH_Owd@e7R@) zfu^G%Z?;;)lO}g?u60d;WgY%df3de1sFqw2Gz-_fpd1!rzoqQ8Cf2!KwDo-F?EzsDLI?(6=9KqkeSHq$MZfIe$8+kS90wW*|93j4iTx zpQkT279W$z#7RuOQ*aMaFe*8i?^IIpHXGU{XD{BdDJfQLDy%p3m{EAK1MrIx1fp$A zy09?9h`N|~mBR=gx<z`rP z+Nlm97kk3Jr$UOOIMmj|UNC=@PWUV`JiTcg`-Ad>YGsPM>c`4$Sb4IFv`D2eAI~T} zXT7i9d)GdDwj6y0Hnto+1Au8$cWna#u+tA4dqi1ZuBa4F#4=egmQL-uAT!h&>mVpL z_zNO`<#(nx>sJk&I}unr3oMm4-wg>xf2YaKnI19wfafK`-mrL~XWG1H95MCDq>q2| zZZ3lI9sT93RYX-qHY?WjtqFq|;0$(<^@fa(=LCb&^U^%WNUo4|bNPF(4Ci}y`OKB8 zljjP0C=c9*e=d6?tlueC&Z$f0sFUYNtwz{FbWucP@77FE7>EWwR9F8_9JBUW=iJ5v zL_-vxLKLx#$0NO=EY|pgqD{l*I zf`Q8}$Xr+K1A%h45<-u-I2FNMKX+S?2R={DdCBEZ?{8!+Sy)`|QounQFDw^K->l&h z2qxk6sLW8~ZZDU=wC_IgLJTQ(1itYx4fv_eFUI2M3}^N^Au9R;q{@~*=)mR5mgf|2S&)?*iXW=K zc=IT(m<>FT`ghod8e=8tmR~dC_J`r-ff>A_B|k6wDaDo5@B@g^E*CCZP?(}69R-FS z3(YL20Qm$>)qIM){HC{)U~hEsu{!EGGFKd1+8uC`p!J&^RxvipQ33u-l4quo``3_@ z`h7h1p-cK{6M%&Sswp5;`&DT|MuEcn3@7*eq&79QuEkDn+F@Hw?sMkctRTUZJrq1U zzZKJPZiB=Cw-R#v2n56}>I#th6vq3~U8CjuS=W_!j|*6=+boFK11Fg7y6an>4{4k= zr^Sn*+jfCIbN-&+G_)RV#5q)=qgx^-n-iNBUHF5djxdmdQt&E&)4%2=_LLBcqBZ~`@WKsZeO5;LK^sV^mL0_QjE;A}OGYX+3wbd6GGIq2ACwR} zQN;9$|AATm;4!fFli1j0A&We4*}06BS|(u-?(Rn=v^% z#O|`3aQg7TPiAUP_immA{73F3z5fXld7ySgcumM>dxU(j|h{$8Z*nyP`zWO?m9IGh{n^_D-*@SZaX5EM&rujExa z`5=Uy%ch)0|2LBE1Kg-aA^YHwHRfD&if_DLOZcuBxt7YM6}V+XM23p#6GlWPU@fcw zA+hZ}jShF*UDK0b!YPC(p0_*72LX9iM)&+%hX(~XI8V|1C{+`dj2bt`D>b&rw?EDR z%(%^LSrDGrhle={??x>ey#K|ARs%&H0)GDhiGy0DD@HPKsBjv=wo@rK+VN3ak%EY) zr^yK`l`!LrB@bs%C)xvnv2>w|Am)adYZsW#B^E_+5J^sJe>8pYJ2wjo9WDMfN=8Ji zAI}>MHb@@b3qVv8!;e z1Vu3jV5Wglo!&SbHp6QflBgeHE%6UQ76D7FOI<$+dGm6}YBMgKLH>05ZKw1QZie4@Ipzg|fAlIh<-=O@ z^kcTGPeA4BwHD~T@Gv zeU%Ea!xWZjF8f+K`;so=*j~6ZvRCycp)$HaN4&!82Zia;$9LJ!vOym~OWm-4EnmuN zxvBR$w^wVT--iVz`8qSEFnAmdiE3oxhyvX2qVIq*7moF0em_*Y#hh+<7K|udO+d4T8R}Ow3vBQ$`4rb%{bl5}o~U zA}PHkwW}o~(Tsm-FBbkn$s;}m?jFOT3iV;#L|&2R=g*^=5N6%i2DJJ#VTD9#%0Sg` zj+kOIR64>|BUN?sZwV(SCyh6*bi|*{4`D;Gy z-~_-n9pdVF&QGOE&k8VyC&dI)$*l97OS2XP)P|(;+gmcxU|-1oJ4_9S)F@|w3uNPX zoMxLz=tp51fdA20Zao(O?U+Kcf`cWbi8}gnD0&c*()_@b>y=FJ(BszBKfGJ1GUSr zi<*C`RFcvM;>^G6;hLtJO5Us(2d_2UXA`i2q+{o5YLw;MQmJWyrArW*x*|@3HOHB{ zM8@dam5}OPJE17GOIiR;$kpnTj1y1G-+r3c0nHX)Eu`3^5#(n$lCeTSX_o<->9fEXpS`(f$RJJbJU&s?IMUN!yx8arjwRl+=ikCS_eV85PhfurVnR8iMg< zy+dg|QajhxHyo)XDSTuaSrG<-RJrnh!<~2X?^#U%> z#WZw28ZDvvXQ-f(eR+7Zy10*j0NW8hC)bKlsuk@IVdE*!UI)f;(_95~UVjh=m5KZk zl%Lx#KQzOSc>R!?S(Ku3vOk4*lW0+OSzxF%NvMvAu>C`0l>P` zNQ6pqTB8{JSeCBa^-_my2(6Vb8QNgP#S9mLl4gF$a@K|!~dpW*h9jJf)~S9_;J%y>wso_&Ua zxQ`tysFnTZIab@-#DsmOfyL}CHPE5<`ho}1S@mI6_{sNXUdWcLt>FEABw=PEj(FpL zL$6{I<@ux);3wQ;;n%@=!HT}$?kYf(-Xa7ehCzUZ9)24dU8w)_kEd!0j~XhUG+U_@ zr1NdIh){JjH?RXa zEE09-);V#mmrL++0Di8NFlkerj0fPWhdQij1_aI7>I;@Q z7YX7h^vS6mD0Qtg^@=aO3hgEt6HMIeQK84O_!Ez6h`tZ6zUb7l3b3TY;v;+X8Q;3# zA{i2^5}V)FC&LzaI!$$iPs@@dpz>Vag!JwGbs+`P>qj25(>*Ah&RB_JM{kRVWhGC{ zK%clm)sQ~7bWe=C6<d6>le>uJp~X9xpiPhGQhyrRfvvt(2k2NaAJ-U9)!9alTdrkN*0qaa|ol;CkV2 z=YE3;Be6KMf+q$!5r+o6+}47)X6fjDPnsY>esPa7!$&WoSX?xyL^1G)K6_Wmke&~Q zEyY{ds)_to%7QR;wJYdkg!E<{BZC2c|A+IlCM*<&D8T0N9VngnD{(9M17VYO*oVR_ zZfE}JE#4LBWoB5*0MkCv8U7NN&|E)(MRQj{0jFY5EuqvvS7Xybg6=ls+p@lShvrJ` z{Rsrgz(v{!S5_~8uA6HJms7WZj@Tzt=o8qn(6B|ao!cbk1^x#8ZEwQ-OhH+qN$TG+ zoTINUg`ZuRQ3`tZ<-C2|tt62k28EzSlrDpy@3Q4D(2up*&acIKvy|+D9FcDkHKdda zjS4z(3-E3Q>v|D&3ZNueF5D z@cbj&wHsG^{GNpjZD&^Rur;X9*Fx^z{6i`abOFMC52Z$F(LTS_HtoI-&D{=d-|fAX zw(j^vJH8LxygXmawEjHZSD}yip1WVlXe4OC?C}pYZ8pf4U6egLyzql)C@~+X!xVLe z-s&@|{}9^;!M^Yj4(~#6_SAPcyzp`(0_%q*Z+~%R0@1up+-x;PcVzY`tq0>gWOKB^ zb)=-H>ym?z%HQm&kO5%^Wh!y(93SZPB+6rl&V)@GcH%yJ8 z7ib4hYkl~onI&h68jcr`)WubQlDj;*9sZu7Scm*#G=#O` z1cH5e))s9IY=e9GErzr11q$!R*H8Qj`Kfn;^xnWWjc@P7AGjU_LUx+B=Ay5J|L0h( z?`^sV3awvj98c41BhUT1f=FsyEJ7vL;}VRuRVcet?N+kEcF!Ug>{|zt$=$OcdSO-( z7Msfyz-LRr{z3(+Ppn5TWu-v`u~Rcbiv(u_PanXMWLtAIvtak~kmIKQql{KW3l5x_Cd?q+Q)mgjn{PuQqnX z>1!C`z|+?LcPg4J{nYEaK}8$R*k!>w=(U@HBq5`-px5ORA=9peQT~b_zmfW7JIrU{ZFV!0~YCu6;UP4*AcP-y=^GL55LLv80<5H0F5@j%l2-$s)Voe(8bu}Syv&= z7$+H&&gplP2^#xVb63Z3*#!GgUGnWWnwyQ{0m1v0HtIt~NjwkTqZ`2My*=hyxI!_) zsaB3W*Je+9h;!!KIYovN9ATrN)Hh-btX3XCwk<@nOU0E@>qim_}QzaWZVo|z{1`xF?aae*$kN;MQJpN%w9JfpwE&hc@ zdB>j4A&$NxP4)jf{@4%CA1s8oE`{^ztoLUa&Im$SzJ?S}({$qj*PGk=Qc^do!%A!( zE&A>jyYal`*^i;oSQgzsb!qr|{|Q9?H+BmtOZoJWB7&(jU@L9Dhs}jEgLDWDqjWdl zEA%9$tf{WbJlIc3nT-V>1}%t|?eVl()UJF0+G`115{n*LJvRCO#D6RO8kO<-4u9aJ zNNd{?K0M#u?FN1a`->J3W)0DdQFvuTH7L8Qz{B1A-%gBH5!(Lt+@@P-WDn*P;$vx$ zu;a&Nhi%?Qd9!ve!nnd^AD%_J^7V)~2f@km|9^45M2KY)>X+~1Tfh3pOQ!i|E-aj> ztXYUe`W5VJ(m3;qn@1m)@X!_Y;@?v1!ds0~HHOHk<>;K?VZbR)4ElHv@TMVv(U@}; z$uh@7zzn~d#qSgBA!&84|~x+M__- z+$C95Pg$%}78fR}#rhUU=^`+!pXmkyGg?zqMJalU5FXI*(-Jp_JWrv=8TG>e}X zV!-fls$`6Q#T75Z#$y;-W9ueTy!swQQ!L`12*c9?YlRIv;+4W)?#cfiimz&pm&dk3 z!Yk^vg>PPB7!kJ5Q&u|x*O4JVmT{*2Gw+_JHYRYo`)b(}#{c3X3D~3&adWA+gK}9T z?Jv-%fVztWaD>erQ~gLcko$r$-lFszhM&ucR?95&?wn>m*BiS0P5cGUt$NzgR6eoU z0cEbfS!ODNoZ+r{v~-kwn76&kOC|T?fpp;{kG&Tuo|+ZB8niiHWo6%{DA?; z8)HF_uF#B7TwGiy!R-fT7+jwe2CCS{wTTYE#zy>ZV7E`E5)!0m^!_v>5-YqJp}dcn zyqJvAPbHQkLV>U5%^I3fuZT6Xb{?nWjQF{)o^V)S-%AiMVCMr=3o_*-^ygX?!0YBc zO^_Ps(*o{o_if)Y%o>eAOkzSD)V#?he5r!;y)x8|f5#Iff-(~PJy;iNFV8Sa;DC`L zd{lCgAxrqm7CiOB>yV&m9RUm!bn!)agCYHxdw=9J?ywMjrsq6~4@YF^HWG|iEJmSn zRgv-~UYv6fevL|k0g0#h~X+`ws=ek zQz#);6o28);XbH;ysd)k$$5RkWCD&n*;B{h)I42Zg^fg>2xiHTKVs4H+PsopD^NJV zo=RuCHs6|I`8~5;4lDISinK(~1ZMpqDZGOHqUw5wSHeT|m^H{Me)6a*7jM zYwZbD4WpOAdmG=I#fo<@28(Aug%+I3A-oR3*nr${#zxobl2{#5c5r)KW$}OILVBBx zH`}7=S`N1EHec`?kaKTny;JkP!TObE2mC*ok%$C@P*O_yl?+1rt!)U5Pon7$eu&Ox z3_lp&TV;`6;+wR(!!h)ZJ}67UnmH=FVc-0#Z%rn0(#nLr|4IYIVPLX@A)CKC?xh+c zIqka}Aiw|DTq%0Cc$7Bv8Ky#MIpWw;7TarnvBlsHg&{T`8VdEDp)FFPTAD^>3E3n{m?*}T;LCeOBrOqZY2zV z4mnx)LV*jom+q0pzA+Ed?@;UoE+yt&k3d``*iZx3&7B?SP%&>+grSg*lq{ooKX6Xts-Ss{&nU?QkS z@@XSXkJ#`Qg<;a~(={BDp0>y;w@B@%>?}C%m@G&t`aWTYO!(AG5G|T6%oQ@mY(cS^ z%*mBN^+4Dj~>BoiHPQGH|=S}<{f;fm-9|sqs5)`T|73~8!cG2hm!U&L(dl~Yxy4$msSCf0$Fd`a7j!7k3-kx#2EZc<3IGe^!C z6>NA+LP*@h^OK%~@H_lrx)h_;CRuu&XwjI5`BW&^-ThJ&N2uj3py|c8eD9!c7*_(P ze>J0BbeMkKYyD1i+8KsTezR_&*X*J+i~*5Ke0C#?h`sy>s60{An#ic;pRJg8uJ+Df zTri9mjI*^Unp8riC&2z(Rpza);|Dk`B1i+Aib-|%-~`$BA_rN2SpIIDS79K+mXo%E zko&k0LmIT2YRmY*ThWyo!x}%a?fE<(4(Rh`Is!H4!0h%QLAqg0MRi=&XLN{z zxL!@OB!%2LPLLam(cQu|-t6+d7qKQ(wHgODqh~Nc<`8QafmTeJa8c(Y&0HUyeGki* zm_7$2*sCl13)=tzVa{b+&tmw;Qm15(uS5?&5|Cbn_4f=jf>~a0e298LXCM^g%FAD3 zUBb>lLdLX{Tojb3dIOgQXV9e1e6s9+w z-sOmZ64}lO(VkP{IVB1|CYbbK04k&$SE~{#WCIi$;fM)V(@QPUPk(Feih)2MYiqJF zChGs5!IVShQ-Z||tT1Eq$^568Qyxg?_Ta<9V%VF3TeKJa0zn)Ld+@pK(*GcwWV_O%cmyEHM1;Fz3bg_J&+?xl%>=^@Rbi%(x~uFR$Y-c~@Zc zPSejY)X-gv^P5e*%F1>-Itha6!I+gonlZ!lU^sRNlRy$8P6!hOME<6JbOjXjwh$RgA1l6Wx-fotqv@LlC&(9}^K#;Un4RCw2_H-LyaOSdU^BE^>B- zOXbOW1IrEZ>}^L$7zkC~TwCi;rW@@Y7DQat9eV9?38lB!MaoVUIyKeSjyXOFl|NeZ z`715{D9;{UcKYSo{xUl2(G>s~tV*g!C3<~z zP{Eur5o`P;*S2!*lF+SQAZet0+HN{$E?A)Z%O)U>#96HWpCxd$QC=v1-}al4x@i0m zYJcsgXf5+W1%pc}nBdnh>|h>QX<$_ygd;%B*}qAN3my>Hd}L++fo+H`do3EAB0@|` z0Qzr|YKPYiMzwYo!b8z|K7;NZ@&)3qVWS5u@#w@a6(65=K~^CT4d zj;Af|+_LkIt{-i`mn}Nrev)NeVdMmb*k9Hfm3vn6{DDBk{0I79@n$^iEo@++fS3fu1l-Br3wY^F4I zSs?66+?RYA04?)x$1GNAHbO7Y&cE7lFkha2$qLJk#8O|cXx1z&vlVH);IU!>o-{*L|e{tHMbTje0ALuhR>Guc?hQwhSQ}LB=4($LkG?`Q_7bCD$17##@Q9wYrvr)%(CpY$F8Oe3plW! z@@1+#5b!-8dn@jwJmwlFB2Lq%1)_Prp;>sR;@=J^ESlqUkdO4v_Q-92N5h zJZ{V0SQt+gztnrLv?jKGJmsK<+RicCQ|hesD3F2kf?GK%_DWMqR)DHp>!!@|_{z~S zsHD4y-3$&*@Z7eXmc`X`qfE2&i#_)uQgYGO`Smp~{+T4ZnerJX@3Q{jXT9a$7QC-R zgg+N~VTY^nKUA1Sf~Gu6Tfx$mU$#yo&DRLQ3bIed2s;xAnHB$v2;hScKPKa*hoXce zQwA$;XcrxVh-+W|J!Es<#9hcbePv_?nKh!aT`TF@tQ4Z0yC_MsG2gzuegB~>A`(I! zJKAE430bR=wVLZ*sqpVe5Yv zzCz_bPL*pM`S&kyY7-?TEO6tCvpM4W^2z9nwMrWH9ndcU)`F1orTQ`^_s^Jo!oIF9KA3POZj!nw zzn)kD$HjQX+*v`IZ8m2T3<<}(02#h9@bs6j{&LMqb55XG|Z0QWhj3R?DP=_;o@}Ly&7f0H% zjq-U#pr~^2Kf@qL_{l{ZMkAnLQwWDV{j$**G#IM-PC;q&Yxl_tPKy1}{Y8Vu`nxWS&@rlD z6nM<#m8{DPYrD5KUb!8$Q!uw%Nbk*- zN$LYFAZb9E!TUz_tmFvMvL6H&-%aa{8H_1~pVBjWhKpQo?Omi%0I2BD41vn9jM@D! z9oJxNlnq%=s_RQubtXnDb`;T$R^epRv?2;w*shhwdj-hl#g5-rEg_s;FIO#k2rR6E zYdQ^r*^VSuzxBOe^iDOL56!yNW0NHq0h?(gb*c1DO$aeT`T-oHeo{p>!`KWY(5aQNbwG6bH_Vjmk#GLrls%KkDok|yXH zMBB{FOk-vokD2W@Gc$~7%*@OlGc)s;nVFfHnVI4GdEfnXul8zxY)jQ@r7A<&p;T4I zIVU2 zH{_y&A&JNyvb_<|ePXe;ogfjjHhVH<{;hL~7%2wwVlvE!Mc|asRouVqx((XNKPITp zS$Z$Y>m+`2VXnC_R8er_?VW2SN;*T<3rIRaa&XK|dUAoiohfV~TK(JRt?wjor7-!R zZ%c9wo*Ywe{mt4achSwq{uUXcDm#X=T5&K5cebX$|F#emdRrBfYDP^xqG*BicW0e) zC|o`p##LYaST71?SrO?6HxIA+Tz-_!F(KJ2S7DtX(>bTGHL{&Eo`O z<3Hgb?@xFM{8v6JC@@YzyC=xW_dMp;5AQF1o9K_Ot*i7eqDZlTnN78Tj)0wrfKM|v ze9r}DCa){f0H<17Ys z@RH7D6q}s9bdQ1nOBOJbDK9hFE}DguV8X;4(r!Av2N10wgr4Puo<)$)jL4MV4`0_9 zC5#*h-mHWzjod9rd7B&YiO>!wfbBsA`6Gcpj#{=G`wcqME&~~v?AXul_-%*1P=F7S zA-YkLw~Wi0GkpR~+S>iQpb6I{`rwa~SZoqLha1v66@o*Qxpbi?`?EE|<8`~d`wag~ z<}OTwSAckxZ%#?h8>HAZ=v#_@f+O_`y1)iewSV$H=TiJVET|zJ45Z1QHGdS$d_uqSo<(xRcCu><_(9iO5HT%Q5#mI{ zs78SNi$Y4f;;(^ku%T{$LSu|-5CjM8=&az*Y1a&`X;;dH=gEtL$M=YPoP@#Q8vhz6 z#+9&nTfhG^%(VFK2HOWzc4$Yqj~E)Jj*~)MtEtI1&?khoE2_P)0X!Q7kp~^QddS1X zU(CM$QmqDN%F=Np?S{~-P0VWf6$KrNsk3zJyYr^$wy%}skN+dg^qty{c9W3javG7Z z<~KkOTe(@QG21s7!giHPo;d&etT^}kkV0aGJ&bGSK0okEPz*x0hCjKp1h>4!atK;p zzBlCI$*^$KeU_M~zOjL|5+H@C{4~=_RKsr(9wWh?boYg0hhBjSfdL~pOU{X|t|u{H z(H)F7)_CHtlUtwY>eh!E9u(hYtfO?~oN2?3$}-1|Dl6`H##?R-a=f#$ zZFR#$yeLu+^6KLp5AdzdB5`?t~T#RC%A<~(mZ75%RN2p9ELVbOI+*} zSLD5EJ*U%XmDiTC2)L{)-h4$#hC|t+5`LoIjlQI@5qMAjbt>(BWj&1wWWF?2FM0p6v8i zeOef45|Vt4tmq4D)X0EFC8GEU5e(ZXPA#M-9Eg$`wyXDjCbWc9SVo{Cyk}4u z$ZjrqR9d?s#C+&i%Ex^O6mIb|z4i3%tN zXZJy9KwN*s>2_;kGz0P$)xPDeMZmnp@ldm8($p~638}x zf8}M21k(b9Mq2g$BQz#zUm!_ohXO@Vd8k`Y7Ly$GBlt?4frk)Nm}`OGb5rHXg}y*f z8ekE;nj+_)I+}6}bOnsQ3^I~B8meJ61(k%$P#RM%$kQM^cL2Z8xxGbvpY_}BE>2tu z4yutbb~$o@Z%URWL;!Z+{O0>L{0g32KN3gi?`2}|-i+qo8urBCs?a7^-LSC>orts| zXdn)My>Rt;dLThKLRloM1ruT_2O;em12~}R=RmPZrZXOIu4CA%g%T7mb1{UD{JU21 zNLaMM$Uulvoa^x;@Wp>@n|JrG!oy5SPoZ8#tzDqj34Q#99LnaeLcitL+^Srf&th}6 zkW0!~ReNJ^_<)qN@);@h>d;g2Sluq=)v>pzdYprNo3fRd9&6&yb76K6v`q~}9t`9vEGNT-(ylNa_M6qs5ikMr2XVj#B0ZpWr{c#lz9f*t z!l8LtFdeL!%eNd%oq%|}^qS7V@jBHsju@K>H7sik3ek?o`6u?K9csl`sC*J%;jcws z!(KFjFW6ZT6i+fvIO0zp5&_Tey%Nf(+D1V-hub#WmokXNId=uutBeRGAnltn0*1^= z0vTFLynz!+9aX86Vg731tk|)rC7vuX15=H5!F2BVt@lEQoCyaXc8EfbDL(gmH7=BfdWZ?G7y`o*x%8V>1Y`*lJ_u($Iyy%5S&Ltwxu4;VNeY~ z_3bqIQ9?jMV!sprXdv@nqHCQbMZS5L7XcO;F@4$403R~C(5{B1sjuOy-6)L|og-XU zPC~{?GO>a|res^xdi>Ya*5G=^l$_;8G7fzC*jzLb97O5?e4gAK-yHCA6q5hkF z7(Rikd6t46)MaXmyQ=qps7=Lg{nhB=tj$Kl`22n_bP&;ADz7-aJIkeREFk zIbX~1I_S)Q`Jppjz+&_7br!7D%Ax3#m(U(ehL;gerRB{9sIz}Vh2q6JS$x94?6JvR z`*vM_mohhAg{(2-pgp^Bk}paAI%Tuj0q4@02uT&S%wZ#*d2$g00lt2nm^Dva#%dF( zhYaOJ+ACa(WyRZsy!{E7d3SisIgNXc;T4+i+KCg=5Lt(bwBIdLb5#Eto(1l>H~rH^ zQNml7@@ADRzb|#8r;7>-k3Sj9z}w%Ras$?g^%{dT*Qbbirte*tWIUMP&bEvA&No+(pM z_N8O#&7EM4MDvt5p2;Aw%bCG484XN!lnXI-To_oW((~a(J5y`#;{_^rK@_axAf!Haqdm$iwqZvJlKSKL z$N5s~$;Ln3wxug%Yt0Mp^JRA{$F_}2jV{K|bLX}@pT}3zj=MR^j?a~w7mcU={pP0v z4^K~T=SOg-rH`8#G)Vk9$_?=*mG|agp4fWlPQK@?=0F2kI`j*dW1AWHFZ$27KnMQj zb4Tn>t7z)%z&^UFLdA4@Nq^^hNJ^3KLd9a)l^1aAh2o(nq>|#$ZJ1p5 z%e~$)q#L_$u|v}KqMmDQ=@%H{$4tbN2;Y6zhlV1F_pTU8nXO&4bcAn3BksREW;)IE z>KXRu6Vn%(a2Vve&g3gGeenL#jx^zvc(G*?>St3R=;Wnu_OLts8#5Eeo4LU|C*lm; zgD%%5B({@k;}pjD9XP7n!1`KZWM`)^YdU{GX6%Vk-w7M3tyf4lOBmEhzYXQ?9vCSB z34c%S&r=yTDci4CddG+V_t@09KC+qlyp<4(V|M#?`;*Y_ip=>BL7#Z#k5~T^OO)h% zZ|(AKik`yd`1n}wGKE4a%q8SRt+GgLX4#a8l}v9z!QjX&Kg9gR?HEozy~gJmlvn>x zXkOXhI#UTT$JxtI@tXs08Y|!w@gYTXsDhq^I0Kk39>_Hp$*SvZ zshuVMT9PwbIKVO3bz^*M=nAM^YalW0f2XtP)$0n2N&R zfNr9u8sfxf#KpuM#()2nd7#@gYSo=Z0d*m~)HC2?;km5w>?6uk^(j)W~v!uph|ld>y`BLeWi4P=g#ZGC;TKPX8VeVrc z&4)=c6j&Elu->`~_=_Q(y2FUPOq)k#(kfY9#8f&7f4v(eV)WzaMcWgcjcT5(Uy& zB#x>Jn7QR)r(%z6RI;Wx_X=8{qIEz8);Ke4gb&Gui9zsMpI_$VcXh24%D!= z^ex2k_L8A7zV;z{l4;JQAi5hAu4=mB7e3PLRd=CRC|e z0HR>wkm&SeOL9;O%yJEYnX%oY(1bJ;Gt%|?UH{MO51a7`AczV=sVEePDL8j_HM&HS9JZM2wnt3beRDH9g2H$kvq>W}2 zAS!*@KU0O#wy8RfYD58#Ho6m5=iv{`i$s@3B`@s$F%uf#wNrhxo^sfYJE(7hJk3Ai z+d8u00aqEqwdNcrAS_IUO!EVJ)BP6c_y7yD)K*x%)p!o|H`sN7Y~#-#(_mS%J?KHu z#&YV$xh}#=^Jzm`+kd(N1&eja86J|QLg z4gr^R!>j2qg?m;OQa{~U5Am1x3mU5Mjqf#WN6nvizay3m_Wnb5yMKNszYq_!45}K8 z7R75KHUT}q4uu665GrThxR*rKMoK_8=}Qr=FFjwJGE6v-2SFB9P{0=HF;@)>&V#~1 zJv~9LtRb#?{WM`7Z~->%eVk+<4Wa*g)_qEutA7v*aRG|Y;LXe&AP3%B?ns}5*yh?} zrXiqoZ`ftk1|ti8N&FegyleEQarTr>9w3IZF+q%8*?nPc5skpgm`DbMh$-_M7tYEX zeSfurXn@oT`BoV3Kr%c0@tJ%lPW>jfJ*f)k z#F%5i#3W&;DCEGYiX=K2sVG7}7hS>cq)HkxEd)|_T>+kMJe~?uh}&4*jCHrZdVnrA zQ)!ScI{C`gE*T)mGsHFAHWp3`pLc18&zh&BOj-R~V0B1#&-l>%JYc$=Jn-M4K#`+9 zV3Y?idOQWUjor=sZN>;N^ljz|V(47jy`9yU+oS;CA}I;7ZqBvtHf=BgmLr=Ve==cy z9?gv%hsf1kGSpk5!-wP_2!5Asy{#-sMJliJk0I zB&QwGx;LP)(Tm>s_syU0ILd7SpMt3AA8o4|mly0!cIZunf z{f*MHe+T7|vdFuYe2ae{ocR_ztGq#pK!i#Xm+9Z4*$oN#otNfFiU6Ha3Q*SEFIvdB zDs|=Q|2zAZC+nj3<@!btJa%TV$~VyAZf3N;<|HbEg^T^z5>IoB+%EGR-g3vqlkZnP z!qCGCQ>_vbXXCRxA8?6Pe0Ue^W6ezxU4H__H@lWT|5r1z{?B*y?*97{K@z^mVsx}8 z*d&Yek?<2;;48t+IDm(6<}A9j352Nex5;RFyFe=>$TKg3h(Eq)|I5r=!oaBAu8*83 zrtOhyiNKlvNi4*G)H-ELtt9%_AGW_K|9CAH(n1(DU=URprV-&Z#>F^;E-*2XkBU^# zN37?$MsiPle0RYh1)7e$B|M8Sb6fvb4wZT?_~(0O%KPo^v$FVO1DIYN>X`0!^WI^A zc)Ia^yh{;lqy7XnAJSUka?{c;)k@3@G9Q_Mu!o>_9Cp!q0($l5)&7{aC}v%y300b>wZg;~>Yd4w0MXbk--Bzm-=H z?U`z%xmiA;VcMiBCzPUv3#7#)wO?@cnOG>YFeprJGO8y+;WQ{K0(mSAa)A zSZcF+kj)f=g1;o^$1QYvoo~k!%N3fi4r|SpHTazqQ+_&F`e%pDUFYNs?d44OCSW(M z&<3lEuT(>pUf5)?=H9BRy7r@(b9kO4AOoGZcIdV;bEJ*p>c&p*Q?oG%?`XD(a${F5 z*?u}Zt9l0K=?Tu#!hX;(k-n4&)txPKk{Vdri}{cGg-FPb#~@`WpYMbqeew9*P$U?u zm}2mQkZ+*F;On`+br`E473HRlX zG)5RM7&NEfc=qK>mk)DK^jE>vc#_s@F3UAzYsb!Q8+G1#?<*J1a6VatPf>DN_phcy zN2VL`@q3%`mp3OjFE42G8~uPlkfC$n(~K1ce|Cy+LLph8j#OD{a_Kr*bn1TibEC`8HPWbQ1Az>@UR{rdukWBxXSkI1sl+CJQ??jtxfs|Y4xiAVn&4S^w?GI?FjZ*{FRYc8+57QH zEECTZFvhP_q&34buMX`-?X(?9oa_|`hn~zDyRCAlX8Ua{;25nvdb!3)U-5rpLE6i* ztDy$ieq`-%FV3B~VqBGQLpub8mcr2=Xx-Z5m-le6EXMcnj@ZCA57T=O4`00tw~=w9f&Zca{4(Ge^Zo885fYba9N)U;jI6Ffoo zH{R2sh@MV*mi%k*i|BrqA~HEGVwcRKVP&8Q@n9ER$0_hq-&$6;)kX7YH|*v{@Np~E zjb`WJfQ|=uz{%I9G(Cga83ya9(%!v&GrDGUz@zW*|doQL{k&ozTpqgOpehsh$h;p0Vv@mHv~y;BUzZ z-gZEURki@6OB4*ci;q$>KC#9m3R@r3P*t(8o^Nn9?a{k-YomO?s&MpFk_Mh0CzS+& z?cf17ZhT{j>)jZfFTeP3&M*AJ)DdKEEP4X&7MUc5dN`+7^lxr|i;92l|i-dB=zw_|n>t4d@z$9N8rgF$hvzmH0H};LZyn*mSRcnx#$5S5IL-mfb6xW{u%$<= zkM!Zu6MU{)1CrT>F}Lk)Y2>-J4up6{$)BZ>^|MjlQ8pfb0)wqG;FKY5dlPJhz|h{e|Ybby+pW8sclV%YU??=hpoY`{x9i>(z<;ea^R z+`j`mtRsyE8I}7jN7+xFKKdDVM+MRVcmJJZ;y)jzFd^4Mx}qVL%`lbL!_(!_cWNu7 zZ2Uu1wLg;YkW}T~Q7mnbMD&lV7$06tym$>B+04&_XBnNfTFZh>!uXu5XK7d9veNLc zx@DhK>)c5`rA((HmbLN4>6Crt?-DAB7p6|Q)^U5Hq%pWHI`{W`af!v6{HmDn&FI5PkEd*;8m}>Nz=cuVT2k&}2~bqq z@jiWhcg$ZxA<92MfIB;Z4nEz(L?kfJ-LOVP`E-^+CqWEL1Ao7&j^2-sq5t(!q!nEc zdjZo1jKF;D7?6SqlaKwzwJG2~ESzFb+M2EkW9mr{f@C+t!#y#i_Df^vsZ`vkX>JwF z4sxeQ_!Tl;gp9UQ4pLnsi*hG9a@AptVPM^hk7#mJ$tm>rkq-W)rNO}5qoSCR6HiO> z$fw%syK#qjVrQQr7^!%Nq#@<+Zp&LvC{mhD1IKe>V+>%i;g}YSn3Hbd4IhZ53>!dW zRNe^$7PI^+YCo+2CSlJd7TjH`je~{y8ga=r^HWZU)xbJ-SZ%ZT?}sS+=|8ZV|ELpp zJO}!QL~-FnDl2m*D^Vqf)s2zTUl$Dg>{X3F*~caYkh!99KdTgwoRYGhzJcpd=OyMj zMjPmeM)*t|?~Z7SI9HAtxfHae1d~A!2cUHQ&|%zO7E7FM@Et*QVnTGTJ>?pX7Vw2> zaZ%Zx_+(lrYb4&&)!qpcXr9@YML$dGH&T+#=|`p7bDxI}0TewPIbeM-FDTKb(ArWo zr;}2>jq{iK(S%;0(f)+1qz}0?|CP9Mb){Wr=)0IiH)rOfv&cMt`09n<49|1`72C^qpdj6L`QR&wdL5le3H54--1xChl$yDQV8uAcKKDRb+M(nz@hQHRXGP4Cfw`6$iKJ~tN&(I+J^Wf#it1J?< zHZzI1f>zZe`=-Cd4Hp)QYej*v*g7qOj~LNFH?5Utx|ItW){R3|FbR7&p!HpP09%)p zxwBSkVS5fB>9AIZG~^$Ri76#T!%Gfj7<{ci~D{m=R_pN>7qv-E6Ej}`w+oiTl9Y?T>8Mf@T z4)OAdyn)IQe2MHP!rupZKva}h4o+Q_bMzV3K)lsLC z(_f!bt*X&nwVly&^A3fL3O(=S&e;>SgY)z*ye{Z}=uNs)2k9E`4W!ADvY*%{lh`Vq z)nye-DjNC)9MNTvW2|i@tSu6gK1j;wL6Z}2O5cq&;MQx7>HpowV-n6rkSZMhq<=4j zvwP{4Yuy&6hZEEp9%tDyPNb~=F+$uNs3fo1C8 z_U&BCSmSB(RgvgWa_itFs{A9-#ASl0>1gARa?==Xd;Eg;h+_2qyD0-*y5SLWmYg96 zmTLFBiP0zMWjoW|)u|wo}$<5Ak86Y;6_R9wROFjL1|X1^cH~b!p7ugv$h6mr1E!rN3_;Go>f< zF@XhTmyU8aw)|%n_A$Gf&WUn`x2H~pOZ)y^#=eLH^8^@YW8YtvaC zrk3*AQvzOMIebfcUe@(_ZxRGga#Q4%>|_y;9EH1Le=lOg+c+C@cKn02x^(@6;c%@# z4_evR4JVrw<-+d7Et;nU`$ThgE?a}!o>kIS?C7tUHLXjnE#0D(Y<}j<>wer4i#BfQ zNJ^>yoK#0@UGBb4hSp#(NWX-i$CK3l@DEO~fd{7KqOGRP;ekrzad5q2#s=O~0?+^y z79bSiNp>2Fr-{DbO*{Z;0Ja)tHiOq>tOy)zqB9%PvJUn3ir*LyS*I>dkH_Mvb*;)| zNXOa6OznF=q(Uo71^Cf)=Nb8Mau25P&a691hlx zpdlRg7LQQl_Gr~~`WhYHph=mDkaTT#>Z)tzwM&c_?G!0N&95s+fZuAMzthp1(VnyL zVnA4V&t{lU^H2*Tbkx{H^k?7&jVhn(f`%zTs7gvS~@}G>DUVX?0N)tZfvMXrFKeCy3uXNbuf~LXJ#~p5NNeYHX2R!p)jX~s{Dc- z;14WcwPL0;(Tj{E1wM~p$I0nDN_J+*!_L_{E$6JN+?0+8PvcdqHx2_@%fqxsbSo=`DYIhP&l{;l5fW(=sG5{CIT$4;h32H=N*oUV;> zMA;MGyDl~T-x7lKd(kn~+&$+e!YJ|YNl{`qOE}T@nBF+?Q?;Z$><>OBkKgb(-AjR^ z%($20%5EJor1Rp!Qp)lnBK80uI{}7>Bh?0~K-i_+wx`Bjn;0S$wzWNc1_En=(oErZ( z9(09Z&d$eP1(7kTk_7)syucs9)w9k(s+)nXcc1a%la2YgLEFiBf=LZ4Id8YK#XDAl zoy*S)QXKi_F&{>n;Q&J=jw@rrF>p=f)3vSMfVNbQez??)-BbBw`>hpyJH z&+F7f=i&4fYj%qlqI3LlO8nJ#aE{DG_+pLs3XNo=F=$duuVaKS(YaP+Vp$X zs*N+3%4{flvE4#$vs7nn+uqf0jHmGbM*FvQ#Cu->t9+b2y%%|Z{kr5mAc_pSx@`=a4a7_+H6M(m2OL znR1iUheKjF6^|8EWdY-%V(LF}7Er}q!v1%uwismv${Y*>dWJ znQZv^EkfdXZ9{b_#&d@B<4d=maTV(y+%Knt&s(e6wUc|m8ymGtn6*39(tHDhg6H!! z9A7>I5y@V8&V!16PM?U8NU-DJTK;rn z&}K*$A&D>zFXSs~Y1;oE;qUYx;SW^z9}9T%=w}5}GDu9Ol3bCagN{4}X7n99}R2u{duD{A@<8_hV=;L7zZE%DPI^ zQIUJO^Ecqtu1@72vG^5{IdfWb&a_xqj?nL3SRDK?)GnhxC%-YDns3bKax!6by?{ZJ zmxZ=_{VcVRCZuCD`uRVCzuJB>-P$mIJ!G6KrB4o2vExyDl1Iyan~B>x)rqe7Eg45O zPp^H=LMp8~d70xeKoQS&UQ}oZjj(CvfuAu)H4vVt%~()Mtsy;;E@Ou!i0hXAL559(oQilW!9)(-~$wM2ze=+@iyCIMP* z2=8gLQbY$Vpw0@2PF4Y3%pX5lH*D=325wvizS98_<-{ys3htLeT9&c5okEGipP|SL zr4HHeueefeOw!og8}JJ)NBSt=HeC-3MnRX>)B6(H5Qu#X|$)Mg(Q{ z4GHMLXwI?_S_($~_eA+*BB-&@1E_w~(OR;--F$3^`k!EBz>LUJO&#G=74284kspBC zG}?NOZ+fvD@J%m*8j}}-8>{53vDCr}9Wm9-z?nG_AdYlDRrsB>;439p3tA}3)J;Go z3#x;*Mahfriy=2aH)EDI7%+7=UZkq00MTsV40iu|lmFxN5x5?4P2zaijW>ARq1;Pt z#}IIl5q4P#YWSAs#_zMu3EY*RQ_U zfE|o7Kp+$}BZ)a|?^$kAw{cBGfiQmb2Er!t(E&n{bNy=n$#A)#PVRe%#O-b^OhE*+ z=6>B3vjTGW8lrF0Ps7e0Ftix06R=^cDBKvWZ~IN(_FMk5zpl(hL;O>~+IPJm!FL_j zv4JZ+B38{SgK6&*Fuq2f0#%s;cmiC4td~7qwDGED^s?PFT&?T%^u6d+=O4@$sOi_rZcE! zN$01scg~+b#N_7vY95d&ST+%K$e^l8yuOLFVIZ_>ieRd`zf_Z?zSg;rh&6P9!h-e~ zV%~_@I864ru$(~1$c)*F11b$|y$)1zdAf@kfKQ2Pl0t1t98To>^UdDSV*1a2<+S&G zNXbzSIA((CL=LhkZPIPjc`5Q^q}Bs|XZb2Ef7CbKlNPa3XD(BqQ;JYcitsETqy;RZ zCQh|R#?|`^WIf3%qwKAQH~|DBwgnqq_=l=G4I-j*_;Gw(@5xW*auD}+rl!|spjaUv2N?LPh%up%nm#ok+ zjL?XB=F;YzDgk}TktY%TPOGf${k{wiYL1$#pP|zK-81eoftEUP#SF{=KW94F_aJ51 z`YET~Skw2Op{XWdC^=$j&}5lY;|1(-N;Vj92VVaE$%VY2c#F^Nw5sRQ_$^&AohPVD zif(lTf3()=RVEoVBOqMwB@)jc{tX%*F}cabI|jjODR1fS!0-MKSiF2M zluWdnZpX0B%ex#_^kMJ)$%uoJl}KarQUA=!mhQCep8!{nSjsU0(SWqSOaeL0<+cI~ zR&nodo^t8h+8~SQthYJm0p~lkA4E7{q&fKL&iD>A9t{nt5mF19*$ zreZn%u9|Uu81o2^^lWzzd45ClGr-Rc<{Bqqu;|20Y(8NPtk3U35MT~5j54+Pj^R8Y z8iuR=!gaX+P-KLP8)A}#QaNC|vAM7%>==2wF&uaMad5}E{TysDNo`iXLILd86UGI`QdqadlRO5i4CM3bFOYezTOMN=F42dWasCsAem_VNbIlZIa!u|@ zIg+Hhb)+=cF#R-m0HRh{CGIM$pN)xoUX>hj)`e>LN#r-NU_#kQ-o8Lo3G6LzY#QWY z=<28}n0X$}_tr5amPsfBN}xfQ=Ml>&*GvYdTnH#jjgZJFn*F+neE`EM6^AVRHxteD$! zo^lu$H=pFt9*Rz*gO?JG6;5!Pk{r z)C<|B3F^jqw`e}Ys9fs#JG}#n4w4-#)`_H3!>KQ;vhLvco6)(j;lr8xdU9k6ZRM8O z0RIH@(|tgEQnJIAtM}Gto=3(kx1uvxAUzz)L3OS8ZtIJTIpcX_|;!5=$W?b_gw$W zB2=!NBIKD(`Rti(g8(#dJ`Jz1sQrDWEL_2li5=X)!lIT+S7Jwpb&3pzD)6)51JHuQuB-8AC_Kx(elk zB{rBl=rG=W5dLk{2GWzkgClZE@5TL6j~y)DcT#0uM&#&r5QK6PV|FNv(iaVc60;1D z0nN!wvI3IPOc~`4K%X;J^V?MnV+T+tJt4*zs&W&H3WT?XqhoDyoZ+iAjP|WuFc{tP z>Dzbj2hjlH!6@V;aQ#PaCa@>1LE<7z)kt2(@OrOPa+at=rmA1oAxM)olE{6}?*aiU zCP)I0u!g?piSsgn=ikea_+AE798DsQ?+*YF5T_^!h`Tdphm5mrrlp3Ks8xSrAQ{na zRRF|)Po^+I@*ROS1jK(&29JlcNq71zZzJozscw(U6n&t{B$VgPaTj<{loWXQw&A?8 zncSZYIQ#xH1|MM7&S3kxo7@7I8W)xVmweytpe~&RF25JR^xpy^j{w*pOy7R!hW7o> z58cf_HO@hpi>4~c1`jIPAg9$gxdDGQE&#Y7Y?m}<+OBIAdz+3MjCyA~$}Y)*uMU6{ zz$}~!KmsZzE9M*^vDKv24Ja?3sx*4oJz;1K*T4h3@sI$VSHOHb{ILNZA&HkbMY%1R&4+bd7+zn#!aQ9yOqC~B^#J@Nl@g2aF92 z)OWq~2CKfMGYZ~!r62ob4}8%VV|);aUq!P;LfOT$%z?Z@(jyMh=i^B@sNzRM`ShV+ z^I(phH}&id>1}rMdtw0}etuz&R~VvH+ax?8p5uM{zS@QjGbSKNSad@KE~MQ~VD;Mr zDD|)*VE&Nc2+*36FwpK|#lx-@?^?igRy%_u=HqTxlGN-QJS5xE&9gMn-5CH5(( z4x+c>`Vxm@|FaUi!OGaJsS*=3=klRSx&%_n{={YmGU)SFF+z#`FZoG6Ybo_pTQ2Jl z>0+#0DTiyb{9VP1;TQ~Xw-s|YH;7wOE9u&f92?|e8S;;;;R(waJaZ#fD|U{jx3Cr; zJrl7reG7h(!!kG0yHTV@5~f6PBMtMI6rH3L8$rV#GKp5qrQVSKO0u1l{&xh>Q3a*a zw#q+m7!p=(fTet5f|jW0{k;1gNm-P=()tyXcS@e_&g;BB)|&Zu-yb-CJ0vaDuYR_$ z6|&5;Bf*H+=XypnlS{e(IlFN~dB$?1y=Na6&tuU&fTJ;*0!^|IAt`Du$RQSWf-%m> zYtjdlh|3E1e>tDlcU}HTj|ZJB;nroRO#CdnKVB;%x^nkYQ9G?}WA9Sism)OcYy}zW zMG$ll*9?oOke$NDf$<>`K06kjgL<4fJ5)SxjU_hOUJ3_IKUDjdAFX+N_Q_DSHpVPJ zb+c@Av%uw7TJvm8ngx~8nAcsDs(J=rkR;vKL&dhe`rj1Ie-5FV_7|Ou4Qq&0|Bgz| zYp5(UzkB}VoSt#+f>Gxu{~mCrl1Mc88M>?+a>D9hN=_13KT0TpqymH&{FgY7Jl{Hy zqdG67##4W*oNFnk=f>4;fQfYD zZ%bf&&cXB)8&*QRO)T0_6LayUt3e{iA6(lL$sbrr+RX)|j!E%TLcyCXQzV{3LJOt( z?TeVh2N~p5{VDx5@#M5Xikv}=L+lPUh@K*kFa;b2OIPoHv~c=7ksjv%)N2u|*X3fA zv!|pcWuE_-u)mQ;SaRmI0SBV`Z}r+z+{(~@>NTx{v2lBzyaAlCo4(rIDQYm$YqP-B zUATb9lpPdjJMRAhRkZ#Ws>`_u{T2?VYet&1ypomfl;* zeZCgZ7AF;gvqXL8FDcT&1-S4SBs-o@A2zA9)(fXm(cU zR8&&juB25g3Zj*FBK`b-0=48j%c9c{R;6rtsE$W`{(bNcDc++z7r3zklyGotNSvJ` zQ?hbXr-@BFB%7@t+hPi;ZH`Q839zXGi{~MgiA@vPs zUS+OKuAeq|e6V~CJrX$o0pe{~H+d{xMeu(oFui9}7E>O5Y{u(;oj<=as5ezv)pLYw^9uAMV@ zZv6ky_Lf0$t!djRPH+nv+}+*X-QAtW-5Ymz4Z#8dg1fuBy95ocK|=VNy=Uf~d8N*; zQxwIIMNzAtZn`hKuUnwh4;{H5$UN`LmsKoZ{ z#$s4TGpQJy+;sfGWLY$VP*1x@jiBxEzXZ&fe+Zb`4Nhi~iPyidlOCJftG<&w6-qq6 zb%KVMm~^pSevWM(tjqrp>Jd~J`{%Th+})b1l-uvW1x!(r3|MOqKawj+3$pWAL0c+D zM%m}sHowr=>fj}y~=PK6gqEAvv35hDCLNL$4+z193d3llDsz?6Zc(c8%Lpq#sk(07f3upQO2c(?@q#WK~8F?Nw- z3>uP%q*N1wXNuj%_<^SH8MV>p|B+Gq2D$h5v=X1eYA)q%)IX+`jQ?v|NobsN8YEh_ zf<#N?>C~K!2HHwFyfK1W>I}+9)*;4S^i#uJfGd_>QvKTp@wYWso?C#-6ormMc}FiG z92Wt#&v@&+aOrhr-@TKOyqXUEJC(P=McWMbc*fO?*2hSuIxRHeIzDNSuaqI?o(G{w zGBbgNq&ji|tsDmCmp)jy<~h$G)l1~i!*2AoBJ%U!Y?QPhg~qOZHU zE%*;e349r*GV=-LrM4BClR4BQC>|RECJFU@u%&1+{n7?6h0npguby(*Ar=opED5e*OW}kh8oUL=rkFrNZ~;nkAfZ+Wq5V(g zo_HVuU9H&!H4xuo0rZVRJPeZB;z3dyJ_kr@!&eE%*YF`DmJ*H!;+LcX@e$lOgyY$I z4pE2AJVW#n4AeaNKYU2Qz^uHSIz3R{7R7@2x#OH|SAaa6c?|}Er;tBXFw14SwZn`- zOd~uOi%)DLk<^AnEj-s5UL4(S1fEO^EC!P)Nd=QZa08R!f|FVH<%9>iUDyZ`hheA! zi|Sx^j`2BqW^HfsB8m_kPt(r6(?7DWLj@48L}Na(ew#bPC1m)`oFH$4XCq*2n zfdZARWr8Jq1Kj6G>WH2O8x=wv%|I)nsQ&ny?2cQq>I^7j6@vW>S~BeIdDHoAPyG}< z9Uw6eWjTshQiP~bW`Xw09}&IrQ>Jpv#Es~&;dK_H#j{qhrD{P4{ZTBL-O?GBHgUL{ z9BYp!_Oz1vBRd@=VvQ-xN+Lt`*JuUVcD-8Mg}(IW&Yn=vn8}@h-DYVuraqbTKN(P$ zRg};hU&hDU!tBp`>&foBOQqG(EVXeu00;%76w?De7s1aY0Y_3yRGB)U90CuSRa-<8 z7Y1Bi@KeCZ1xgCT$lhk1w^~*Ac(rpOQG+MP&QfM`ccD>Hr`hzx9_c8a_P_xOXw<%} zm()-J8SDC7cc9{Xpi0%PF{B1ZJ2o*em+`&13+@Eb*>{iGGU##z^)w~FaI*VoP3zIRa* z%RgM`eDl!VwP32I5F-A3JYh5T>~M@Ice#~KN8)$G#$G3*}rCjG%ZNm0A?i+nNI5D+^3oTs0gJo|L@SnPED^-||=I7sD{ z#K-pd4hPN1@3GOCtdPmrJkhR0E_Imx0Sgb7mVXTRM4efBawjf;yUfRIjWJfXl=nm+ge|MwfF5B_CL zh)}-+s&hMXwbf2_GHM^VIZ7y`I=UFQFa<{_B!B|H0H6hQ0v?7#svi#XfZp!-!3H49 zMu~+Kc}Vt$Rp!NfK8_?nA{KZN3r$TOwn<`+eka+EFP=ayH6Fb=2hg{37jNiTTiw!l zW^=D6x?8>S{DkDEJicoBL4uKkQRmotLIRhtS01-$K&`64jvlgTAc3U$D^73oc3co= z>mZGN;mXj$GZ5Lsv!)&U8<3yBv8H`_^e2qB*G(Eb6^hc}RF|@ol32h4d17NwxtD2T zvx##MfIkf+KC&Q94ApBy1-Ze!OHjDdBSp4#)I5nPUxWRWic5odyqoIgg>~~~K^JkP z$BS%LZ6SmsF1Nauc~4V z22!vve@rp(kGa|OoF#5tqsfEY+<7o|lT`-x{6q+|O<{5AtegsNW94F=qhM`_e*i^c zNgHsq%eyLz-8VE4G~pd;QF7Id6z6y$f)^0ZkX1=D)Wl@>VJNJu0gNEsp`b8)A8}oF z3QQ*n6JaHu(yB!nP1pqnbb&HtAMqeGG51R@UNReXzQz_SAyf1*0SjCw$uXu!m9{0v zbH`UfN#8xT^&WYo>fe*z6Q#+wK_*;I)UdPQvo=m+;r~tK3i+qV#p2kH1x2;#$~}TS zq8L)o1rYOy&OvT7HH1_LxmUdhGw;NKLu8wy z#6m+9RnMR|{2(FQxFY*;?N}-wp(}0uVsMbQwX^xnVj{trf>{SLv$9E~J`DlaNk_F9 zTwimYTYs};B3i+`2Y_VUv?keS%Vg7IFOvl0v`*V^K@_fGN0gP$tM`h3>w93KidqOm zVjB%Bx^P%VB3_k^FS?*yCAQ$^`5{un&Y?2<8!V}|f&%!gEoxOMrCiaXIsj_cwlQ9l zPc4bO*A_FkM!!-Q3HiC{Lw~s#%r)RN=o-`b6eM$PZNGW)-V4b6e%d}!m)pfy{Qa># z^Vn9;7pW~H3T}g@CvSnxaz!hEP1qXLXYt{6Y}s}B8}1e)SXH3|DKMQ2S=r%xqF}0; zB_%jDtTJ*zZY+Ai2ER_+m)wOf$Tj%JtsKU!C3WgxnRJ%~z}89+Fjufc56lUM4*MmW zGTXEWj2VOQ$#3TVU^(ENkbrbm2tYnn1q&d3_yf2D%tl8%+suL5BDfFZB|*Va3H>h<9y70IMe0ua_-pG!i8)TcpHXP1rnHoj{~`8-vndc^c3lyh2pn}st%h7?vQlP z-WZBn@6_)90LhqT_Cacr%-%oMq-~Iz#9wKOTq6uplju=(&Ygx)R){^8VMrYDYp(Q3 z{9}EQaj=;kzQFgi0G%+fvh zTtKWbphB#r2_*Km2IJH?fRwq~i@tTsyyosB!hW;hfZx#ZEAQ{FmC806UdWozHMAeU zc>y&<2Ca=XR*E%9G_&dUXYNQnRL!YT)v@3vXfeeHrN;HRQ{fKOFr*e+n5=$$c+Bg> zYg9jDgAmrJxTMxu9Yx#@06A@Dr;|BLi?46seK75=S)|L0TPp~FmeJy?*$XSkT zwwNw7B^_+G>3#sVEqAki!J{87>6ZBP@%dDaL2;2iy*Od2xOC$(d@7MS&oGp6vbQKM zTCTA|L=$!2;_JZy<_O5b8hr@v5+UYFyYnUGsupo5j+<;)T-ms!I{f3GhFe8e{)(Da zaTqm0C$tjYqfQuSQ6ggMp<~(w9hJnc(KJ~6jQc^6=PCcglqWHFaw%)C_i6bJzWLn? z1KA=hwY36ia7%>W^xUmTW>oYd51;=Q*(=)|)z`x()L-o_z5z=@e68FRMYp@xGDpwe zGp6K%M+mxIRX**WE+sDTuL27{t76wIfXjE}dIKkacMqU+S#|zx24H|r^;{<{U)eDLC3uPZ2}BEF7~UHu{`}4!iJn*K9~*&coJowoapt@ z*ngF#agY4Gi%m+p!UcUY8L___6Lq@Tx}52sG1mZ|ye=rmZS#N&u@)vti_(xaUm@#< z3oG5J)#OyBhVQ_((@}mp+*B=%N>4s&qQ}C->Ir^pA8pZv{m5aaukDl(%d$DYyJ}Xu z#4EUZX}z@Uto=bXoO?3*T$!8>givZbk>{-_XJthreL*kkgYsk?QX{%@k{kD!r0TM; z@76uI1mT4!7A3NfCMe`&#_zGWN3Oo$$M&1$zM)#FJ#n1+Q~eN_;U9yT{gJ8Mhv0(0j`umDq|7zFKSbJ~Rp-a-&c40}2e$-VIM8Ng=I|G;YYb~a+Ntp47-kV~G9Jz*G zM}`%g<30S=Nvq1aDHl7IsS7ID$yWxTQNYK)7wqJ5b64Z#NmstY)-G6%t5d=xymXQjYqobf=m%?iiG~J7m3@WL^|v%MIo|ZajJ>gLu=B1w5IyTw*62H z{wzHs15>~@53SefQElWjN?o z#ptN>rOdC#y9VFiQ0{ckUCC7c5yz7}`74eG*Pp9Rt?N9yjb#PWvY7sFEer9ekzeTB zf*C>Wuc?4z?#wm<*mgH@i2Q3Ex*zp)tc8r2c1iy~=JBF_)91l)f~G_-3YOTh`oE&- zoOyoM7MVFfNUfwPd&PV4lC~eCfm@sHOG0hz{)+40M$E8~dN?j3qvsY{V+R@^W}r8v zF{bMx@}3~q8fflpUxn}tMTXqlK*eG(%tSN){VGBJbnZ}LC*5V8^d}G$#5<2!3d?PC z347%X$@tHqOGDp_$oOsJw;mFFDczd`@N64k{j$E1_=oBMo!*N zfe9_Y(D<5=xu?^_jZobg_FNVc>@e|MoM;|GLk03})oG#!Im8drX~~8XFb+*Uz8Wzj zhqL;?@Wu}O{_n**N?an>f5-1$s{p(y<$uNR9*F~s>6t#FxT^yK&{daG0a@L!XmWuz zgZ_Evhpo68ILsUic%~|h3hY*~27MGT``jm|^mA0Ow9r$2=dHQkP2|5a=!bQ-DEY+X zpbu8xrK~PEnU8j9;Ba&L)faBmaq&KQl|YVdb0A z7eTAS#{7NiIs6)<#t?A?;F!PTl3k+8W4q^G+-K=0E#+EAcK-pGw~!ZNBZ7%=G zsd}I;xpN4CVu6E!C(D%3N(!RO3W@b-d_0S;{3IfTx#x%VhQhQ$Q5s{&46brG&uq@i z2^!uO$?mw`8%;!oxTSmL^3%%-s?x&sP0lk8PtCH;GLifK%u(4vD{lq(Rc&7-V2dUm7jp6#p`cP9pAn^UGNJQM3G!sG2~wRcm?RDPpw4V5+bOitnGy zzOUUiUIM1P`w&P`ienu5jkQrps=HGH+#Qvflw8edGTIzr2N|HOQTot2n$t-L{>g!5 z{;^;ET+(asjqZ+n3%lBDL>y7@&fWD1hR6q#<8j7uFro{f1xECI_&lR!nZT`r2E-8H zo~;-q_7&$LF;AfW8gYh}C{Wn6N`}AZb%u;CU2I_sXpZ@6>vURc_cI$q{?2(~=<%}rfZv-9!Kv!Ri0WZ6e3Dbu?tvz>Y)jf(|m3;5pP23P9#|m~wk?hmVPsB3AAv5lMsx{c7?e�CdUeO+r6Nd%mW5pI$1D#e|nY z#qfY7v!r3JomLv#=@F)mdI2dG!yagaG6k809ZCh{&@_Jy!r{2YjP^B+B+`T3KkdpF zxF4zoQlf&F_rRc7LRPWGKG%yl#rW?8!@35gD-fFQt`e9h(tiy{Oh$@McLX?biH}#5 zk?02K&EBBMlX??BS(7nS4fJ-SVGVqqJ3R{A_jK;ZvRrP3L3(e72t^~Y`WL>bH2E)l z6Sn$}Z(tTZ+xTi)#Um{mpFqhr2{<)v&!#!#C-0r`oM2a&u%Zw zw2+rb(+S!twREZ0Y(qpOCZ2oDEw*l?n2d9|V;Qg^Ni0v5Gau^X_ixiq= zjG_(y@LGFI3lJ_BEIxkn4s6SaI$O|H%cq|H!$Yt_v+wtafzQoL~-ZcEpX$)F4zWR`J zbIjxsnH@JC^7Ko83Iac#K;YJ?0F{4*6=BX3DHpDE^knkiC`VZ!=xaVo|92je$^$RP z1Hz(W?>AlWbl;&%b}!n|5gCdpiQbo5$a0 z|84Xj|ISxQ*%Jd2^*{OsBmb>$(BAdf5rOS@^J4_uQGnCx3A5iip_(9Ego$wGNo#IP zhrN%YRd|XXX!PKuHXaPUo`@H@)M2=nkst|Z1y~weIg5@hlN)?y;y&FO6Z<}##M66` zN&K_P?^Dh`)3Fq`_XH`TGt+C5k~L6rfzn!Brw5H}MU-s1sZ;M~e3HWE>3GN;+ny5j z!}=XQl^ad>seI&l!vvfq|FM&#^v3PJ#BPNyDVf&4%^q~WW1WY5?WDi3&X0e`I;MI5 zz&ddB3#pbAV*iD80QfXJ4F868N?rbeb?kBefps4LH>{)e!WnY(e`6i$A2(oMvX$1$lV{It(luA*s(=P?=^#DG(zO@(-{l_ZlPTTLZS&NI?HMYV!tt70_NJiJn3>K z%_(Qy)@1SGEsKMp_rAn~w0Y67te8)Smm&i*M!L>gR_)d z)S$prl>kEhsEA9@oRKf1RX)RLHT5`IRLA-O~dWt33G%f zTN0J3Y{c)5v{aSj_X>d3hL|B2EiML6Otm5=4WEY3Vw^M&E>EL-G*1_MPcqo0G@F3uE+ecjXBW)k=p0aI%I2`I1cZ$0dP`Bi z@7Im!QZ6r3VR<$RU<7w}Fbs8y9U$QacVooIA44mayjukABPQo)8#dMJ8zdXA(Tl`B zNS?T`GRtOQT+E%@C@?pxESp2Y?mKNYSjGo~e`Zc_&C!PZ%>1*2vI$D|Xu4|DYTGQA zc~{&NW9!f6r$6-ZZR^&&al$UaO|XK8DcGN56>l!%qpCjPdC7rah#GrFL0`TrtT>0{ zw|=kFDzjw~yqIJ)McW2}>*5A+SryD$Wq32FTYi0V{pP7^80&Jmt@O^Mg-Yj93*}}E zh^a7vZtKI*1Y~60a`Z6a3X-QXIISLXGZ9`S8df7vuFBq*J@~1MT`;c7e>{?XR+t;8 zdr8%0kg&c~X@{ns3S@?5+hKjHgU>rF(D;yxxvS}4gV0;`TnFLnQPDM39-wKf#8kR+ zZ>N{-Tk^;9(jm$m`eMEs^r`P`v*Z7~j)C2>uMKi@ARseNDzbAiKDEengJgg4(jcxA z$AE+&yE{6NDy_eR=dY2%_C-bj^*x>JX<#~G;Z;&3Rhm;7F!@ZN#XTG(m27mwxBA^CdP#&{1}VgQfUu&%ZT9ii_T_d(Q&L`Cuh?E5jLu?M@{%|iv-YCKg0aSSi-th=PKb#CC~Ssws^Q_D z290+1(^zY{p+ambPeOF0RZS6z$MfSf{N%NqNNeR|J`C=;53!+Z%H@EvW4|cmljmPy zU%bpfs|s{TIP{MoPlhcqSZdK+#f2C_3Hm8tS^={D3SjXyNc?ZZCNUCpF^Xh|00%-H z^vk^497=9Yb*to}w|F_AHIDD;t^XUVeG^Z_lunBz)-d&^NTjWOT%gq;6-O^As>>Pm z!AtjHu@sdyfL+DT=@u==BDc_FLk(^K+jrETwS~jIf(O|$VPmKav=i*|Rhc#DUKxPkn{hjV z315{f^~hV#1@Vgu4~MytgK~c4P$>tHAC1;_1f#Z!A1x86Qw!u^R}!~?IAjBHuxOw- znk}nzkG|qcqBz<&%mSG#qd_D&p43b9fYaJj@b?5VIp@tV7waPY{hN^`>sz1_Y{)Sg z%t_zN=OUb)m;A+o)aov)Me^0tc!(!Y7O5hwmBu}2FQpm&eP;lwVBmommXz=jhwytk z_B|B}^FLCd)W~u8W8f^bpxqO>iw1!Mk0d5Gz*vh=FkDcqbqEw|jeU=`#=a}~1r@pn zyX>A3y-%)SM_^N+Y$!LZ(|b0QTS>{)pZ=q>@X=f_aLUSknJBNjKbP1sG}+Yvp))S) z6Z11aiLo2V%80#yl!T2ngpFl%#83R8Ph!9C1wMk7X!ZsEJwJevOduS;TFWRbr6B^0 zuW!qNB$$UZL_V&KMV%=+1HaQ2K|HPnrA3dOckSyHz2@D!{i-5dw+JLpZQ+5ZV9Vg& z$Em@;Pp2&6e8)!dVK|h?gI-`{_)zAr>FKKMd)_(+T=( zGX`7S!lNL6L0Pqz5{v>(SOy5n>;m62yWz8ZJ3kh!TC5V@%W zU{rLG;EI~6Txg1n9rkxJHKvmh7`unPj~f)d44BHx5iijKxI2OnEhtV#vT#oC!BwY! z1y}#F8P0^>Z3eqd!!4}VFO*TnFDApJTzN%-_`LBv4b|%bpp@!GoY>m7pNUw->tqJ; z^R~iW_w?Rd`h4`CgrZ+kb2)@sj>6W!-5mpVJ3UTJeqAP`a0N$PgTU<8$=j3o`jhU# z${-m#0?I0)K<^hqfaYzeaecCY(}6Yxi*^-ld5(2sG8ghv3h1G0CSlY>12u}3p|vG? zXp(Rg(^~z-8+&C@f7BD8k?-0XcQd9Q*9_xKyu2QriU#Jdu^J^V!Co z(jn<}y}+3g_ZM$``0+!ogSE3t930NEk)E2QMPx%FK^3d7n;V6&m+VwPWm3 z*b7wp8;1M7yMI$PGua+jFX$P6V#Q^$Bk){E&iKVs&A{y1B|kKwONG{GZmFGVhnurj z*1KlbyUZ)IdYQSr>>TSu7j@s;?3&Ppnbim_I0;o>8r5!T=^!(PY1q->?{KccsI(V6(cRJo9jZt-CUy=2u7_uwx>#YCb5ZM~GCigmDsAQqwtC zv#6QOG#0*vaDl-?kz~WdcHJnq=QBB{WvYU&ZT^cC>JZ3$vJ6h7NybJxwI!z zi4jgxXdZ0lA0OEg{~1mFUXi~dhDzWSLW>kFocLU+2K|Cx7psZqto zB>Q}lZuYD^QMNxN?o(3bWYe7(dxd@)Ayo~?D?0Ak?QBz=u+=ytkWEiGRN$$!PhT;nx<;CrPkU*E^hQP zXvc{3Ex2kO95kI`u;{XzjPyWXnbx~d6uTOmMv?zGzcv%6z`h3HOH1X-wTdR^&zWW~_Ek*;6M z;NIofNfG;jJ}PF<`Gf!0x4vzPZp9MBz7$2Nor;IhF&j6+Fmdi&nLhLN-2f{^wN?H| zy2HXEOWIo^Ucu7JA7+xnj*BgS(+_=eDt^PU`4$X(zaNK`XjxW24A- zP&*8GW}K_>wKGMkQz9bh#7- zg$bWWdAafn!{h;*R;Q|3`V&IQbfH;jR?I_-w(}+Kc7tEwM3~r=^v?BZ>*=O@-yPYS)6}~wyIo-Ef#QrswYx}`Z?GcI zr}d^pE;Ody+9 ziv$zcmP+3O?TW&bL1Td>Kwc0keEGZ7w@;w5j9$!^{*n_FsHdz31 z4%YUzl+o-_q#UcG52ulG;2yEGUPi2|#b@@{Tx-tydD!M?7S?A`8Li<#EFsa-dbpr&##r^}nJ9 zn{hm2DoDXSrM%gHj`BAx!4;}}+=Ign8`z~^$MQgz{1Z8}^0SnL?vv1~@4%{Iz^z2j z=4-JEMsJbvep2nmI3*gB@&gm*YqLwe9Na4S(1W5p*15$k+NuA;BA zA%GXDs}NNPHdhEBM2=oHlq`rw1?@5-h619N$$aSHVW|ji7yr<+NXAVBrK?sWxH(d#*ev^V~2WM)$_?W}y4~IY4 z)D^FbRO4`@GcN{DztOZ_O2OjZlBYtEx`qk(yx@ADg;tdvZ6X}Cbjq&=8m~yxWRS=Kzix(s za-(+S@jz1 z0d}4|y-h4B?0`>t%S|lOWM)prd~Nm<4I(g2aU?68prf=mI|#0^uhhqDio^Z>!r#c` z9DQuHhw39UV>hueU`ZlaBL6!|+`wU%ux`2chXjpP`mKsF*0Xx>3Gf7PZCN+Wp7*(y zJTjqPWLV7ch}z%xBD}*HH~1XsYKwE&$c8S;NDq)BX`h zwEonbWk0k7$*E5BM;vfmZ7h~8)r+ODQV6HVe#>hTbJ(Ou3z((1pL=zj-9R7kJFnBRxJE^Z4J7{wxmk#6L5fON=?3ZLEL$<;NbaCor}L^{4YK%0GcyjP6TgTg}?N z_~IX<$(l89<@1#IZ<`~V`H~??G=AVQ;4wi8g$m)B#NaQ+tj4T0aCL{oker~e7b-`N zi59r8h^3IF^__gs%j_G#bKrU9poeYM8~?dMlg`euQq;9zl`FPMI|HD3vtjc~&E^t0 z(z-p7f4luA8pOkyPrpd&1<*|HY!qe^Dl_{X`VV+V&f5U_0@M#;YJ zfl!OkqhZ}`8B|dqh)cn#l2F?M3$b^2D8$9}mNf}+@$K!+inD@)PIK`$SrknrABF$i zgm{X=4&R_`@x440a_l7{f5?T8dH$}u^7uUfj~XVum%KBMKIG3X&=VVeaj0Di=m~zw zr~&SUedRz~hv0=RngD+F6CQkrq56jixMR_V0?=z*4>ppkngErFc*{VdgU++r&R}!@ zz=Xj+qBSg|hXJg)MjhG-p@*TEbiI#ZXB@7w1hNrqp$|(LK}a7xhzcstGM)Db$287+ zdNYL|lOz#2CRmvFn30%(HZ;hsEOkqAPUOuA*ar&QitU;K`@q67zJXCADsyA5LQ^A- zp8><%!@v+SmTNue^+#xtoxD^~cWN#mxb;a8omy0VKJ8wOZJz6fR1+c?I~y8}erH9u zgexEiCWFa}Qo*GT6t1+kh8k2#3*n^z7Pkl1fVE*@EV9)8x*VHpwu8CX)+Y%ze z-E9anj$zCN+p;J2;@$5jb0Tp2a?nM%wCb5<&m3FZ!%1xX31~kf3$b29-82e-%q4*< z4yi>=L_Tn@wCE(rqo9PJ=r1H@MrlLHjDR;WCZsKCPV$}+!cV*s{qijB1tyJ?c%_O7 z3vbYKUOi!+ zUNcO-=zU?M5d4!C$?RRRRr6TXp}4kV>a5~bKyhupsHc5BA;U~6WBqc6V3Gb1mHZeK zL1B@CiDK`~o~p}K%#{X_&yn~k*-r)Q&d}-e2u)P1v)$M|<^~oh1+TW_43Y@`+<3LGk<%s5{sz z+3z9Hf=Ss{Ig89;fyuf2=QlMy1>Rb5YWUINVz}p^PqC}~c$MROF5mW&0E$Vpu6-1o zZPfVCYWV0OEI;82b%H#=$)L#$XqxCGzX@LaSUD1N`BG0Yg#o|+c(mRi0PQ;%9^OBw6}t*@733!!KH9JGRHt{zs?Q!B+pTnWHT#M< zQh#puoXpi3nzPRA!Ns*u-TO~8EdbDh3&9`!*>{&U@s-XJ*5gQ&(b@!V%&J1PM z;^Wh5XJajmrOo)tUg183_5+ zX21RZoUlH8ZuX+hWNhn zpWl0VIi`EYca}L+m<~6-r$P>Ha34L^dT^g-im_zl^LMJii7#bJb#vDTT$7n+>k1V7 zAsm8#TU5*z{H*&?u*L+lerO}QMctOnzY2~_wLUyoyxVW0$Oj%w+hk8|s(2@#@=+Dr zTvtCcPpqi~w}aww@nbjgZH6u%1 z67lCEFIgih{fVee3;oCLKWjvUM7c{eDZUN1agxwP8dMDqvIZG7C^Gb}=dc5i+kVoO zI@F#P?gt2(%2cMj>p03}#gqE$_Ub)Fg7%eQfdT{0{Vd@$*BZ++@csUP{7@rJYQsA7 zt;64pK_<;CI_zcCCu#BiXG+@Sr!_@X|2j*H7+-`*y=v7B%OOZ2y(kK{(DRptdn?YG z<;oMo`uSho1x9+|cRw$&_eZW9;XeX+s6ErAh# zrK<#|*K&^G087d5#p{u@sw*D^)9kckgI7K$?b;5DPgqp5OyyN$l}hY>eOPyVX!m?r z#eH7iR`h!#uY@Lpq^Cfu9Cc;WPfA=pSb^GPdU|smz_Gx%%WuCge_NQbRV>d17*~NI zja0UJ`~SKL)8A7Tf>Zs@w)lW9Sr8sLIhZn1zD(#~$SO6ylvR))p&y_TV;_g`M%oem ziq+~RBA3#W_xAiHy>w}xysl!3YqH_9#2@dI?^|K$XhqP%H`<4)5$7Xv=LJ;@rn~v! zRUI&Tvx0zL%>V~HO|U~%F~3ifC7P)pGUZGekM)={Rj;W`nXT5Xm}v?$0|iT0e;4WO zqi4z^UE8x~ajjdiCI}6Y4?&Rq#rMXIny2-bSo*A+urjcG)?n0xs68eh_B_6EhGm@P z{`o9NfA$2iUeRZ|{(0|nopJ^JjO=EvlouMpACP~QdF;lW!=}dJ(sFvzI;qo zxX=UjJaI^V-gz=afb_^>) zQrqAJmp~M7B2~>}MZw}FyhF+7~mv#L@uU%l>I+ncP2%-3;#ZXl3 z1h71E-stwwh0-<;A?lfz9(ddG=eyH_Yq$(Nd$;anO;&OYX^?#9m+QDX0qVfcll6A4 zPF9ig9&)gpc@x(*9YpknbOh00-Fp6gsxYX=0(=tlV96Z*Vz48Tc0T0h17Wc( ziD=DE=jc_ixXvcz4a8)O<^v2>P^^25Mu}+HpdT5l@v$v6)7HxmxaWxn!ugxYl}1kw z7}CER)!@5l%RNlWR=mbnBcZx>ay{mGM39Y?Jy1Kugza&zu?DTlJy>!_Wu>!WGYW%n9 zVb*_*Lz}z=w6X)Xx%7YGgt%vOOWd(Jf9NDX=CE`y1#P0!pzJyoiCu5ccyqtv3!HGI}%1o|@K19~fRO_)++ad$+g8X$*8ayMWz?KgLMmDRDgL09b%k zPVW(;NwvoD+L zJ=*{ra6WLppw0cppI$SY`#QiorKm3;>bhm3#{;0-ZZMdssoDTh*D)f0J@0qwn&Ei) z#|1=PVQ$~4D{}$djZaGZ%=u2713!S3_hLD7{I!j^}PA&6jSX)be)% zi=~oK-2(f(`%eN}r&%HNhb?C@8>gb}PZHGfVqTh{jZ2|SS4bWy<(v;C1niI}%r_BF z0oKJU-ZJd4E|V6VNSKs6PIRQ_2*`Z{YV}VygrH8Nk#==xVMAa?`3R%1VI(5VM7!<> z=1X1{$0B(PFdqEUITX+N=2+?o61*je-2+*oNEjqXzPZdB)FH^2N#IcsvhnY*t z0;9>O2QVNuG&tBPBUle+9~h9Xq!|sYdSmmu#^c0g#*2@({kBD4FY^$@E)eJh)+a&- z7usLALWuz)-sD%v;i5f!0Eo+)j-T;U=ycFGCMu{Su$$2PVqwgQD4@q9EqF|XZ0z+# z*`Jvoj2Y3`s~jGzA1qug$pEb1Nu0mHM2$qaKL;4jQE>$pc^?cG>5dHsIt#FY_p^uu zoyEOvNAHnVf@spY2SX8Pyb%R4X;f|%%H?>Q2IcMhNH<~3RQ0;`Iqx~ zrc|9ZD%m|)!t_)HP5y?1S!LVK#4UCB#HAib=^F{18Q)%i(y!#KlGQXq?c`gb*0rT@ zT@2S`t=_s;+NO8C1M22KA~i;BwD}_&=a~6ZCyXkm(a@I147Bd!DMq;S>C$}RieQJ6gNnsD8IFA~hu3e@?{Omg*QLpps5C)Bj+?w-r_|5!!GMVZt z*d=Uruv|IIkoGqX%tiC(JkCG6sd|l`sq$YQ3W>uSYP;*iq9|(B{GLa%4_e&1*N|mP zyB9qY6+e%kN$<(rtiGjn3YUvm#J8w)50reXP_*Yb89@I+O-__Tc?|2oY$S8=STvL%WvZ7-5$wH_~~%|47i zy4UZugo9J=(BQ}ZHkgc9nj)HPZT5^|3o0wQJM;1=`!Y5)%4_ZbfSAyP|6>;d-fz+l zl;w%?9rSntDnQNPW7?)(DJ{bR-H+|cIaRI3%*?6p z`-|%CD*CCWi|1bVTI>4F8v9Nm-+Y{EY!aMan`Qm*ufE)SI&Dr)2E_>~6GG2RM#)GL z70>TAs%#eDUbC(^OM>N_NyOrFvZYxgd+z)OGq4sx2Q`F>XPb)CoB#U76w|w>ch}@D zmjU<*yVVuEkuygUznaT-ycKL5H6+08r;DABb-gZtL)vC>3g zkI4f#pP4L0?{Shz4Y=~Nru<+EXFv2<9+kGiz)Yxe|QeB`u12WNkkrgh$_bKc<@ zD(=ahteC}<{35$8$C8dJmPX_;Pqvmrg17mL3PL*Tg|?Vo%;XS_y@)(f3iUXT<&Mu! zd;!WcQ7_|r-JF!ZQS%D&-AlNV=Smv-SYsv~HtVsJu+-$`b?>Ck0zGO0ThFNY(Gzl= zMFfF2ag?v>1|eO~S9$&1X_ow3I^=Wio00-bo4!>}EUV7KW}n>|4XZ(8BgIEX9*NZE zPcczRaik`YDQ7Xw4*hArc(0UU;0R(HxX(m9&%K0Kq?!xcmpfYhvg&DRO1ml&8(0Gc&rgwMW^bHXx)( z-Piy4;d|%1N)ep~`blERtl|Jfua4|~4>9e1hIn#gWFEp2f8{q(cH$|f{<_L-&r}w` z%%Y0NWJ|Mf{9CARHL{>CZA1t8pzbe9eCh`J4D_s$IL?#fJ&rQfo_PFCr|7{F`2;a$ z#;PM6Rpb6ctxrsDF#o1@i|`K>2%#*G;&ZEy35wQGM$FwF*VS40WDIBz4x8pTCHt>g zVhRyVc3!M#-eo8xN>B_2Ndd@;HU}3!QTFeJNA55O=E(`;Ar0AcP-k}+vPLWGK}5}l zpp|O9$ZdWQ*u9{I1}jbd{HF+ZVutV0EBlDSq2)6XKl1b2NdL!h0&BU5J!<5_MBh5) zJtJh{X6yOlnBV|bjG(GABeL64a&>A%|DStgcjQ`l5EwG(e45{7_ylNBzgc3V#<@)x zsAO}XP5X`wt*w63rr9qE;cu#wAFtVNWPo)LvU07FM% zFZ0=q)N6cxRtV`ls1G9y#>aoi#Qub?0sIbD1dT;S;;!v5(B4l&LBhsIMn3S1MaHhm z-S`}$^aQ_JKx;k@?N;J&42>kFs_9%8Hz7fLw@<5FU!=24AxwT|7Zx`WhY2^aA<4ck zM>ih7^dP6A6b8l&H<3Z+S55{JXn%AZAB(X0E60X7aV*|0NC{8E?4;o)-{~!je2jf8 z4D@&>GCqcvzWhrv;)canr#K37D4e66Vh(r)BTB&6?txFCtKHdFnOhEK5H^|L(lPWk zXj-b|99CSsVky4^9u6vjfRA2^bIH6hl7-hcYLwSXFBekv%(_V$t$qS0hkT2l3ot^h z=%>sziifJws9Sm53$T<^p6$N{EXG>rO57elFz0>rDtR*BrCdN9sCCs|awy9E7*xrY z$Rk}{o}i~e-CUKh+`@WbSXnUFvRWeVlxa}~u=cM^oIzpz5*1$9`Z#48wct3uIPvTU zdqr*1Jh+^}sZ6z>ssQuDut0JFCF<-v93wDtdFXLKRJ>{jpF4SvE!N-M{;OZF_9v}0 z0ne-t3)^8Mcq)e$(z6c>WflEc3$fN_!c_?`O*}`A(P-uLLl^WmbFLBG>9s#;M&2b5 z)Z@&_)#)%b+f^2{G{hgMQ&ZddaSe~&^_R_oU)MZOX-@>RT;_xOaKYTykiwkjk|LiQ z<*tAI2`dg`#~Rshs-M%-%#{<~V?CuLCSnGW490i?Wj-1}l?(u)69iB_HjWHT63q6Y zFWNydgBwDCr)ttQi5nWrhz`&|Jmrs}na6vXRxb$%_yr#3AaTm_s5-idK zFCaWk3Gy((Trm|Ti#+!4HRY9xngfKu0=L6pPd9Czknc@so*>8j9w+)Lw{jlaVfICN zC84ju_d(Ce{CqD$!az!~Zp9fBTR#6Q*55=lr{_aXcmTnee02yybTMhKf^Sa`G}?J3 z$*vAfyUvkw=c=9QeOBwf=LI=b?y3lR;6 zVSuSR7HPPI>_RMyT?pVkV3FXMkhDc^E}GFv_$m(bJTXI`f?R)4dm(`-r~sW|-~pG} z;LpQ|rykogL^)&fx3FN@H;~ynl6mi*-UYj$L%0-m zO&5!`rDR+f&sb65lwK`WW=^D^f0iW!FQ)+`b@Q)WM@BW4^oA1Z4h@DgLDi(d%;g8- zD?;VK3867SH^PZeub~(~GySkzfK+xVx0aoNtn7Ajv2vA~`NZ zg)Ti6<>4ZiVcka&u8U4R5mMvPw%+ngPw-CnOF2p3JBJ}z;Cn>l03C$Q03B=uJ2X43 zk8?Y515&6lp(E%%_DJjt5SW*khP_YzZJtr+{6Vho0RdROuM>JbcH|q`1XDm*Ubih!bT$Hv6in!t9mgN`z9!)X zeBJ0}ePdQ<5C!?jB&?(AAb<#rmlTN10PRl93nU|hs|l9a{4oFX^OZ0dZw?K!2F!7d z1Gw9~`LP4nrvRewY{TaBKiPThAc()ruV+mJOWX0E3&Vr^fj!4ctytK^ljag~>mOH0 z1pN+!kg%ZGY0*w1hP58*cdh`y;nRcR9t}Dw_Hz1dAHsv*--ic(BdI<1>C1cz?*cvQ z`bKhbUFz5U`xRIY!OaCXXF9=7!Ow}Sz-+$Hiaor{w7iQHz+4-aFsEr-VD2Xg0ZsqU zH*;cy_X(eabh@)ut2?b?fjn~tXYnzc@!?lEwCJlpDtbnptuP!roY>G;RkXjqa4U8A z{KTGA279K(UcT5FbJPJrD5f$h~@hQQDqh3#n=Khb2qXW^L0{El~t zhPi9)7|(CPKHB2`8#GetMi-Jl0GaI>bhYeV8nSQ;>}&9OK0WIb#u4}xIKT>j4D*-b zGk;+v+;nESXQRz*rTMA*^pkbUJ7kQv%oB{%d;8mN1NUL7mYHD#&WE5h!Qp3T$J7TV z%;mhaw~!cW-E*nlM+n_lmF$_$5m)e{8|KQj2RcT%BaNy-OW#+Gp9DRxOm2FPeb+7G zYew*X4F$h?oc>{;a@Q4dGQQGQV^nX09MlRGP}WmI=faQ@H;(eV@8f4y*(WjSXJA)+ z-9RKx4IjQtWnrR6)Wi4b99EElig5Lpfl855_ds<)-D3SQq1igkd*u5hPZxSVidhp3 zmQSjgn2=0-Ifc7iRawFMUU>KsdoOUA7aEI!1@pH69qFboCkFx3nN<5eCzBJN&0SmL zE&C`*;`{bfGhqTlbVc#^I_&RaM4dT`=`E5&olfPQTh{-mo z{;2t7P8S5=x3@#R5D7A`_j^{rUSsrh;$eCISq{P4d|Mdv-WzM?dt?jiNkz%wX;~>g zzYNS)J8e_*+{5`Xw2VQ{mpK5fpH}3ZOpn5zu30F>1*KAozuB3tm2ldg?R3%~p6Qm{ zWBXs8X+!S0o0fgrnm#j2R|ZRC4O07ZScKY4!^VeO5m;-BD}H)I&pRVeME70iUM2fcBk_DMBlMN5IisKRyprkV1r0h}N+gA61F??)@+MbHkS`}$ z)vew+?MqW@#4b0J+PF(|t0&D-S84uCcT_h6C7&wXt!q0h!ybzcbe2gD?OR(~|5Y7M zSk@Y7Wsx+SexXpv+L_9aDdp29_1l#jSabd!$g_VIp+Juy5y{gC)Bbaa<&=o93AV32 zIKOM_qFHu1rH`q0il4Z(uZKK^8%gHJ^C(DZy764mKVL1}>iymRM5ZvJd4bA&n4Ugu z@a~f5d!(hG5$OB@dHSQe_AE*8@2U_K(Oly7`I{;f+$SRD6J6=s5BQrXtZ4f!3KQxz zBBG45V>)zRLEh(Gu`EWO%u7Ei4ggK+@>r6) zkPKfXCWi$pS1ia7C%Sp>f?<%lKmT~MAgNvA5@G1Xt?5ki%iz&yqDRMxTy3n8B!WnM zaxrJ}>YzFzY$lx>Kzq$SHj^iGuXbLt0YBD>Zhsp~#uwizdlM{MB}PW(m)miB;$HF2Ta7t$pmoLoD6T~kafqg6k!kg}7%bY_fAC{vVZ#6VxuZ8HfP3E|=T-UzWx_+QlXU1u zJgiA?lPa%MSHkSbyZC3$V{ccFAQ9B}fKIiy!f%N1{XWS?I(2j?S2lyfL&C)75JqpRaG-!m&R9cuRu<=8t3( zHs-}Bc38PzI}|5p9BX@GYVuMS93H%J%da$eEf0LYvEVCVi%kLtzp5gD(dqA4tW@Je?ftUF=tj`yHo)6G&hN6kgHpqCKuk(M95(7%H|yz^pG zd|O|KK1>SgL*jc=WX=}MCE+7CrR(FSp;ft=I#1x(ZRsvlemeavE3xTuxMNJ)k4-w- zGz`g9aWb8#q3_rkPNhTom2JQ62j(3@rJ}vVof3oNUcPcb{Wc3^Lz^R2VHhrwyE;v5 zE0#Vb9zpq>rTw7Ph^vb=8TNcy!3N8Iaz?!=XCD*w0PXvi~ezOd7Go^T4xqCK4a=i<#@cXPnh%u zehshX{yxV@uGM9J>d3`{TwZ}*FIzaD=6wTOPEz^`<@Y+5H&9W|%a+D1W%GqNQ@ExT zxG?pEWLHq%SqJ40(8DLb9d9vdrQEidCGV&t!zV=)3}sNevBmtjmt?J*P4o)`&92dt zd^4AV-%K>n%A7CD2i~r&8ti5EP3doJr7{xi-2MGB3`*Z~d?CMbL{AV5}r{ zERPMfky96Q{<1f{X%oFOKjSCX7)jEPPl}GK)d}wF3u$$Gl)}b%@x=H^-deL($rp~9 zVdBROs$dwK1flfz0$s)z7Oi{0L^St1En^H58NR!0EM;29<1CdbCS7jbA6whHc? zNc}B3N5u{g^_kE@k0w@b)9=k~A868hS~Yd&F@(cU*>jVOS!P+A7N8e~ensXfEuPQr zf3SD{$QNV4;YlR=T?vrhJ}O)_8uO?ua-^e^Kyw~YOT`zqle5X@cKw9KAUCmLZlt0T$xD3c(*4dp33*B?lYMWNbB>7^@ zSAC2uYVry9;ulyHB2xqqK$OpCA4iCLt)HWRwu3nB?MXN`xxumRW>sfvzIa9ggPN)6*k`8a6H)Uy#Uh z!ZQLZge92uR_GRKjxm|A7ZkD_wOQKrviA*z%6#YQThO{PTnk;BGdI*roao4n!Lwgu zRMGvL1ta{X$u-aG`2OB@y>)S{Cv79d7kuy?^CgA9CN;FRqU8#eZJL%AU^jt7J2vfLNvZ#2W&|zyZ)`DjURitZvR%jdtX*&m=`?)#>b;tMW~HR^65f63a{Qm9o)X)|26N7!qfT;hZ1wFLnd* zFXpJD?~i;r$=8bpuAU!imx%}`+Rcy|?$7l#&brTr^H+F^YUfBxl+wA6l~hzl(g}fju(eW+1 zKNo_VjM5Dv9du%M?N;<41_jxEM_Db}x#L7w_uRKTH~Ix(a1bcbdW;|IV62`abcU zU?EzAeUMTMd}<(*_FWV{__+iwjn8Cg_H;NUgMejw+F5WnrC+S3m4Z<}&E0^c=6$<* zz=~V(r9IJ@MaEt)-X{OZMP0&hH^lKk!Xn=I+&63*NHoVORCbd&LwRBuD>rc-%48u=#b6QVp5^vFa#8 zvJ3kdmjB)fr`2R;2MnO~YFXLTojO+t&pbhxiUNcRkiM`0qXv_kMU?wsRf!)vz!N8p zO$Rp)l_Rb0(73k=_Azp9C0E*LqtbMkp{g}0yuyB=a?NnzV@hr9c1`YaS!5E25s#?% zrOYrc6Ha-z7h*j1um-A)8w3Zd2@o2M4SQI;{5v&Hg_pG|G`e#x{=h+^_Pasn%rt>w zWwx!mQm}+(U0+3RCS_=V^ME!fTS^t|2b^vOVGDpByue<(8Yz291-yjw!yuY3KnpQr zcauZN3ZO%QMqD;7z~-Tc$`&x6r;C?50#K=F#9Q7qjux`hFGJrRp|TEGZFa!A*GsI} zvak-=2&zEC+;eIGI=ZEGC~;^4BGvt5**pN@Ghh?VE?B9fRrqvljTC-d_I4_XnK(8< zKdwaFMOQpI8phoLwh|i)uC3E~cXdXE2$f7y-;}(C*>yD9dODuy8M6^*f!EGj6hFp# z4o#kqS(pc>w)fw04+jD~K&$eC-z!R*= zqtnr%Cf}V?w;roDw4kUbl$lSKUR^RVtC6zTEj<>!Sd7h?#?Aj|CtiO%GrN%V%xsGQ z*`~Gf5j5qiz4U~ryZ_5EZTrJ9)s$}PCp^OMCzS4>Mf726QrJ%XgaX%Jn3S!LZvTs( zF5CuAVgPCR54B3IO-&G1Ap_9|>Le}|eHWmD7L*jzWq4Da`7KtX#?gM)5l2(|lUmM2 z7=pg}y&tjNe*ygvl_|*k%KjMpQS1Bh(AqL7kA&^a_y6jYLgWA6PN~YSN1UwRd08E@ zIttUDL0{t{P|&wh6%_P!E(Xcfhs^9wP#}jtqpdc`SGSu8*PA4!|Mu+pxwy%*B-*@Q zVHR;4gRTVQD+FUPXceA$efBkYMHb)bV&Gf5&_Q}4*^i*hlpWl1)mN8Myf;^RO4_TA=tp*dnx3mw&HyYeTBOJ&i>1iPa?%X8O~YI3u_sr28@;qrkIniI!eLJ4Tj6L&;QF~I4Rd!fECj*L(BPCMU z;imy!{LD&y>)_Ih6!Ou})(3&MN$7WeXscn?yQ+nKAD=ZA*5A5cQiR1h7yXH*opoSQ zPAb27e38u981`J@5fG&bpihHfpwzFURkr7#dvXa@#_AWpI>Q-S4b>`JyTm8de4i=W zaDBS^ZKa^gn|ju%cBQ>Eccqa)8)oIaC-5!rI~p^71>8=zYn?I#rhYP1!mXreXg9GHk6Vb>CDB*vIut>omIT{%x?ZPcR%!b8+bSF0kkJO zQ=Z}s%%_c2Jc`R#Yh@1h(QB30+JZZ4nUI{8ShQC28? zxK5*6yIvjU`S}P5J6w=s&YLv1Sq^yfLi23hP=IS4y-sRCwUViGv0}2s|%IpP8z6tuE?#!|7%&3zer|P#Eo~>}lhzvS& zj3C;2NEF^}f@=y<-6k>8fqIgi#L!Q2{6W2{A?WZydhEMdebvS(he~dalY{yco}YKz zc$e=eBz9aaWcb$HiZuIcD4@xtho%T*?aj)WfBmy`IeO#Cc&h{rS!6x&Tg->SO$+pZ z3ltM4C5D`M5K4Vsw6_FgM>Bzf%(;pG3^EU`Dzb>B#odf^KGquixbc%p&ILlgV2-{&3@fnOU^LAaB)WT z!8Q@Hwm=-0gf7i{c8jDL8MFV^yTs!Dt9OZuz`w@Cd?9{RJ3fbd&iucNmoA_vGyPYJ ze@2-tY8w>MeZ@fl0cmzQD>YOWKBZvaGO~Y-GV9U)J<5FWZ|1H!=U?WoQq^%(kpae^ z=B}fW`ajHF&56;^fme6T>c`b?lgkfsw(NO~;RR$JX6)$O&SE`I`+7=6Ir#1<(%dm6 zH2$VAs4e{=^B_TgO$-wBw5j)nhgX;*=+hf=wpPmBx7&ngt_C@N)prm#D99jHbik_8 z>-(%&llzCBmzny4CBm;vG%e;e4TknVdOl)WCQnvySGXSfP|kJYk7g&vO190pR@ujT zg}f3TDlMbo2j@ZgT?XoLx&$o)m`oUW7y8S+isvRhBi~~9bZy77m0)Um%ueb2=VU7tfSsTq;3)-eb2)ZG;dar zMIT|`lJWE%r?PwR)?T*_Rt#%fN|`!x*&Mm9Iq5W?#tm%UEuKyXS5GeFUv@;kf)s?e z>*OXo*K6bMj~0bv?{2)sAu|K0<(0Q?>LAIV{S+q6Dd3?v3xdyWo+<7(@TUXlDKxfhKVj$l&NrA*1 z($Ij^v5xt^OD&}{+|!bM=Zkr#w?%elZs*HM_nm^sR>&k3@=e0XCpEe_>w=2tp52Yl z|Il+G_l_VvXEBc*j!ykihFmjqv@@lxn2tO{d5)_0R`N?(7D{m~{PI+6EW@IVSf-^K z2NUw)UBID+1&@C=eLwkI8+#lseVO4GViJAu5gP@4$o=)Mqa3u6lds6H4$@Frs188{ z#Q_qOokIw4@(w00<5JtlUy+lE9AFQNF}(S1*pxkvlsebJXB3=OAwT8FA78MK_phC1 z!L06vC@6M?X^2t;cY0fS46#`YVUXx!L=SPb=108gn6!W5oT+>)=;XCbU*bLl-FQfd zxaF(eJ@#+=f>#Y41WrlX2#GuRcw92%H-u1<_m&3hS)WmG5=d4w+64=>*7V=CY-Im# z_@cR7-E-drN5-qLI314hWt~2J+D5gtv0Ob5dY#h}){S{hpT7WUJ+2S)g^fKw)9I7f zNM*EW9mO;J=$kOtEU&R;FEUYxdlu$XL=9Q?K?5uMZF+f;Bb8x$ z8NA(9#8HvsXTx?;uNM`2A#QgS?&u>}0@gLC6Q8n0OI@9L#<_)kpqHZZ3C_Gc6-O1H zO)4Vvd*3o`E8V1~l#`z*bu)Lt-OApVj-wDXBt{!|+UbvV1=UnK2$9?dvSPC-mksW1 zmpTVk@uDLrR11#veF+RR9ph~#tf4#@6~9Vm>Nu$Zk;@JmTr^2 zL>BbL;4ojai&@E||7F-35uZW7hYc-#B-7j9u9vccAGT>p=R+6yY19h8dD}V#U-1#}9OD z)9ADir)F)u&virBo@I?t*y0R=CgQ!hLAueyqRlQd_-<}#E9^l%X$`_LZExw+)Psm0 zq!kl!T2jQsJ#8D3TbwP~SO)+v3#ldaU$+H}mC6*yf(r0nFvV0b6HlH?G8f$R#l>gy zb!#MU^P4MJ5S0Pyf&|KLX%d@d$5PEX+_Io2XmpeTx(wFJ0KGJRx|E91yu_&4{-VSo z{Caw`IV-oqV^&`n=O2Z~Wh`#fA2}0oN>bcD0rY8J6ajiP?>;*&PqmKIKidr)O>V6+^37ORUP+^#k2!ehP5M(i#^GN0ofSXX4Mhh4X zQ@Yzre&OK3s_>)UhZ$%FoosZpz07l>U)Vu7it8vN-xHYsQ;~m_eyjYuZ`OsXndJcI zktuU{*TQM=&$`*4wT7RJ29tZcRKdWwU@H1|`Y#!8Z&1M5u2lnTCJ=yV1~BCz6djT1 z>Ao3kU{$H(#Cc_O9(z*C)JE1BYaAIa0!>WpPyI>@s zZqJeaSGCn$B&1*%#G=KBVex)6Q{GvmfZ3d40Sr-fc#D64A^th+cp z0*pHnp`1tG-WO4KJFQDdL*l`Z@pd+phB}hv2rp*aUFbt2CH#^wRqPVj9O3}4ajEKP zuK)_p0B_o9=w(so)7%aC8zI5O2=JHKIpk}|ag_A6%#R0QpucT{QrmK~pnhGT?U#&+ zbEKU(w#9q%@C;AmXSK@yv{j;gh7mmOW~gzEs7WD%!a5-Xa9`-_BndAQa*OrvqK35r zJuNYC_%6w*=!#m1K|*E+fSfu|QFyQ{R^Q$ILt)+Z7!0*>?l^!M?S|a)R79K)C|xEb z4u+Kr$RS4s9jhb$sVXd{02~0(1h-CpN|y(-4_8S#j@E*>AC6#@LqT6Kw*tD-aL|=v zX+J%qmY{cIAe*6}gd{TaVRmyDQ6cX3fI79zcNt&i=WI5&{qq^XN} zl;}}+Xj|biN%bFm->}BtzAp{~{Arqr2c#s`0FGJU8T4I5MtnbRf7$bOyaz^|oUp&> zzM}d1pS{`Ij|*I;YM08O-t5YUond;qdv5Yg?T6yBRk{*QExsgD40MpIQxPdwipcbx zyyReRGNj~p>X>z#fZ+=zD2_vvCb;C_c4`lqhIdbvuX*L-x z#qde15Kd-!JQI4`{yJ}#{147AtJzna=55$&dNs?NbK_EV5W$zueDOYPN@iwNiPe2E zDky#J;h|hXeYtCZfPrgBd^xc8?jqdgZFI~(GCd4biiu)LS18Dl|F>2QjrA==p?#e} z<+B<5Gkua^rgI86Oax8D;4_2wgEGTln|;1O^lu6f{-;ZDnUK6#AGvReyc;K(jJTj6 zG8NxfN)m29Wj6uZBV($hVe4dx0q9zZe3=S2HxDA2%{hGa$V-%?uezYdW|4E2xNM7T zUGhQ{_kpq%04o?2Dp6ZKm%P+OSslCENCtd5k2)97WhKp+7$TV}3 zTnXG6KwmU8S4&d)TF6$M{dnLrLyOru{czP<(Ixqd(q4ZG`My8c%egf;ENjxXBr2_X zXU3782V7OUSexu39_zv@7iyuHAhApc@Xfzd^M~rR5X_vtXY}VFZJsczgOzrUR_U-i zIa%JQlhcpyPC8aR*!M%3ke_i1qXY1+A{9GYQYz@W0&09AMbhZ2t%XV^3!jLP@wsM< zeD~Pf{P$EJUCw0~MDBKyuuq5Q@DQ<6L!c@u#G?}eP@^swyv?|36YV4Y>t)K3N}O{= zRzW6t=MAklM>`&fW|Sn_4#|+VqsgC{KFP~e{#RGHc=-B{t}xY$cQaF|f&_KScty`+ z5z%3Kk-ejVb*Z|6H|b*B?P*Ty^K4<{C9DQZ!0gD#U|HOsY@?1(*S}S^qxKcYUgw^5 z&2N=0B{@iaL4ubm;r*80Q;Jm{uDblg)z5FWPcjUYuMq^Ap4V~i-?oO0#(DE~w|AZX zA+kTnlehjMvh_;6U^GVlo}bpEpaJ2HLPmWS#(#@!qhs2B&F`Cy(+_;sykv3F zG|_Q-C{P~dpFVfW`en%Ah0tVv1!|nYCr$11Aw)QEC4>z8=z1CT;a=Mn)@8yWYPe>9 z%iS^=F^)6rCs~vyk8wAK}FIZDn*DTwGJ^6K)w4H)7w@ag#hh7_1$#*F+Bohw1@2T(3x!_rQYHvDunCKhf31Y^Iv^G(NFuy zIzJ3^R!=AP$ZeVX5T6{GP+}wepL@e7Icj0IT$g;% z^5a)89*adR?-6~Y(cP_$@KsBKumL{7j z)>WYXx&0RgM^Mq>)xUb@pa1Ti+Zo2(2ij=+_>=gGXHx(M+71>HgZx#TFFsG$QJq)XZ!{q<>4nt9mjw=R^Vv#<{OD@bWOzz8woa-Iv{x?wE;_EmJy;L%%rK!7`k1nWJZdz#G(Ub> zbC)y{5y~^Wkot&8UH`m-I`L`?t|7L@en_p{%rB8AY;YS- zv$O6(bJ_|TeB5X_jt$9ruN&KXFc7dn#Ct{vw4p{c#0D}3EPO8`9Ihglk1=WFLL?BL_LTXB0>;nW>6S$7Q9KQeT`CJ5h(4|-1P?}46UD!j zh(0bzu)Z&n8y%8g6u}8pf^xuIFOf%I-_Pq$22O~7@i+)WA5vnzxHz*4-n7lD1jor+ z>HFN1wB>&94jA|5Bipa8oeZh*YNLfyR037hR54H0vt;XjqM~_W^8cLKGzANQHWAu~ zPLmYhFY@Fhb%BB-LT!^SxHM21K zV7-3CGkMs(=<%TlGfJ+eAf@~Dgv^l6fx>jhFyVKy4((wRwESK$-M(0grLubO>y3wz zxVYBNO5^@P@CtTV!7G#r*Z=3RD92++bvE@w(k|Y63GtlvxmtX)&{$rdbs#o$Amo%@ z%y3})>S(7uuyl!u<{XC=L>XeJ_|k8D3B}p(^A~Z8QWnlZa=|feoJTtZ^ZMZ-eBYZ8Sk(un2i}sb z3O?STik@?n*Ka30$AF&B=SV`-wb~a9J`9nnX6V)dK-P2YBP6a<CG3X1s z3;M!3gN@Wi@jqc&B6C12^G;1nUWa4xsBCT-M8GB79lwhb96Tye1L)6(8%pzO-OxlO z;4c*$(AIxPN$N)7W28&3XmT9N&z)|eZ%4x-tqv54+m5QMybF2~lb1IAhDT^kxOE_}=O)z1oBSM?Z5g>#mXL7~fLy#sh{$R46h~tj ziTPJF5m(?YJj%TObmSemzSOwQY|PuhW;&HZ(eMR%%JaIKA`M3`(f>wIda4JFMNZ6c zZ__U+>nOg&TRJuz7Ec|rPQ`&xE(iW*wt$A*;g=7GAQFV@OsOaJi#j>x=WUZoD1L(< zTySHshgyzEIfq?Lo^Os2m5jaB$E&OeHIxefn5Hxfev=|O2T7~^cx;IFb+c!Vtn5*) z8@)SDPXll56#Tu%XJn5Y;%5HH910w+4f*0(`%hGy{;iPa=IS2v-5qvirxBo*DA<%Y zQ<0vI1$%_887bWQ41Gy`=x}=r(t% z{f$#Q>Fvt#sma%dKl6rs&(DtBdpR-xA! z|M$k(`|l|bsHT=Px%^-G6pUc5eFV)2Srh8UH7Lcq30=4p-wIyCKJgV9#G2_0sIZvm z(#z2N)0luY!jj3W4rfhajtG^mM9>bQv7%L&+4YR3SF0@uZPLwWEjyw_Ly9_jgigY( z^TJ+?8u9wI;g3;?&_6D6Ec;Ky%C>ehb0i3{68Q{5tZ--hM@u9^ z$Lao36c4rf)0tqUF-W8<2?^{iV-|lx?sKdKbyW6V@Rc+Xo^=V(|$fVRb7W&?|nkh>QwArKf@?Q$-bNV0?fQEpan4 zm^HRiqzRXp;H$_t#!ywiouN7MygjoWL(yg`p<;TS?h2G&dxLNkDhxt}Eodc0hT9Bq ztF;(fslUaMTB&V7bp%62BQ=)B0z;Ff0w)JV+E9sIbGdov2ZZA?W|izu6^Ec$7rOP! zLh?PEZN!HAcMk?bkdGfc8LZocgJ54#AlMgqPLJO4LeVEj&<*YcpWF8hk+K)8TOV|X z>lT3OMhqKBMy(k)UBTx&x>YYS;2W@noouBFE zfGNN|S}#L4WFdW0%I(<{v7d}N+k1hFY6o@8NnPhCAf+~b)!y7866Q)yVCXJ?gaOlB zgI@l9k;``9q9N@h1D3!Wk0?3e8&lx(y_`{ZBDX%c`B5Y1rWB}HkjInV$n3lAaZk4S zl0cj1Dwx&7bW~Bi-zwW$NL6NJ{TZ*1{?r&2PzX$oMWeWkjEahrVh3;w8i{pq>2E84 z!1H{DSEem}SYdE;(R|?MufN&r>Mi& zj$-gQpU!>y;LLRX65!SR?nW7jmj{CnY!^B;~d&48)G8_=sHQWv7b$z8;d>sifB+k?Oa=a z-UGo3>a`|04zu61xZ>)+7ZrDeNoCT%Z3&X?h?SD&xUE3b(YQgii;iJ4g=c%P?qA@J zQ{?o;(}VFR%RqSPVlLoadx#rb(*S@{0db9?aSdNB!7#PxVQ5*ai4;wRaTT|JkGPo4 zpb>XANaYO0`e(o{B5O?KZgeF$R}9!FWX7DdfH2n@n@~vQ&nuQiyk<(5Td)4tZq&u7 zw_u-VXgwwFoj5UXftXZPg9m8*unXqam|M(q5|x;cxm?OuI0NHDDuVN&96#mTbdBta zUZB50Lz(l?kI&aAmo*=kG={rre!P{=K0pa6>|3p&du@sy(;4Qrs-n^&tUzNzD`K|J z*DJ*3DN+F2>oMPdjM4l@Bf@j+Po$jo&gN%0uix-%L9L9Nd4U&^v_jvKe_*ep|G-|W z9YmLaX}r4*qcljgZO2)ragrgQGNpCw1yQffra;tdjAL!qzOj$9e+Ep*DF|#TWCpuH z0h6*&mI71lKcLs@S;b}Un-5xipXu+2lR>+&C#2uIF~PrfWAW87{J%>PNxWI+Jj$|c z-ws4o9_dwbK-9>-|C1+2Ci;K#}9GGMwa z*xY^kX5PYxWd^M8tJGgywDrt+&=umDUaBib3O^(^uSw{krGQSe;utGf`)kdWv)4jA z=||FYKFD|1Qb9ohPdvp}7wF?ctTaZX^Z~$+raYuGY>8Y``czZ;nfGeotb(gLPew1< zTy-m?s^*VkME~MHixI|ZDWGD+8ZAZiHR=@4_(z%weGPB#oX(XiS9g@}?%t0}!BzU& zeH@y;Co5U4T&HiKt~ z?)32pUPa^H0UYMs_ZgQC5q7%FjA$M0_R7=njICSwzz=6we%avq6IP8`A6Jv$H#zF_ zYWbW0MnjA(vedH~S$E5E%YVhzvSN4Vz@_?j=GGeZc6Ngj^Z4>N^xMmZgTsH~RNuJ# z5BhDB1UPLTxLN5{l^%H>aJhr8g_%T*; zp+}m*J9--{dFjMqEe1Df4zxh_SxPe|>-vfrLit|7p0Pcta(16bh)(celzh2{7+W( zlFxr&Rp`X8Gd+3^CYqOZR@cO{zo0`!BJ?7x6bv#C31DKfs( zaldytn&+ncXC@tTT0&L`w%LCa7cPjTfzzPkLTd76)F|2xdVCF3r*>w7XvwPi=R>t{ zKC6N(xD4C9h<=9gvlXC?wxL{8t=GQfHnBsVLh0x4lR6>n?B{w z2DOiu+dy?Ap%jy1m9qd`itpFomnwLOBk4oWInZ{(sfIQ$m|O=I-}V<8-X`!*3|wi3 zc9%M@o_TWWYMHb)HO3dS_(G2GyBm58vc=hQ8e1Oy{c$9=VdRZ+G!*!+R6IG)4gkh0e|CfG%$$`k>giLrz~gRZ%TWnirO@py7kC>X+k0fyVE}1SO*eDorIC z4Y1q`ir1cA=|AjEz|D_***-Z7dy72$A)}zGREaW)VXgi1`1ZvaY=jZ~wy6)6;#ap> z>1$=OftS>%D3GV2layXyg&`6Rjst?`(1QYo&KmD|7+HunU-ECs4l`U(l-~AH8`6tu zV0REho>f@%In{gZ&bCoqTZ+R<*1jS<+)h)n8v(kmP1_q!PRjMBFC|~@JKdabS~6!# z(&J1oj#GC`m_1aFpBORyW^ODNLfY^;#qq?`nUl?aj-WKy0{Ewh$wI38o(L!~k(DKk zxYuqxJ_jczX^(oi`sl_Vd{bl+NGBnp?7Lzz>*OgVu;2A|9^g5-1u(xyfp}+4!5pZy zZo&q;g0!>lP{DnmW^lUVw2Q+ESRb34GXo^7T<_V>e9fv8@;%?9INu8vG@xky`1(w4 zWcq_pEtHGiL^jjz4r>LDp33NbLehYq2$LIe>f-=byc6d8Y%8qlCot=5oE91@w~VkB zOM$ zuRC>a{QIoM-ufnS4%IEc;cG1#)@>dGpdD0cMQEWH&JJ5g-avRcI@D9{f5P>&*^rq zcR^Rc-O@PFjY1TO&B0_Fnkz zvI*MCB$KX!+X{OkRy`xd<)7~_Xs6uj{N?iL^-Ee2 zHGM{R2o8zAuk<_6jVp_h6y;^-65+t_>z_KTJtFV=A;4-#lHX;9Bf7{sV99<%0DqX2 zHf)Np!W`u)d_LSPMw!Uiyj3rYIviC-@l(f2x|g`w@d;j?k8UT4`-T*UBD*dgLO{{3 zZDSH{4Kqq|+P4i&CEUjc1`W4q>3!Y}TKeDHGMRR$s))9C_DG`|sJxs$w<=erc=BSH z+OG3&;_2dz6L$V#a+TSR?v%l#Hh zQsGrbQmfY}Vd0xp4&NHbMv31}n8GAtjUq50gQYS&(i~lajhv`{7&4}~E8Nn5OTT>nS;%pAU3W7QEb~i7G}+Ct3+IJe3w}Vqhf8QZ zy4;)Kop*I-JG#BKyiwR?2>vA=iWU9&K#AG|%O-6n4pb8rconRsG7+qXi=nAolcn`1 z5IeB#ESUy)HfA$A@cs;!&_H$<D;9YkRMBRFQrs$osr9u9|^7 zB*ZUaQ6!C~rMWMv+XE6V&pO_;;X|`#rv3~o&xf=V5TZN6)Zu$QI%*w$$}wlFg7`&% zZvxMqmKcq!Gq(N{ZJX3tc&JH;ZHqSpwf={#W-`_|I$V%#gZ3$)r>ryvNAeV{I!SbV zWCKkW9b5+U@;RZxQevv!I@b~&<6Cmnbr|`0#xJ2*(C-w90%S%uYF(C%2>yUe1_ z$di@4gteQg9z}W8DYZ(W=j{$m1fNPGAqt%0gAju&dF?pe3bA(L+8a(u6ko&|9TNmh zaXcaw?r)M|)NSy@>28@zylC7>`r30i2y`bsE;+jOPqo^}|wfYVl~#91&7IbvrIHn);^UuS5h z7t5bg%@zDNeX)I37=&v{bIF^y?XEBLKHVfR@$B%e_S_%pz3qBi5Mnj;uD51~8Y%ke z+sptfia;HEQ7)t1YZ$WnhIjftTM^u}DL31DpNwl^(T@@9UT`iy3D^#e*6?6?hu}Dp zlkaAo7_pt{a`s6QC0vGu?14w{gZgA-;j9Z4)<5SMye2MfJ5e1(@^5zbIFO7Gvg8d` zzJr-Rk!fE!{w=VM)tNX51FJ0o`9?|WRlIcdppysc)uWkX*(wgV>mxUKUvG-k4UHKarTN1h>Za2i_3Fu8(0 z-Evi-3b_G)`QZEsZ}QVhsD%5Ej!caA(yjL`W$xh2uf=YEY<-8_L{QV@Gry4 z1*50&O>5yY7)uob;|*uf+`RZO?~V4cFQgc`S4{m!RDT%uA@gp4cbkWQJlPk=({`yc zeGNjZ%G3!#sY}~RHMw%@*Kn@akY{q# zO5?mA>(z7Q=~UMQBEr1JgZDUjCOTg(#)?4WEoEtRc}Ed}>~}=D^4OyvO7W_$-rLJc zO|SCfElM+4{osiQes;qn*>dN>z-E3_n5#d%7L6E@j-C^u#Ms#wl2t-c%!dr6MDS=H z>#S|20Ys}9Q?GAg;}cgvt>Uo+px?S~lopF;1JilQQ}`Y|cd$IvteN~}<_-kSozzw$ zK@w$0=>)|8;VdrcP=lz8G%e%|A>u|eA>=I1E0L_LSXd>Gw?jnFuxC#Q9xa5RtI&B! zB=U*xCIPfX0QRHdWBw|rfakXj_QHSR=ex zfI7B(_*wQ~q!bV)D4_{w{{Wu$O+N$2WA#R9oQNOpmql&K-_|2koqYPf%+f;)4* za=o~gWfYdl4)(0hdEEZ3sLB*lA4w%ar-c#kzv8<3DKS^~5x;8rCDDE4RrWPk56XS{ zAh_~4iF*kwrnquc6y+{6fl&D~JW!7kGG-(S|vZw`U;f`8!_8TC7*aZAr!2PsZHHTEu`= zjw$ymVmzp9>mo(A(?QWy+<0C6C?FwyJdo;!Zz}jZKLodvgo**?2PqxE{IJt(zGhsC z9B-Gb&gyX6`8xBBYS?3Z2dRG7_fpP36spO{iixaU`eZv;2xj>xhA zx9IiBdF0R5u!`N>L7UXZEgmk>+?#Xai}qsSKE7I)SQ_EZ+rM z4oy1f8bh3r6>15$7q+W|^)jp@)sUQ2YVoO3 z>$w4~@zp;|OW3f2K!C4fQ?D@_$hu^3zIeZ?|c0WnpGZ>7_ zTYVneAbs3&zwOn&^&heoeI1?w>Ir~C^m<=~V~4LF#^@jXjp+oz+3P^0rtHfJJ%wPY z2+6*+k3PZ8gVqv|-S3F|&Q}M1)Oz^j0RYm+{F?$`Gaq!&lFBo0)6)G9d;JY#D!A}? zUau*PEfYScuKODW9=MV$De1HN2d3XUyb5Ednn*)!lQUity6Jm07cVv(d-f`3Axf_M zx|>>Z@-sHZWSv@sWg=Ic4W{_f_D&{sFxVc!$3|8PtQ-+36w0pcY(4MDxV2@?uFdZp zH440@Nt_Y5aa1DUEk1w_RWdf)(uc~*7AMeQ^JZ+8HrgutXHVLlTb(+-$;{`X$CDOF zYm+qka&`g8VXqBq7v1SS7bUvSM>)gqYx!2P{S|K@Ib{A+C!Izh` z3*e5WnGo{anp1gypt1IFhlO7V*WSnwszOBP#IK|mN&EB(!?<7%@u?uHhznE+9{jIM z-7>TY-)hh6l~H>W%W2+9SV=Y#S6d%}8q0BAWN;pl<1obMrQ&gkYII!;mf0N5g@&A( z(e-_@{Nd2Z+GZ|pq7lV7PG-q7zJZX1ZuH{w&{|1cHa#Xogz#`HWNa5R~t_14qsZlI9 zoiLjPTlGlU1z6!4vf)+hMcYPh*4}6nPM>ykn>gr$q>=+3{b}o76)DYH0#~iGX0z^) zUhyksI0EZOH=fG)+acwOKznWEM< zgQiZmmtV^=As1&Yo{t@2fg_%Y*BVZwg%eA5sHZt38y7BG1REl_akvZu6w_q-jIi~z zlg0=QRNU?IS9k^)TNZ&*EJ4{caXVsaClQ?YYf*NIuvbrG!FHvMb7<(3vph@tkE*HZ zH~pt?3+n-}ufNWmi@Q?5>iDs6>nD4q^NS=*xHQPv)i=lKqCcFv`x0MsAT%0cU&U-q z+$9ZRFUk$yB}g1PI)sDPL-bK0uMsQZw1u#NHYaQ69b~`^X<3CE&~e&?8D#92)X!42 z??)Vt%o>3S$sl|_dUO20dsubq_9G(p3ORVVF%WD6>yu>FfG({JC`?!bJsr$I59#yn z+^<6>+??q5)bjbEWuejo`HL1*c3{R!cmB|9zH&wy$bxF1FqDfWGo0~C+!@KLgokW^ zHguurq*?FDsD%P@n1u$}UFHkmpbY_bP?r>Nb6Fk9mT~Zy>dwJ>i{;69W}Ptte+uz@ z!6XUlNXqs5l<>mDoEUUDZX_dM$c}v-k^NEJ&WZ_?_*FrccWW%QamU4#CIjl566Uvl z)Sv@hNHkH4BbN{U9FddIV5>O8j(9SB$%wm32H^}IOoUsr$_3cBLtPB9$Td<$hV1Ob z4z40LxH$60M@F-`a96EOsU0jfRbt$21kFs*ZXGTcC|H%>3G)rwdb73Qx$*Kd=|R2G zHw&;76kjWSqVmFPQk81gyB)-MPW31p3>3|G9#&6^mRZIvLj9<<52Df-@qkSxCUEHN z5Oh+!2kJMNhY%U6P__%5Jymy084I4OsQa;Mqv4lED@|IxIVowmEA5MhcCT=*{EShp zdbF~dMJ`-=MP=M~R*gOlj?iUoxwrt!a z&)A^pc%x%bv?#$06J#ekng`SyeIrvvXp#S^+K;Z;Ri5Ac59hldzM#k_1?Lawylw=6 znS7CH-v2l*4%M4_NQG$62h)<>^!^zI{FXlh$z=6iI36UCAUtLNjsjhwk-j*FDxA@d}&Q8WkT{@JJ9S(HhsS{jHjv_+CYUOW&KH z`HYa2Nf4sS;2vuG0AqoFuH-qSMQz#u#|%?yvLhCR=z5OWom{qe?S^to;!T2;UZ}`o z#>e~}quDn^7H*QjY1i7s^Gz;nCGk`twX9Z;S->)z4bp|U=0~IcysI0oVw*|)u9b?MQWeyfQ)uR^LsW*&oP#EGH zHB_WbUXNyln#k|%TIja=r`;As6P)a_wZ!#Ml4{hNu-VaG+O>Br(D9oG;>mwA3nnY0W>Uf0Ox2fyJ^x!K6uyY9?Xe!+L(-*u5M z<9vj$K{fro(c=EiJC-_GINzqe9$UNsb3x8UDQ$V=ZPKEAx%YaxR2Pfu&PI39G?&P4 zrP}<4{!`@0;*pOH0^tQ?)lT}^gbCf{>7(M?;R!Wq14Rmn@iz)fA5=|C0j+mZnXSVa zrL!*U!Z>|Gs<9;V7tOytzozWmHjYQ>4=BRPokAvLGWTi#*2N zY7;lQ7DD((--FjC`@3M!2lNjDXDM?r1x(0`{PpML#gThmrF4p&$g(BzyPp*jG!I1f z7gQTQlqDBhjBcM8qE#QQ$mt$!i`Z~f_adD@3U{W{hD77H#OTpmx$u3IiSoar-^`+xVCv+>N=O)>2>u?B7_O+sZn81NqTxAA2UePiltNMX}4*FQe zL`FMTlbA^7;v!)sLx~v5Mp9Kr6v%*@(6Am)92zg1jR~C<;?hv^ro2;+ct>D)j010` zZ})HB;uU>ZnjN8VMQCsa3mRj?*%qWLNauXyr(^ z2%DawFZ2$Jk!&Ptu*IZB#{9zLlQVm%~J}T1VzsxV1n|wojL5N@Ee*aK%0W`>|)$ZXO z!M(F2vg{ynzMF?r7R~Rn9RFp*p&&EMNt;clQCx;``->Je4zaR62Z7fo+g+wykFD7& zrF2NG=!BhL$NLXUS@q1x0o@8EU0sA`G|>i})Y_JFl6~SI?Nk}<@ISvBZdKU(BGy>V z=v3sLNmX2NI?B}1B%MnRIN@zuHq@Xm6}+r~#jaTE39*y_9BvKp+jI`c=DJv#o6;&rsru~x&vpL(2!agS9d$6v~@Ho zok*gB&!;VyVR>bhe;r%(CUwd)`vm%);uvp6!ThVS!`Vc-ap)GawPu#IY0efFfdfyS8~HC8(HOaD)&eakP-Yc)TrOlXb0^{Sp6Lef7|RjLnx zH@BL`aVQ68Nz@eV$MJcPYgtUSr7N3!>at9gwK7lJ-!J7@Y5BMh+ZW}CvrEF(PG4)< zuSHxp>02q|rTc3~;=xm5^-!#wj{S1}pnbOaz-ZiMXAOBRldN__X7o1Oi6gQFRwI&8 zQd;Qy@I5y?xOcZrql-;U%bUu9cFrqoUbg+TRQJ_8VGb4Dk9Q5jM`a_%yx(xJM4Li7%m~huXnLFB8`Gn zGv;6pkIcqihv6Xt`MHIl9v1lG@y>p5vP5lpuNqO?%&pAly+RK8v^XQ3sEo1+Whi`l zy)fo8e>cLl4n$yY6=vf zdpU_C{Q((d;6eJ+RBn|TOMr`?X$V-K0GxvjZy|mOrazpyuy6-xT2Rd9zT?u`VI89< zH0I#RaNxGv+ao=I0Fx`y$;l@w1yA@B z>ar;DF($5I_wi?MC+SarS>!PaapLO9E9lXmqyZgP2qS|pA-_8(e!p^&0vjW}EsV5` z*DKo8jGCO7!7m%PnQ05Q4>;~-;gY7&g9qS}?vF-1T&wS3{f>_b7W~G_vQO2v?*a)N zK3Qy=>nvj>&~cYp!&yXaXkRDQ0@lySUKDq zl`?0F{>YXX#1-q;5bM?Pzsbwn^wm>anPV&7tQFw-;J^84)?B!h#vM|AjFkPI$)iZg z^U$$43d+5B+$}ibvGb*e#6*Wubcd5wkE1X4N@1|f&di@%tj8qvLo3cX)NX{+dIg9qB1BoQ9KGz``8&`USc@)z3}FS=uSz5OmK?L zbM4*1gDNJB5UkK8amDK4_1Ovi)0~vj#mifU%=#I8Cp|m7J>{+zZi+3kr3*YW>5qz3 zlu}%|CQ++c*`m}XoLsC`OY)JqCr-?OCZr9^_F&P}#^@_U&KB zSZKPPgCn*C zBoHys{@uxwY+PmJBz&$1-pt$pE5>$R47gl3!zT%w#xW zSXd~Ak^#h831DHSOpu0>QG|w)Erfty=t=+n-3uEo5-kIqAXPB2AYC?5m&{tRp$#nU zh*&F0FJ9_m8A^H)&;L0lC@GH-tMYCWT`xVwpjp|sSNgb><`Jv@2nT&9tKjV|r5rAb z>Ck5xh;ucO{kWWqKJP%Y%`wtJjXF8Q&f;5gmSVWi>FZq`_x9r#^59gdafJR;CssKY% zk2`X6-CSvM2m>i2`FUt%Dcxl@`zfdc)ZEYm)Xn+8LZxCYWlwF?$Iw% zm|2<}1 zs~n1H1Djlo$rIQa+Es$bd{e5Qnpw`bc{@ol$jrcQRL-Yot6G-!E9Okv9HB0uF0EBKX!C} zuY2pNAcDT+n{(z;6MU>R>E^J&oOvRh=tlwZ+~r_v)adfu^Dw0mnJ>UumP_Gpax50AD`L}Fq%8M7qe9n#z zY`qR0@bU&~(|`iCtxxc~IH`f%O5eR%U0pLi{G2cUb?ccU9-DC$cN0Jlwe0EC6E9UD zuHf&3&0ORw-N-u6@3JWPPSzy&4s;FL5-&dSUd#s4tk|PA5R&%E_m8V4+73g z35Aw$0}N9kj73%OuZoD4@VOPRe~Q8f3+8~<*@zLcrrVG%1r(B>4gr-v4iW+hg@~k2 z`mvmd8}JkO2gB)aRb)J%!`2|6LpQ{M7r+jw-{^}9J2`Gfe-^iPqK;nVq3GEO#~aJ7 zVl9Q^P6nyO%5%TWQwD=JCdy&2pnv~j5CwDEa@{@gPw{Y)!=tOa^OEM(Z)0Co<0K10 zmI?}!aG-8(F-ZAGq{L;=@|-FST-I(3GEBmL`1aF%u^Nsa_6C`Y*vM+%)V)|pz6W8V z1j$9JLYjM^)`gDgO?hR|In9F5)zw}SNm(xWC0>+yhl#JCUioJtt9zVLk~dE6#K7tp zRsxy6d%3PcuK2I~3CemC@W18H7=^(afXFNJ$lKsap5jZ=`kzKKi6Y`D3#KzH0!gb}sm9+UnB_|xJ5*(ZT%M|q za=PKOt82F7O&bV$GC7P=>ngsQ9s9SLx-SZL*8R}sJ`MZb7yYoYYj;{yM`UkQl{#@7 z_2WA>Y5G__vfTbX#6b%OEv9u{q6;K!$>%#0;mK`N4-T0jm) z5e!%qyf)o>h0(`Fn?IvXVZ~%5b?%dJk>13)!($%oO-AaL_H&srr+qqkl9Qb4*>@f` z2Y)7dXlg=bz^8>UGw5DDg+d!WQ-qMH`Bgy!s>9fM@Ru2@d|eKg_j;UY{%~uSk{8Np zm3DHl_#4`Tjn=)B=PS9>3iMmi$x)t@zk5+`8mIBM)~kxH36Dw{ATCy&hX=ZqJpMX| z*Bqo7yEvOp+4a=I|G3oGVW{oBEUNb1aypS%tdF*n;$x;_f&NdVP%G)mg|!0<_d?%w zE2fQJ#oykr1M{6d1}_gh=AlB`IhNB|F_QR`gG%$hpZ=zO%FDt6qeR(FOxA^SE>wFk zYG_SvK|BsyW0dS%*Uygqu~N?!{>uXVTunUX#S4kK3eLG+E{6fte>o#kC!$_vFMxC$ z5)n3rovN$IZcnbSgVK^j({3JDBVN97-foUC97aTtUgAVrK{ms4)tTkcvUUCB)L;K; z+rD-0?=D|-s#fm5&BtZ`{C#J(+Wq(0A8h6ETh7yhpqkhxI-=@)-=T7uu@gK5C z)KCf!ZA0_HP6*K-_lx)cO+)Spa}=1&j81b7|1S#i>E9SL-=M85 zk{SHo2>UDQTOct#9>Do}9`7s+%34G(L`nk!aK0k^qaQEKyHIy?Z^=|6YZx_dvp@GF zD0!`AApsa_R65i7q@aY3qlxHHZ8lXZJDZ^E>Ryg-%Ne7jcWz|vg&@YsKya>42;^m8 zYy!rFh?O$rn30P2-dPjn?v~W$tT|ZZdqt!nnZZw;noW9@Q(nJ6B4D#~Q1^~K zSi0Q&u3Z3#$Uc^%+a1mGYnDT%(UORGpqcK_Y|Zt=MuzxvmPk3?3GkEBKgTE610ber3uFGjXpXsW1-nA;pmw z4z_gNymlk8v9zsu$~_kDNJzF1GN$?F#Rlg)WtVmjI-7%-y;M6rkUDqL%vHVQw;i%% z8;Ke#WD0%c4qr*fYZSqrcFa&@n4Yo_sN8u8f(T@fl0=QC0RIWt@X0iEQ%2BNE-Z~; z3OGtLSq0I@SSg!WdQG6>10O(Pu3lH@!}!*LphUF|%8J4J0695uRc?hi@8<;Vk0`}S zYayuF*HE4Ipm3}BzMq7zi_vLty6Nol4nphhMXcJyz*wo##qo>IH-DA(1GrS+2*;lq za4B>S>@*mGwb8p7ZTWg>;aa5Q`TxU-Yt7DX1+GUf_Vw->!plTUWm($;()}WqV@<_j$Zh(7q zso^}mBsN><_4U7Z>x|u+UY#xNmV)_xc}=8?>7i|#hX!Ly z^C-j$a6a`9;M*e~3}^}!0lOgn9F5}tf{7!6vdcya=hWiTU?4jvibB!)BGpn+U7_Q9 zeNnnB6|nC#{|HzG)-D%imADk`Dj^|jGQvBzE((E?%qJHj@^wW_;7=VAe7lm&?qgg$ zmFf8fGzy88pE+L4%x|`jfkg-wGuOt*-%Oi3IYbeF38G)IUpWYy$_S21cMlDR~^oUNUAM{8Rh^rmk z86?9;n^6lR_QAH{Xy8YC041=3J%NJXP`JZp7XdA%p>!^hq4W-DMw2oY<8fSzm7Z+pgZo%p zCx({S`f*kLlr9C(^;+0dInXum=4CUGr2#NN!^uw3tq5wFY*0aR5Jl($Wm4z@XMNLWp;Cnm zkqu5hvCb#4C0(qX{#+k0B3i-K8IjewM(4tVLrx1Tpk|B+VDlMg$Z41iw{*a%G1vRv z9eiWyDo<8M?^zlsG#<%IAFLSl8XyD&0gMF}OU4LwPr@@!G(LchVBf$4HNH+Ni+~~m z`io%yd!*%ewTtw7L@J(dT&qL_S1v&T8*X!?ChntJY9fW{Kn%c50(_=x{sj@sZXW# zp3$s1N7SH83$-0|6F6`uZA66WULtpy1pDRyT9zx*lA6(WC96Bo0>${vP#d5JZh9qJ z1X)RBbj8D4B|Cr=fs4K;L_FO}1KN76!GY&52!L8rhkz{q9-#R?G(nv=;2FNInN;#5MEP}pp&+%m>#)k^*2(Arf-8h z@x^~+4Kp?vj!r|=xW$o+8%$CKx_)n@nE;J`Pb^BC#sR0r0$3Cyx?f3Mqxy))skFbL zZz0GeFh$aZsilZj<@|S9a-*`8R1KE$07WN@K3BQd+W83vt8=1;Y?`B z>iv@dnS;oZ`&!)JY+)*hLB+hgP-K@DrM|-np2AKBy$TQ5;)Iy`{l^A`+OAL!<4!+J zxaq!&1wi;^G82UZ*dV84k2<2lNgnqDGM{-3J7lGrd`(d@H@U+YhHgqmQYSi>U-4WqZ>s z$lYa~yw4gXSv?(qej0~4q5dEG>11eYg43dJ@G^R#1%55*w}k~O8-wSEdY-(S>%=EJ ziLb7;Rhi2(i|T*Ca`}S+<`!VvTLNL?WvZq@ODTPf*(3SQ(_=mU&KBzRmcDo6YUtnk zAAY{ePe15XF4GVZjXaBjtG^yf{i5xi>`ZduZz%#9Yz)4+joNmP&RrhBEXoJ>cz_pPEAFz)yjmd zR(H7ot58i97;#ZWwSIUs%6*3T8AiZ{0}6cJigV_HVk>0kBccQW1wO&z+itJ$W+q3< z?rv@A>ZEl;#+`OIUc|+Bb!^0SYYmi|;|1j4#7+~*n6RzZRONpEfqX%){vu!Ad+7QY zfJSN>QT#t@r0)$WDmZHaNe&ChL(@XUDMMO5|KXe5#ryBRNz%Um!8ZwU+a2S$ z`Ga3>F}Lt?b{E}j7W-{{BO*?~$bXI$MM7q!nJ=V>ydY{x(N*JEBI|#2OwwwZT6y>U z19tfVV3+Fz%QtqlcUpXm?@{Bz7g~b!dvr}0x<3lbXz5)Q*#Ff>*$U5{ssEPuh-8R< zaB8$l!_TIqDl8wg3Z>C#<%p!zspG8J|1Iyy^%5oztBDJJD*oB4z-zE7=jRyESG4!T za&jsvPu}ML$f86!+ja2Xa1?EGVkq;QwP%uai@o*9U8<87C01?$(+zo~U!ybYQE zExI7LQ{3iXtrI^e)Q=n29&2JK0~RA<=s}gY!mAAqB993wkBl%+DEjLZ#q%u$xU{rg zlTTK(62@y?G9aa?{NiBZLXbH{laU#=@6U7p*T?N>xkBu{%LCz;(}|ZwpwfPMyjcrX z<^4`dK$hYdaSh3I6K_BF%w1x5>2sMW^_8W(w0MS*XJW3r2|AN&g|1oiANEBygvLtD zdqeSsjuh;Q=NwNsZvKuAze!su(_Gk#uJNB7(^Q`^P1X6KI>L^#k*CWn`Y>Owy@^pc zqvvJuQu*{1y+Tn?#B)LS1U5N;u5d+*NU*5pd<%R$F9L^z9W=4bn9~oeASMxy3645N z8CCO+6)T}gGGv~T!Y9AR!WpGuM)KdOUT12a4eo%XKo5)(0E zCPtOIM|YdE3^rM$0CDX26WJhyyZhRYvb0ECJ<{m+x-VK0PW2lsPLR18O z4ca1uf(RTIMo7B8XSa|C($i0&xO6718*G@;XhzcMXA-~+d=fLhUTJ6+MeJbd7sh1o zWic?s5ao6#`n|ENLJjr);)Zy2SdXSoa2@r78YxKb4`OFz|8{n~{yx25aHo+K?N`pQ z_ziftWg+t*WDEvPw03|caRf$?W#Q3=ZHwXtS1OOoA6=yH@6i(TBl*~19@7LJRbt9i zJW#?8z_V$2Tpk6CE{3pI$%76)o&mcB2#XTip>k&AiXD|#e+&oYwqmmpIN_@dGo=e%M>+x`5kv zqAk>(XNlC7;>OYP7YOIs_!^Jp_*$87d?>eZgVDwip$qOo2K3$5M-G?-^-(dB0eZMrgXP96$M@zxQ?H*-vm4Ai!7)9%UU^q)nt2-Fzj! zdkvkT>)qI-F95lIzq505)B!X9qu21->k?)P#`UU+o~QaVm5vrlQ17}h&B;46graiO(6WEl{&TXO0ddGS(xF2*H1+>HXvpOBE!Y5 zj0b@th)?B}0U!z35uiBpPEb5>&NrW*R2{$Bp_6t&Bxw@B62OS4pMfN$n`ywMp2$o! z7_iAk;DLzF2L%_$svsZ|RH-0{sHgA+Q36U%VUyB!Z_?&&qK`hGM4B8(8aM@nDwP+gbg@R6>iFl@ zOVKja3m%b)s@s=$L}rOSSlZ^ylFa4&ugO9^o=hz^xPdZAz74V_R>!+0eWZfqo51As zsZd}JHR)Rtdu_lxoX0;Bzw&&4ZDnoqe_eP8|GMz}7X}Y#o`qx%em7Z-gLAO`H?WKM z57<3AgA_*)Ne$4KvS0Ch%|WlXEm9PB6Y4>DYokz|fdN-1#f|ybf)`wWUSgyMq6YxG z2x-&c;8Q3v-(*fnm;QyiAn04*f1z$oAOPxe>z)4RG|r~ZOh20MlB5I(0P5nJ3%htFQfHO^g{dSZ9eyl3fV*Zi|8Zzt$?-;swNBF1+gsHn zcSO3NR|gwh{mVY~wUmXQaq!dKDcT8tOr?(} zs;Rx<{h_Tj{mChP?NHnHuTO7b50(O(91Y@#gg(g6J&ftkg9)b@EIVUDb<_8jg&bs3 z00E8x193$GakZVj(d3n7(4kZc4=dBAG;Q&(jSp%2KN39$RG7^2T@2mRvhiZIq&we7 z{*RGQ;rU7Oy{*`P$L6xstt4b+rSF>a9^+(fq^*Pi`zU<#JNYW+wWa~9Z)CUchs}AqzbVBI*;&a)5mcUNj@PY)Ru6@4JMdxk7~H^lD5k*i=$M zs-^MNglfZrmHXQ_gI`sk))Jydb9v|*uz zZd#o!L*AA6b^kxoJ=O$@|D%sj)2D|QPz0eQZy6xIOl~PEMwCdjE<}3M^(U-}$@%{m zJDWmV&o=n9vYU-zEo*ZB%8k;kh|n1lBK$YqGpF=FrF+gz0T}gSMs%Y649e)Fz?VLa z0%_>iJo)V6J2ECa7iHhnt$Fek@Frq}hupBl&FkOgjMsXs>)F@hTZ_w$5dC*>d#o^`a+|WL8 z!O)Gfj`C0GUWT8Rq7I2G4F~l`fQ^l+wvPYLq)*|0N&3KKfLT}%DQv#|X8>#ZUjQtr zPQ!l!up;ymHD~8A5+A7Ve#%{*yR;XBCC>o4N|GP%QBtgZOb(ws`!+-s_ptH-K-6vn zh+5C)m5qOhTJV%q$~;I=AwR@QP1nq0Gv_MBA$eRn%}5b5){C?s$Tdu)%7-u!z8RRA zHtZ5$RQCI%7Rrtqs4vLj4Bw8*pubOOxI)BXHY5SUKFmFub{dK_2Pxw0Z;fes1&ewZ4z3-5CE3i2p|noF5SK*jlRk z<#jnH-yiSYv7V`qk=LN@sdo}@ip)=&hGi^H_aNUr^lbgFVa?0KIMfp3 z^7q_2*^2BVbTlnV5RJcM=Djan#eHTya!IAEuoRs*#!fL^KpfFHS*!@59=G{zaP;sM zHW(umciuPy(`~1-N-g%zsnK4!tq*duB{{)u&yn~6iL$#|W8MOa-ohe8-0zh1o0OIrhNkF1Z)=)wEBdpXfMV8;K`_KA!2eR4bX+pvHr2Qpk zRO}Z`-A*T2tI%s)r8PCDN9!KZWRH562=3hW@QV8<)~p^L2U^NosAsHMHl@nDy=Z|o zJHnz_tNJ?;@kF*#Bag(`ukoFE6E&KyPQxuQKQqUEOdsNdtO&Rsg=jWnTKSv0L4Z=( zzQdRx+SA>lPwC&=l7joPQ<^G4hoky|OZZt~D32;vu0#_rDRgbPZ^Gr}Vbz-CZ=kJ` ztQQYs)d}pishxqDn%VliBu<#^BbksWY_j?u&dB0o^y3(m{88@Dl>@uGN32m^2L|Zm zC_OY8{f=;f`NtpKhc+L2Wq>a(+VfvuTv!uTKzLT^;ovK_{MGyoHN?XD_N$)eVZq9@2f{AS4C24ZJP1!r(ca`c<945QwB zSfV3SL-Fnh@=i*A=Hh8L$ZI`1ooYAblD+c)1T8SYL@KURe>gBiGmFDMWoH|-*E!?7 z9lqmiqq07q(+N3?=|!E;d^LKg7n%dH0df#y)(lATl+yso_mTn4dL`-s=b++9@?)&y zmRQtXwP%-?hqnq@_FRFZWs%cx-o+zWe|`4Ij8{NGKfKV)%DmCII1uc(ev}bdLMR#(laG?P)#otN_V9i z8AM=oGK_yN1>R>BN5~Mw74fBpHl#fZHsAy-|1ml+->iB>FEoF@86ux8KmjPkV#%#eIq_IRutzD1@aWo+s9Z=Z9@zSA$C1RJ#Z2(SIf-~I` zmuskf02xFA3Xj+^1RQ(i1ZoOya@H|q<+lV5>l>C5plkIZD}t(h?(6lqhz^Vzg-5i| z(c>EgHH{~Zf5>8bp(8QH1m3FO337@Y(Kj$s;L#oUnC@gxxS+(SlKd{FOOmb9cBGgT zn_Mp=C~=S?q2xdk9Ql0W1)!}Q!)(-fI0a!AS80X@(7d08+xt{TIDKCT5n)9=2HXq3 z_<-|(Jt6k7-QjUS=YV*B80!UEP<=V3p?HP~CrTV+OJ|#CL_$PZ21Gy<;25qg4XFfM z{Q_^7LV^Q}3?oPe{U;3==8*&zk5p&~meU>#5^yhnA@Zppoei`$2MbcS#up!RnHst! z#@RKZbT?o28`SBJ0YW`36~d5wVDbo9GZPB5h7Q@6y+Qf4LdDkV-RLA~_hs}7XUq;{ zS#%D|+V9UF7MS317~stuRIrvxZ-T98-bhzd$i#)!?@5J1z!rT(Ia(NNP@Jfa=667| zvLV10X%YOmIb0{fOqjeWiPpj0D2zuvrq@u%vgtsgv#=l$%X~m2w4v=gX3%yx0!SG( zsk&^8UJ6Sz>V^st;r_%V5tp0ir+?ihg8;XQGhs6z1ha)qtK9`9P6mxb&|5ma=%<7- z=h0YV+hf}OEgeGHCja4(GJH$=1q6V=8{X(TB#uHtN?9tK24D}|B4BM$f;)nwSa6&F z1>qu}=Wr%!&R`hNqs)8!a1EZTabhM#-3Q)RizQo&S3% zKF#9aP<#=^^{b_pfUuleO92;#X9%{vIOq0g(RqmpujThbTB{(KER%G^#w{{9M7@K8 z^A=zK_y}Ai)?D}E{-!gt!E{ut-@CN|H!RmO{5q=WYAsPc%GWjH*1jvMp#;M+#eh4x z0XW}Z&Ult2Uq`+lmQlXdj}rEz=r{3i&ebUYQZ-sUs>;CH6~L~Ki<)F6EDnc2MXZGA z4yf<;7N?L_iC%fQQX{bL!KE6tOerLX*94*)Sern(Ou2U3VaT14$Xw6nZGw3x>H4Zx zTSG-QXfBxlBjOUIR2CKpInUz ziF92nufr%*X!RtLf?fP*AyC1~~%;HDa>!##zGUO}xn3=KNlZcgGgK z3(w>AzMJ{p=W$~$AU0u;*X+Ty+0kQ5I#Ip4rQ6pq6x%zQgEV{nPi@$66_^b`oR|Hu zYzDD)%ERjToR+Kcc9{EL{Nt7mfPehI5jOT~ z|06U3IGk|(7l5$QlPS6nY*3Tl&OA&Tm*j*4B?dG={I!p3*hB^Z{;@m2KTcteUPYus zR9?DeEF?i0mFdDs_sC8}Hc`R${C`eP{9lvz?^gf8*!2BnY#dU^w2`?zLs1H&GYINp zLl=q@67sM$5C4tWMD_rrxy64-bJK7%Jce!pdxJ#B-Hvw~_Q3zmJ_h~&vyYWmT(MK+ z&zfrV2z|$B6_kWv*$_+S`OfF}EzOSG8(hWG9YgpX*bDal1>hrZbFGU|f}o7=R6K`{ z7q9_DxzqnD%6otC{Uge;AiDU&#za*S0aft5s139B@*l|_di}Fue~2p$hV%vjRq$9# z{}G#Ll>AS~MnTT5g#5bbx8677@`NES(G!be#oy$BbJsS~DO3zcPXT04J@nh)oyq1t zB;+y8E8l*?WJFH2^1{EU`=j0hT^SFgf2ey;cg9kAG!Ew^t8P@K=q%Yv0Co@JYdt8I zRn-asI{vOe-&?_}&Z__!u2^@BGBL5gkRPPLu91%daF|=of$OyZ4s*ystalP-`W2f5 zHD%7uzqI%mvV;Oai-+q1w0KAeW^epoEuOP7oG4*sCI7v#{wUucapRC4lc3g{4TIMh zLhg)HOo{^=TD?Q0O2AcK{-S$om6?JZ0Fi86WaAddJ>4TctcR9udU^gp-?jZ%J$bT6 z#N?TnFho>?C7gJ4#u*yu>o(#C_$7>YLtRqoyXWJ^&lBN2JoJG+!`ptM-b7 zpubwET%khO5eOrs%L`K!{Qz;9fNeGO-OMox7bsPXE@n66f|@^!Le@gTdt*&KC3d-2 zo22GP+z(5MA9k?!vMdXf4tg+stJeXv z&AjeG=&kaO6uUZ|wSWW1y+(3`4Lv2hzdrW;6}VnlYtGIFT3HgG_hO^bQvU~B9%tjG-Lqsa- zShJJ&iU2k-N(Dnny@Xe2@b!`N0Mxory~zHSqc-E0Cf*e9HkUxOUpq_vnVH+WgyCzv2uw7ZZcu2$nnnPaqi5U&-{{5SR!d!IN3w3m}hepK`f~` z0Nh^M-gRLv=PtF4>3za#;UkC)m#LW$s$&4K;=e*T`jo@(>a2o{5_0UnF=Pnwq~n2u zYE|FXT?E~*B!=6t3_otz(XC`KV5cxyZVPY2TB?q=zY?RN<-3p(Zc#-)cT-G9#G+{-sb&f$M5N_Jy7vU3esKtrqv@a`AH2K>^k^rE(TW)Y z_PLkUV$@>bdN9f`!LUzgnFDLnyaDylMIRJk6u`rDs2ey@uyIy_P?&(X-VG4veUfcU z>6jtgq&wVGJ$u%N5gq{bqp_fhWp-|z#ipFepBzeU{ooEQ&0fwe+x z&wu(`o$gWADII)hi|zYl0iZR5a*;U`;C-icmPF9YSnBdq~hfm#LqaI=Vm@7`vI z4)7(tX@;VFAO--h#$+b zPXxRC(MSgbm=lH12=~RiL>`C~Fo}H!OkxPqIilS4fvskdI;npZve5XCG1b7NC)O&% z|Mr8c|4n62{`G^I{`$dqbp;2J`l1PWJF{Bjv=^9UH(r?1kdB&n;smG) zf>`4A)!-|q13X3LzH4wNQp0u<>V|gKk~788q_H+IS!NrB&!#fp=T0%Ik;&`Z3(PGh zgcs@q8-3uT*rx_Dm7|ReB8laoRQ=3%_O9y1Q&>ef9xA%~Csam-o zI{lM`ww%j}O?f0)P!WU8F!Gm3+c6CqvZl?D;~owy!E8?;AI;-(c|q!s{p+~_nu^w+ zp2Avz0Z9jrSDHv&3@d~~d&PveDr^sChky=P+{#O705*Oz--oNVPAZ9OlBeqd6M z6KE?|#lvfAUvvJADVU~_|KkbYaQHTR!x1RJnlw`|;81TRE|QH57;Sue42@wqz3-qH zz~|@gZ=FALI^QegrLC`KF`idr_-hqM8#5)pomHyW=t57PLnFh-vQW3>S#3zw!a)C! ztoqS#p!E;?&7rN`hRXwg!29!$!L+yP?f;zQjcFgD==!fVs557tnzhYBm(n+wrU~ZF z1=Rlpzlp8QyH?+FYM}rXr(&jP-aK0aTf?>#O0RvIm1Nhljxq}Cm!tBq~T z89U_=Pla%8=3p%(u598W3=%))1oA73d|N-DtA7#b!2@QF21o^Db}u6%$RZ~niyZ#AkR^~fs; z?UntkXZoH@&0nC61V~pIVX+%`b%dj@>mB9`9P81|#pSd0?|Sl?0K0g&T(^DB*e5l= z51>IwH~+f=DmE+n_!;%_iI>>E@tO~yL4PPvLuU0CBoq8Jy`6R4-KQ>QpiCq&_C{sx zhNf-ItwL$y`-2-?UAa_$`~X~k7#a<*hG~BOuO(0dk=T1miZ*?#e?IsU-nUj9W$m2) zQ-!8<_NCWJ=rzE)6@NhT##6~e@OBh zmLi20m)&{WT)z|C{~;5`VD@8qrB~#K_cW`uBSlDFYyU^m%Eo&8W2VQlKyIj_dhg4? zTv`;6@0u3}Z?MJ_yDw648t?7MJ&cAL(|uws*NNWQ5Yy@)@fvKV)Z)6XW^jI)bEdSy zv$^qYDM<_1q@IZh1USsDF+BzG>mLsGiDy1`+p+v`h}mPVA-sDiP8!9>N9`%1IRxL_+9_tr%&x7EaxJ zBzcE}UN|KN+us+eVlSJaVQS(bYGOnFMETO-+QAy)In8L`?q-U%Twr$<6D-suG%^7) z3YX{Y7*2xaviq)o1)1L>zi7&p$r4UQV%QrNg>eBYWF=f9>4hs@Z(rT-x{cCy&UXDj z2}{?B;VCxwR?1siTvI63A%4>@rMFWh`q%R&LSWYc^MmU%K&Me4r^s9SdH!^Epfvm6 zFf*Vv>jNzox5QVG$O@9>nRnFZy3G+|k@|JQZ;O$53QW;Wf?!`$ zaaeuiXJ=!{O?Z>y2_n7b3BP{^Q_e(PcJa<4I8*lM5dt&R@g$@}%6GAh8u28xLE8RO z>N4fb66&ydLN`Zut{3H?x)%Um2!ZfQg)jbNs*Bps5u;U;oo9Gncb0jSS5BG3Ky}DO zZO{m3$iVa_wRl_$&po}GG^*r$*L>9KBQIG4L3o}`%YD)$JUj>f2(j4J7e^d8L+lG% z>_hbmw-WBAvghv1onxdv8sxt9dOopWq@TdDUt-e`X`7^1!Y?)*tXJZGa=Y*!3ek@M zSOCOF8&5!&bniPz5ExWC9|HypaL}Z{naF4sM+Xq{VfHFGvHa1*Qrf#PR*4Ri&Cn(e zdZ>ZbYC5Y4$7YoqxokXAMelXG40g(NkW>;?qM}q&)Wv-1ZB*9Y_~J}6u9s0<6*dJU zh3*)<@!a%_0v-#!fm~;T@@P@+Va7alne-giO<_d2xg`XUk zlh}eRa$o=tw77dB>HAVkCJ+DE{(iJ`c>*m7$v9~ueW~2sQCm*K{bvRgFZm(;n zqPSO{ua;-p*xfFbTD%E!LZ{!50VP1rEk?;hkjtyFs^m?J-|ENr*_%l!m&mF2_9|$q z2{TDHoc8n&WL3BibjcU2rm*T@3qA}e(3y<}lrUu052@Z%^MJIP<6#0qT>REO3cOYL zUG*{1Gqj+{`*_f6L0n{s8X~7BpJb97RTja`YDp@LRr+JfItN_{Du5YnHkH{dwmGeN zK;l3NQ{S-taDm#afgO;FfxaWeWpO1)7lH$g11CwN^4p$+DA0d(J`d2fvvqdhZMxg8 z+qa87lt8b9DrO;9fGnoRo!nq-nnY#uC$BS(!_=Mf!zBvVtFjxWeZRVwgF}6RT#u-m zvw`MomM2KOKtcCxr)KokwPEQXlY$hblOvP{*CNj`pAh}2m*NP|RadpjWU#C9Sl5ps zC~if89sh9YVwKKFLRD|2KM?d167Eze>hTEniW@7lLcv7w9QQ^lRhB5z{74TSh*49Z zGG}J_$p4J=t7ohWaVCq$r5X$gM@-m!FDPWT55QWdgs&85oetd{4i{G9(DKLgCl)%m(_(IzSTm zaU1j7(+JOq0mYuzsg=i!4m^}SKSMMIOT^ttRbNDM{!jfOz;mEW8~dY!s=;Y6Mc*$C zzHkOe3_&=+jn-)s6OK5!^GbFia>q(gG8#Ju^v0dw>yH8sWhJWo2bP?OJL!Q38__N} z60}Pj`uH_q$J;C0)oTYdkdLp=iNuK8YiFJl8v6zu0ZJKjAS!5oeCt;6OrX!z7|`o* zjIYlHc&LN2e?61~ZP#WMXWmuNw&9(Dpl8NX82P?Aew32a+Ka6?Hf>tASNnf8t+vP> zS^uq9S{?)l`P@=K)vBeDm^3}0)<9^Wuq8=zjX`Sng+3)II+eEbU{27YN<)ZpE((V- zznip8mE^i%8$TWuB_(8&nx_jOP3M zp?XtUxpVJbAmj6eVQHbdO z%rZikd0wey(`)f{X;i47uNFn5qkZl)DkZxj)nq@ie3w`aqX=X7ScW1^DT?o%m$uC4 zn+vqiq31rURZ2}HbDHH$b;Gsn?U`#wLx@;NTe2wl8%rK@i(GH`cPx32B(xH;Xz^Wo z_Q{UYeKCGMe;+h6)MDi^B11!RXd)?C(pK)*5-IZ8ybfWw8LRaE0l`u`Cm=B^o5FYp zYV>WoDII?*Exq4BgGqfVy(wXUlf!DU-a#NiXMOc@JqW$4;B&hF!(rsSLV3l=Q>*dO zZRfFB^r2<#4-WQfX~Qo>NqTuR(pz!3eUj?8Il%q=y-sz3cv>jIT&hd59mB3}c^DEn z>fE?)%znPX;91UtPp?&LinB4hY*ANl3CJ6;07p(cctEhdE3xsRj=h-#s)q*bn_aK>aSnp}g<0n8<1A1;_0OaStN1k-avi~viRN|GZse3ve z|3@gM($5Bv%^p*~ibF=wanETxWk7Lf#l+)@ynm6vcnSD%JhBolvWnF-<4eaUA22f+w@a5Jg7tk|Ew`7s%M2cw6`~tCmpS~aVBpgJb?$z_kWh{aRB;){~ zSN?5a?@A9>D@F5w8fIpesnJ!w+uIV73D~!M&mc|%b#RTIOhX}L%uN7lLC8xWZ^CDv za66mIDZ9ej32<21%})NSZ_B#uSI^t-P@kSh=*S1=$Yx-m&lSU_&(-zwHA9}3G~y92 zKW5xVafg%E$yC)CNhvBm@b_~#jvyi`$F_ZI1-noMrXHt}qvH$sLy*LqQu9jWjZHp( zd<2j=xn9+)F*5Q;=&%T0`*=Sk2_AQl1^nWO*7PlHR(BIrAj-{1fGYoi7SV=Y2?D%J zSkcU_GdEk+Cw)PM^Xe)S>TjU?0R7gI`VSNvbjdUxFys{(mB2%koxtu`A0G1>rG+= z{;Os;{?6d*CDE5S01Istg5*38!A)ebsF6G_3c0uFf@?;2%l`>X!IT673|=4t`mh26 z%U~E>A8Jis6fabFqIjln|8{>&6e>3N{3^CC?Rg+ZOvlKfiSD$Wzt6Cp!OUSjLy-Kr zTOaD811a%iTa3Q)LO##=W&efW9I`>{!isQa4pr^OT3XKaLF?9AYi%$L8WV+6{8u4= z<2__#0y2IJc7i2`;f^IKEzh&#FUkW_3mu>7#2&JOnE~87n!&Ye7(+;ey1*ePwYL1} zFx((b6Y?}zZqu1;)i55`o1r{S`qy&iptZ8!aboGwW1Lz^TBycvauH6eVZ3{RfubdL zNX&423`f?K;|~1xLfE*4S%`|9YJc8bR(@yxG>8uiV)5tvj^@)si5$fpgI4b0gN_kB zij|~b982seV?ox9byXi6AXu=S%v7?F_%5q?C|zk2UGKhhLe=ppvaZ%KG~-TYa@6)m z$E#}afVWbA>UE2U_q=q;_eG-Nvu1Z}_G?dZIt^&J^m!T^@RvG|m#xtc&(yakvMbQp1d>ZW z$a7liyF^q6b%TZ>LjjM9i7&U?Y#PLgcRYWt+pj#&8QHO~!ayIAFZEk6;+XodVi*Yz zpKw68NWwhhmeKJ^Ed5Oj-65a;#zn=?7jUQCVp!)%GIAaU`_KGa^|cck0$1LdjeFv2RuB9G5eU z;VVYUmorQo+5QitpZDt5?g3oaNRCXY3&3$i=xV{xNT^^+3cw=x-nZ&6KVPGW7)|Xs zMcJ{KP6N_`RSt7SLJ^*HD5*Mal_y-0XDtW#+x@GL2)h|Z%6`XRcTFug=TJ6W<*wNP z^)&zJ;R7aw>6ryq4q6-UL3u|vO;ohyG*n-K;U_UX%i23lm9TypK{pfu_lOd;WEg#H zf&8(oppm#_3k?>akwTfooyJ#1E0-39y^EV{>962`zwqVoDU=NFNd#+pyGKb)+WW~a zLKJpT$6UIlXex7?=C^(aZa?B_K26i|>N?3izmo?rLwp zDEN}uov`x|xFtxMM$kuFuM$(qe4o)ZiIFn%Uy6esX{57{yp+b2%_b3Z zfKbf{J?oR^s}mD!qFGKqbLMVy_t@1BO&>@E9rt~o#02QR-nuXua9(;MN*fbJWC-&U z_6~44jqYLA!@67u)nfpg;%p5FCnmhgXl{&=>`a$N1i_Ka-Xao8;M&gveJ(|nBZmU- z2UT1F7AS%$j*p_jUyPq7F9e%V5SVB{j(a`zkG_tdwqo*3UzNzq8>{h?Sa`MW6{Y@b z!2Dd`_yeL*D53D5!|N3;u;)pCHn0427KKf0?d=rk{g<*X2Qwp|4v5&b0wdr2B71-J zPka+T9;8q&dTuT|Qw#{>1a2}Pt3&04Drgm*=exd#;WjNB{N-NW{&A660eyL?607 zY_f)ZUP**D3g`V#q;${eOq$fJ{mP5;(dib0??Y5UILk0=%en9$g>DD2f^^-d zNB(JbyAl}+0c;iuw)mmntoh5cI~gn5hG401LX?AV{KzRp=iS)F`@WKsF#x41*2E8@ zKPkiOOTBjxKgxj{6!UP`@Y>&ba3yFcO34O)imfO3L9XOEtBS9~d_PHdRkH-!{ z+MLZyV=XFonarnyDhZ5EB2u%2ZxSkS)n=mh(37OV*Mg^!ex!!2P?YnE2BQvpf#Az7 zXO9h!Nj%bM;Q{S-hzz0-F0>~B$HU2u3SBC5sYHpxXvr!vzOTwn>h7ibt97Ictn>Ri znCQ%Iq|@C5p-v+0fSp~Phka|%?Nm7rp($MLe0dT%yHm}Lc-%5Oi0Xv8WTp8DH`VZk z&~TIcv3K}BEtL~|A#jo6lUolWg1Li>%BaaTjb!D&vhF7Es0*dH$2_nkO>8+{HcHao ziEf5cJsLEjWL-k!DsQ^Mj%6kJUUjX9(Jr4Z`ld#UT4}!7H*2^~(??ovPYcGYgo4V7_|&Sd zz7pL_<=M|WB%@KsT5y_kFu?1Lc{_QI4csIDG-8|Xf7Bb@2+M{OH(ZUe&O1@_L_yl~6sM5Ctr9;%0m8Gh0mDpAYAu}yS7j2t^gW(Rrlgj!ZJ#=-PwJL>vX2R~haw-OUM444 zV_J$)zdATwuL#Oicz(i`bdN@XbnFRpvg`*4D9hP^%n&odtQU<6f18^1$y%F}RA<<^ z<_8vrX$A5NzeB8;a!aD#HM}zsxg;6uPhFt-K`18aG;GN-E^$aoGdDkzXhelIps`}R zcQh_eDDV=vvU)vm(=}GYk?OYCsial9>^}YrRsrRdoZ5b9W)R94 z+Sjz2pr14SV~phT2|eBfUKtGeQU<@MzPWJAN9_CGFt6~Z18@cQgr*=f=)q79x`{Pk zVE-gcv zvV}u=QA^3S7}8VdR%;qVZDp76m0qI-rI9~PjQ-tuUkY2uMoI9v+;YFT-`LyNj`~Zw zftKTtb(RKPT>L@a)Z;F+W-XXkUA@_a1B%$w}QVt*Y%SSd0yWoG5aEmR-=ry2#s0Wv%{kU)c z`krJ6wN_4CzwlIUh#Q|}C^051;>#+mv`Y@*A0s|yXdwup^p(k_c51twN3(xD|3{KQ zyMT(|T?m@nx%BjlqRVC(JWWt~o5pFa!5`KZ0{6>b&U4dV6u}!f!alknuuYKE=JoCJ zsmpipFo&$vi1!NZb9VWLU`6^q=AGomdfSlM4hcIF`dFu=TXW$7qCz4s_qC`b;*)MB zdgTe1ODL~2#Wne2m!>md!*B&Pe3`>~FR;SGK8JRP`!!GzslzVnvB$$4_nm9FaGN1E zO1V`>h@I}VI=W4%5g4c{7V~N=K+p4HZ|hU9)MSP>OW2T6Xht0F{nz70(XDY(f(+pe z_*?ilhG;I-F4*BOf|nr(^%=$8yse9XKpTSlb9XYtc0RkAY%RD1Do%$;Dfm+SeO#d5 zDX%(qo-vx>YcAhIk6(H&SelMI)i%9OJ+&FZHi5!YVD9wG+Ao&dg=TKw^8>3=G-(1` zhB+E(o}Q9%m4?qIE+;NO1tudk+XQfNz3FMHi4Up8LMsP!8m$TUT229<3Ew8@lQRux?4ZU2b+j`NM-CSVB z+VC-V4`FmQdAaa>?xpBOANcxiwEx+m!;7^&c|Lu_f#2LL?I@R4osACF9QEx3RFV)} z%Isscn`#hV3%sRnNtBB|`y9P=6Q9c+z0j}tu7Qe#Y!}Z~DHwIbVkCvCrLbR9RepIc z_yls@Eq2ujygat{b8WjH`5v^Wyz5=%g@0*{HC%tn{|RFJb!VY#H;s=GY@KbgzPkSo z6#0#=T%$BW3iE{cDm#3|RJnFxcgHr$VK+`Sl^JKh-4474=REC|!7V_jV35 zUkE+g(!PkH5s4=Xw>9F(56|lY3wFH7AxA^i(nOlkue|RMEWH3V(z7f-;`?uH*iie< zaXYK@X2i%gmmtt~XSq=mHz}DF^@Lr1+cTFhOlrgHRR~G{!AF6m7!W>uOdNO173hsj2L#c)8sV6DlPMhz z96k*}Sa)CcfnCppax){!$|g{Bl3vD&*;C6_o#5}&OV93-8T%T;{KV$M3ox^zd? z^*KV^e)w)1D$(Wh`q}f@C)A?(rGA5Ec>q4Fz0;oAjbh3~Q}rG158Stl9rl!dwjVlm zp<|qRf%ZGPkB61JX=6xIogvdaEi9OVK`%^J@a!CBpJBj*sNS^`_{;+ILPUBP zDaLa2(omsBm7Y}6?YNm90u1OCc<07&C&+-L4Y9z)f|NwZ12SLy+B*)4J$?|e@raiJ z@g(`<{S4_o$_oyM5&2nsRb88hgmC&ixk{(Q8K&W8dv%EsC2K&keMygfub{zcA*$w& z|EK5lLP9*ELVdPq718!^Rb8ZE660r|Qto#Pch}NFt{+sJ{6aKF3A7Q?rQBrW`MFa4 zLKiC(Su6c6j5oGxx1K!rrxwz=Nh}Y_4Qp0q71-HjJS2r-klr!p2?es4a$(N8^%hPk z%qlKvI?WaIxUCk)&!FwXQ_z&^9JBvd#w zv%6{D=!>ymu)NzG$^l;k93uhdXn72f60LwtnP11@lsD&XZFL8IC4ZfveHNG ztO$w%HqGDZ4v4Ag%#*w|-bytY(e{fxIq4g6&X*5r-6FA_hU|WVuNGgMM$yOsJ+3uX3QG2XC78Y;7JQ=nR=xA!I-I`%;Hd} zf?O*V(Pc=rZDA0)G(jhAS@=M@QqmNYx2xnif=yWE;G|MZEV_uGvw0~J?f04U+Rdq~ z(J)mk*UKj?cZ~~SxigwlyGF)l+{QEbC03+jaXggV_MSXq5^(8!>4wnzuN4@j4%Z-< zf!2G44BfMlfoz72+A->>u@j;orE-v_U7>|{ZOEg`UTPn+1nK%@M#4Bl)2){dbtKFO z1}IcI*wtH_AS66gB|*2sR(=F9bBf!dm%e=AEv-sny*ub}!7h)dBgCxBj?( z+mxVA+Jy!>owi_6Sk6I}QMC}eDf@_kdbeQh2aEiP{SzBwan_)(h8j2)!cWSO`xl4& zLh`L;G0&e2LDLpNB;nLo9LKdQe$!E(JrQM9Ypteo#`~Jd)pcnxM+-NXgtz?_2<%55&uXuI+!OZES41JspneUEch@hoMhITfk>+Wnfle<$&D#j4I4{j z-cu3;3o-C~>vn0vuimNHRhwMT2Sv=G67?tjiW9TMmha=SXwcu^36C!$z@P=9Vz{|K zccWmCEk9AXqxut{fk^|BeI zQVi146RV65=*yARa!lvy2!^|bF_Va;0|^goAwNNm(0&!T{1)ZRvxZMZYiA)ZbC1ZM z`pF#{0y{M*TqZp|VmO+5ql7CyhZ2sD_%;#>p@C5i8_{WvU_r1%&O zYA1`hDctb5IX#5ewwYplk!v%NFO!JbG$Aj=^lT?odo)<7eIcgVFO3l6DhjfP1W}NT zP~LKeplI!rJ6QczhwXy~+}0h8FQojIhI~HWOgEKHuObo0W4+V@)&)kw4{f0o~b)!-*S0pkkIIsFv5CttfUO z)(ZXZH?anIt;NjeDiSQj>iy;rO>*VkU67(LZ`(5#J^v6mHGWps?A+}~o_ZLQRbehw z{v_{6(p~<3BMnd+t4nTOWUF6wYN)}xJu+=#9%jPfI6=~CG1P%a-*SEFk|~fLZJ4WQ zdhj_^LHs4zm)42?8FBB{g0raN3CAniZrc^%=c73G%_EQaW(y7wCl}l0q@udk@jT?I zaF!OM;ezUUBKLPSFb3pAZ5-GpI-`x%w3SdPf*RW75lY@mJmEP(jBc-|6PxXs>u~TG zM3AlFU;&(+Z!;Dx82pE9xdp0xKgbBD!q`Mtcj<`>N(|tI{bXB;s1tr`Hx0%mO7L!g z@9UTw^*}9kcGU+r|K^{$;lVAzXWdFlQvXRf0W}}#q*R)Ph(2Tvv(LN&e)6k+1)G4{ zFcqW?&cz~KA@0)`dVlu`-Ci0*8;6amQlQN1RP1|+6m5Ww9EmhZwbVDkhOkm;+5mT) z;es-ek^JQzZK6ZL=>g;SM3Jfqh1{tRwWTfvC^uwQ9hoWQ#!}4D0#aU`R1Zy*I$VV| zhnsxVA8hi)rB0t5mra4F<5ApYN+_-FFFlAI40BICk9wWvw>?$6FKSd@mbF)7Av32M za6%+SEbeKWdpT_f7}_*NN7G!+3dF@(Aducn`EXzPj|n`fTa}w{mAb#%y4Ufa@FnS6 zw{b*GY{oNwNJ1+cXJ?5oues=oAOg&DaPiK&o)01z80XLM!rL82W5N#VHp+hUR+gse&pcnvW3>{unaY?5Nyg=YmUpbRSmA*eL3M*C4~<{&R<{J*DEXG zYZEgR@~=tQ;PFz^H1bjT2MKy5()Ohd>b{rdq3e^!=M?SHy)Q3<68}k8b#kA#@HI+y z-ozIA7OT4-0f&S~CIGlsua5@sHTM?4fxj&a>NKt}ajZP34K`3d^fz3?JMb}i2WsB; z8>!G!jK=&qhC`W^&(uw{96eVzyV{d-$g%Sedlsk|T1FVWY|x~yxnE!c1vRyOM_bCO z>to5+_7dLBsGmr4xIK7lSRTVPyV@FLNR!vSVZa6VzwiY&3v1J`BP6FpxCIUvPuRwZ zgw{`raLeNK!__7kIpKwVI*{vaaD;wDPv=*bxZ~H~-3o&D%F)MX{N|Tc=`%JA)kMY^ z;y2BN9jKo36w#Ih3^62Hz~~RA-Kkm|v3|=pt+-A$10BWRa4eAvtE!Dyx%Lzh@Hp@j z!n%BwDoaIu1VxU5J9XR80c4j1n!b|@ctk=1QbHaG%f5=Ts$MoE?*rDp7Wbbp1T`$sf3!&4hvKd+a#4&m0n_fEj=VBk5*kAy8a?4p>BaW1{d1YhMp`T2K@+ zI=$a~tC;Kjfr>8fFS+iR7u*ec1)Vp!RB3jtI67n-ga;;`W}hF#fu%@gS@iD8*v9LQ z@+ka77~L$UvrM5O^F%RfzrF;<-1h`7@|^^~b5ynGkz%IZ>(+~)alo%&(Zj8nyC>1* zE=TH;&f)ITbo}eO{tj^YVD51Wi?$Z6Es_lwnoRvF5Rps|CXwgv$TR7; z6sr=o2_$`g>6P^zu!$HoP zxHYG|wa6RZcDBEn%u%SX=}n{9atbdPoP6{Lm^B9N(S=d9UIg#w(v3Y`vLR2)10nv# zWhe7?XW)v^BbP(jI5p}xcQO6dFq3t~395n$OQ$!RWF)pq)Lp?dg4F8nU6_UjhkNCp zmUD1U_FasDoS6LOP$4gGS_~ONNRma8HhwCLC`!<+?vJDOmamOX|5A^+!A_usH^B7^ zvvIct1Ob&TNm0lr2|`Q35iOMbdU>_{tOop`P0FnX~7F+PK=v~4A~aSbNC{@QEpn)$aO)-EgcqCR-}Y3 zZX>K%0QVBML1g&O zS4RomDZD5uiQwm)LlN9FXETQ? zRVu95aukYu4?a#x$ZMj$zZ_|jd%SEK6--;E_N)V1PD%TarRdP&=T9OoC39DL()lzK zCaTH{;K|DH<~L*oVD{quq`iGuYWfcN!kFJETDNpjXh8aL%?f#KY5O!>o~93*kVsz8 zU~*sY@<%Ve_;MIi5~h+jtwKMvNC>{YJf|Iy+UYO9pta}q$cAHzM7=1e&4K@2{pRKi&ccR z=T|UR4Y2NYCXP2L(yE~uT94hXMYcgC+Q?{qfAGZ!`&w3$(t_k6VrO{uy?mJ-DEuSD z{n_(Y@OMj8O86bcFLZU{uS>iC z;cUpRr~geeuFE8<9!1NZg}SErH^%)pB_rTga&SN4TYrU2 zDKDvCd3lZfbKAtt5q{#f-12Z;@92;UIBwk}k4pAuB;);+OT}yw`^@q43E{1uLqCyP9hmA!g*OSeypCpd#8^+j};7ECczHZrMxAvPLGfej3JV{Uy{)%#qxzm8KDjx%R{d05^*Q26FH?7`lC zLx~zze%Y0LOU#K%qk1k;ay22x%KaH~`s`qbIe%vId;4Ca&3aEIxM^VvT_n^Kv`Qvz z1MsKdqWyzn_NCU21nnFRk8GVSY3n zurMK{SaXQL^zR>QXRa@{9UWsp)%|?IRXjx0C-*saRC#n!VNk8rxRHk{8z5>!;x#0? zS^+x6i)nt6@7LERmqb#R9W1LgEtw;t0)E+eLw@egj+XB_!fTKpgZ|xtRoh|m_n=BC z-c;;R(zbmYh*RK4VBV0(o8f`8K_C)}C1{9)#i;W-W8N_<=!SvN`=C1HuCElqPim73 zxi@aNl4Fs2;vN{?O9mhZB{l{RbZHa8W)BZ=uzPfqZism04uF3XFMl`4PNQR2J>oL@ zBV|rVT^T4&d{nAI)Ox=0l^#fPH(p?WVbG!z;a?~Cd2lOfEL-z4A@kSTe9nAQNXXi& zwO$2NLx{yqzQL`%MwD?n5z|f`vp9+8)(KCH@gX!4JUq~XQp}CRT-FeUNXz`f@s+2-*1Kg(oo*$xPda z@HYBjOjc)$ZJr+i3BL?ms5u^u zn9qH)IW%#I=AqKX0kLeN1}37agWxDC1bSQ#9l2jcme&4+?k`F*ex)Lhb%h{{BS&U6 zM%W3CQry3$Fyk@NgAzVFP(T8^e%tv3&{){ z;J)i0Yo&fi8^ww~b}g;qFdy7r5fcr{B{j-v_5xXD{oT%VbV)7`{uI+ZWAdx3tL8zx zmOn-(#~zm27WfO|l4at&y*rOs`Hm%86V{P%n~Z$VBfVo^TnR>J68#9n0x0;GR&C@! znf$rG{)%*#3Dx$FaZH;SmmTWEEsHqccz#2z5h&b*2hdVHGKQD#m)0Pm2(^r)g z@bq@lepgyezyTv|Zek-z8=FT_tQXm;nWdN6)pL@DZGRH@K^wWIEo>TDm2-^BeJ!4% z9JOPmPTIIWDWz&uM^OE*s2uC|#REirb6mV9pVBO5+A! zW)C{@1C55fTMWh{Y!UY6psbD^T)>LZiS%U`<(QV%e?2hooAWdzDC#LoHzaBC!egN! zonWX#0EEW3OB6(hhCfPCHnMTfR9rZ)+q*ZSV6?wLimMgeOYm5{XHH=1sIY|S6RggD zw!`7dAQ|7aJ=)!_<8ylXYjkJ|M4 z-B#D-O{CB1`_qm=&vMq6?vS$GClAx`Q|`N@PYmKKdw4GUU8@^oc`peMP)^#}Z^>(T zKIj~*0}B0klAR3L@ob3LiR$7Cf}fknX3hvBRT~=mEKSPQba7qO43l+e%=h~( z{UaK@%OUR-P(|W3Z={qQ9R-|=uIRBGm6(@L_+YlWpW((U24hXeozqILeYk=J9iZA^ z;R-SO{oY(pZ0u9? z@9WQKlmRoTnKpLadN#};G^bNK;^xvzT(&s%IYz*Y3Rm$uz{QevZZn3lL#U=wwok)P zJZ)u>jVPxvh8}nE=x{uM%m=F~zjl)szvoNe&R1bQl8zI-`He(kVXBXibR?zV=lldj zPErwhJvjL(DM!BVuwTYgS=<{hwvNy)ET5wyV`CYQ3eV4kv)|+o={vaEaCKy4uy7VkLw$eI2%COCt?_3<6S?oIdxC;}?u7rv zqqa&JE+8g|B{KnH6}lLG99W(|@HrgEhr^OLzvkykb&%+MrNd9rkhk`Liv+_KA9zqq zPFwrI&wUHpSC}jP2fJ{LzB_HT(pSNP=Ta#x>dUSs)xbd`u|uehc7dSY<*e=}``p`8 z=@^I8VrpryJF;&BkJ9|&yjBXI-W%w4pIc;f(tvxjcN)&) z>n@*edtp#p+$DpMy8(HMYbDmim|U`LpAWnV)!86q|&*sqFy~cHl@QOFwd)`LOcPo`D(=$a(;(A@ z3bP@L=9Bzi8ODovE|?%?2q<->n)uK1{4HQU%kvw~t1|H0!*+6N+$YaD*u~9?H=FM? zjW&+!st^74uw9AOPLr=TzI*7EZb_c2cct4Fj%!xAa;w~YUJHAh=Qg~i?9!po&ha-Y z|MP$$we6E6$c#@0o#-_Q((vaE;T}-m2bY-Gr>%43IoN_1NVM+uQUF^_f8rPa}^k$gdH0Q3KD;DMhKc<{W~h z=X`1^C8^d52evH5vQsg=Pai^5DGpD7jXZ}=T|`9T)QxI*)ILd0ixY+v{MF6bg{I{j zh+Il?X5|X#tD5F$lgWD_zm0t7K?ux)#y6Y9|DwayllCheE=AJyIL%l6StR~P4lwkv zJ~7@7@>gRAi)nsb(7gDc7YDs${Lex6;KflB|FepxDE?>O7&mc2O=LTq>v!3y2TL99!dMXERX^`LA2Ce6(LA$SfBx@jj*X;24UJAqOrD9X}X!<8U@ zlLb%zY<+lt@%i?jZ$HOd$Z6e8K~$nSrw0#<{Mj!TpKs5;e7-$<_4)PBw{PBlc>DS7 zyq>6(L-*1Z34af8i+Zw0(bZ=exRR+yxstzsP%QcH?greU-@XO2KbwC~9asFdH;J?D zr5uWiz5|6{wzKz%l))8MD462C$J2>4Yd>0gSDCVVztghi8p2)hcw|iZMNU8M${l;y z(VX46JqMk(ddWWSg1^rgWo0?8@gQf_u)J|iZxLj-4Ay7lqaN=mq)z{AgB8yXe}A$mq-w=xrlQnQmQowk*#J@9)q`%juf;OruK+Pd8Gka93?()H zZ^6ZaN1aaV4p0_PyR7#+b>rWHEzY)LpyQ+#>JyOZB9R<)8 zl1yTDTc=~g&Vf{JW zu>Ks?)1S&=O3TOp41Go<_ZH^_njQahc=+PzD3$-`#nEw-|9d4*x%i*t7&oy$P3+GD z#s2&hOvFbX_mkw8Cgx}3Vt#T%e3GMH6y9T4?=@KN`TG5R+Uj#npgrynLw*0dKU*H7 zH`7}v1hgi%4jtd38)V&jl!*~#*O1v#@3GDM8bK|)P%EX3aS>&$q?zGp%^@1N@9nBh zqG^tAHkYJZMwN^sH-{*-`KOk0wWLg^1eWFeQ%ib!%qEv`^@htvuh7fetP|lnF=^I` z>gz;?XrNgqmRu)tNV;WI$yp~dM0u3!#ANmB#H8MJV$!S=bJmG|vrbfBC$dBX%{sB% zI*~`xEu%_yoyZd9F|HH+)vpu%de@15vrf!gCl0!9vrtrCC~~LTSk`b zQjtTI$GBFc&|Jx4k!Z7q)#9LAn60Z>F2d!a*``=vwP>~}YF{jxZHk<=VlB2Q4pzTS zaZv9z#X+-8F>jqXYOIRt>qM4lpjjuDTPN~Jx@Aq()z^tE(Ll3KEVoYNk#x(bl3gdVM0t$s#PRC4DUR#ira0b?ZHj{9KlR7T zK59m-i3)`=utjWH4-=F+!nGm;zvz(W{Gh~gCup}UDzQHHoTJli7kf?~9eb{5y)0zU zxr=8-_H0(qS|h`@iapcBczE_)Rn$!pd(IERsm7jL5`$A4d(I8W*((0bP~+hlbY-Cq zMI2fp?x7lsZcE@peLUJ2pvBP*kAOoT*Z`fs-CU7B8w0e~&F8IR&onU}o;_E!u~%;r zudUzatBpP9?fGpLd!~u;@a(y&{gfj1oWHqJjXjU&?z#D;AJM?C-!0+G>;K%LxmkbXXSV*|>m6qD{~jG3HTQq7 zkDLuD)fap;O@>?Pt>2#!wR&9Jd6pbe7Y`V(}mH??0sKt zLghR>aw|-*4`szdZneT`m6aC_<28&|j$Mv5;~jUpOY+@8*PQ`L@m=p|Hsc*vVZ4KG z!+8zotug2IY{f4JOLJa-G6RriyvghZ&(7hyYpA-MX1o5R;kt(F)|cxJG}rZ(=DJ|v z%m<{n?qD|89aP}D*ds~@GR>8H5>WRGvrs24?br~qPcjm3oeVh0m0 zAhFOM@Tk292({5J*zrAZdJ1~kOJ029>3GxQNNHyTT zs{{IErGe%7QRJOICE`*WeRtZzwu4%G;SYKZv50_j$w!hLrJP6UZ-$}R^CSqNXD!Eh>2fL-M#b756MLNG|Ui$yJ1B!QJdW2x~` zF^=Hh??3XLP%bcD{)6HlGI%CSPn7C_By;5DOlZHb~?Dd~?MZH*zQtQd256z?N%%YEjL)VN! zABsPZ){;Mu9+*G#skR>cS%r4@$DjEWe6;*og=kgyb6W!CHuPE8|0ke8Z5IiKtvScP z=lTEoIp@C*jt?9E-zuJ&{y%e!SL6M=kWGHnEjE6?4fgvP&RV2A}|t0OX)6weHB(2Vepm z%mvU(hHSnAK1(ri-n zMV8>eqKqL+I8#8GLd0>FJ^+;LwR)xA1TSR2Qdh%8hk?oyEr&owCxdn(>qLSdDDZ;- z{6GRR6O`u2ygi1v?}*cuYC0#Eav~+_GA)I9p(1!csuM=#4%6nGoqcSFhZd6||AxYv zEGk5d%3CQ#u4Vr>QH-dL)gnhSBw9$2C@8CuBxRavL`j|~xtKr|qxcbXSB-jQd6I^( zAwtEYHV4wuVLnlZZ?CbK02_}+h)KBZwwi8jF1s)Jos94}aEk2#PF<8I7F;-$(nL!U z8@RqBuqP!8c4aB1hA{lP8nMBH@pF^oH(Pdum81}HF%Nr>?4+pR!oGqGMVf78+&VcJAf4aJ3#XKhq=~Y~z(FIi zqq75qt?Kh|&_#3NPh>}vktchh7i{1`F*$AR$GYBPVVeq-s7x8MEFC?!t+an2(jPUX z&jef5L;8C#N*2`rwX)WRo>#&Gn63XG9v}46_rD#!IPN$4|0eUk-pM_=O~&9fOMmpiL211Eh$wr%hB$Y-~Y+xRfaZhzT-L zk>Y3sv4dVODf)KTxHf?lJDj4_YtP#imyPqwf;uzeL(=~2${&_N34w#2wd zU@A6_Q3!<_9je<2>_Obfa-z-@Xy2U{HXmAtwYR&KvgRCl4z?g)RMMpDueqG5O|XPw zK@kPnN{FBv4_N})fcWkZF+fLv1>{?it`%b%He5=KRP0P~?DCV8u4GCT4_Si{LVbb+ zfFEZyszq7Y>;PUt>_|4L3xQqj^)pb=zEHk14;A0Dp+NSMi2T~?y1(rv=y7^#{Orb< zZl4rX5t}35xj+kw!5IM7Q4Sj2SxgY}002jiUJ@FpJ`Pc6Ae4~>%HFM0CVqEXq4^cb z!dV1pgyAZM;iC5EOc3iX5bLl&bLC@M6VycJ0c=nM_cBIk0qK^sLUUxqO3hHS{l*Th zRp@+CBK{K<=W71`;)}{cCktZTg-(4^Mz5f`l7jlR-nVHquNx&)=$2?SAO(V=9FjA2 zDHq;^dAYz2LsvM6qsaQ{b7YC-GbU}ZuwI=i7XUtS5a*>3(S*1MJd)babK}<>3f+fg zp3*~0D8lvVfd#t`53ZgbHQH&R!xAEw5@}~s$DzKs5WaO;(pQ;RbsZOms3Qw{DOL4c z(|m*p=F*y_40w!=8R8t-O3#G?I0$laUk=F-DW(U98-rsS;8E9uasYFn%9dIba%5Z7 za9Z|j8$QfbS*NTikarnOY{W4`_B~>?jkq_!nYZ?oPn#|pZ%za zQQFJCqAAlX;Kd90>o6$p-$A%vskJ*@YaN$ukJ@vhurL9M1$^Gx4k!VMc*=rM%iJ)p zWVmI<*$yVuXLYXd5jNL}Axy=1#oY<<%VL#S=?kN{LO{8JNb^E7m*f9z;L>CZI6KnK zK2Z^7%(g44=4RD(Y7LK$8YEQk)X?NA=27yd zBu=w~idL^Q%){WG%z%t!=x)QJAI*-Bm@g_C%&LuOC@&hK9BLD!f}_zi=1$+!+0F!& z#qLLLm^Nc6xnWjFij7s-rrQ~J{4YBEM8#|BdJYn0*)vsX9U1dqbhsG9eWgQnjNMS+ zIS}wta$9^7A*$9oJm`13EpQ7az6%Nd1HB0jm&X7E1y(@II(6@_bV#VMQln(Bp`?M0 zZGD0_C?<^rfn8Lza;w8myP%zO79gZdM)#H<0=DG>GNDdP)wifaVkeh%qdjN|e6?&_ zQZh*GUJd1T$b@$~yr8T!(8af04g?_?CO3m~1O2SKJ|Koo&%76&79m{Niu2m9RE@6>7kn`=&->muR&_`L(AG>z^)= zmUkl-!H@8!LXe+Pp;`1Yi=K{?3;aD$D8p>^%bY6*C^y7Ub|3R13}91CKY0U&IDYD!DLzt?A9Br{6MflJyPi7M-8b-dPlFv>)r zQBUNm^7`xx2#E?9`rViEUReriPkJ~7g`3v0DxW}xxDsWSq6iSwQ45yJtgIZ#KC5*n z%kq4h9t^|6sMyt6xxwQ7tbyhECpn>GA-F*Z?u7g9H+992x|~l{Q7NXwEhxvK78EKD zH>`ZxvOHhk3lv=}RJM*8XO?pBti1g#ie~wUT*{_Z#Yno=W=RrP^{%2YiFd+dxiXxJ z6^j&&`eIV0sM{q4i>`i;vPCygnZ2WG^+=dHvDhLx^ZM+|fB96v1r(N4z0Hy}#ac_8 ze!OO#<9A8VRVf}WCgIWLov>yYKokB}2+eRYim8WATNB9a)?U>7>7T6~%kys|EG*C8 zmHcJ+JCm3GR!|3ulNKkoU48_;&XLK>YwX*ssxmCZ&AC-Ki=k&K5BI~B)D{;f(Ggw@4=qtyiT)gugryO&qDy<`1f2wq14RfGx_%F5v&JeQ@ z5pZlCbQ=LIc(#Rz{i2eJed?4H3LsQNG>V7FkP$HfR8iT*F_m2u6buoNm1duQsmA#} zM|NU|av$I^QQp{{_+qZFbf_!rrb0dz5HmXJHNoKB&P(t^_3H&N8PRoTamlCCFT89> zY_d+n;E$BSC6`Wpo);3E!{4ej@cF!RZEMHbrwiN}vI$K6_QpcUuEf(}m4>AsDNCc| z0L|cNsojUARkW~!T|7gTUr*ltaoer~dVTig^Mp|$98AVP<8g@q`%IN}ZscRmkn($bo6PJts)yU-IXM0!JS0S?%xi5}ei=)qv97mpo0 zlk=BLS^9sdqATroNYBVWg?07eHlq`QA)lgM-9C!&lpM#`RPK6)vS6 z0;YsOHy&jIu})CE^OX)iMZ%=)jiZ5IVcpy<9YL6s;6zNf-FFaaa#Gho9h?||3(Y-;7T3Ud(2 z`Sn_}(s!CY;`Pg1!5;S-wyV~#-qG5vVJ4m)LpHX=z`+DTw(}S#$FIke-}b;Ecn=eT?m8~d z(pHS8576^x{?ePwaHvnI<gj(wO4p5FMV#lM3~;ARgfQRsNTIiOUD0Axj=3kdRSeF z&+-ESXUG5bj*j~;Qt^K;4h~;5@qep$O2q%A#<&RtYytuADG*T2w8tF7mtut`m~RV$ z`BG!PcvRnO2O}b0BPQ?&TTr04Hy#e|Wh$3$rx?IG>glh5r8)$xxJT-!s}$PM7ZXH; zekrIiA|M)yz}tnH(tQa~rlv|c{-V>>-mMz-bB(NlD@s#8k!6X%?B3siO^n$SRg4yT*-P$cD~Z#E|_30=$Juw@VgI*zz*8! zcDlX2=ssI@6wpDtU`HwdJ5|vb`qm}N;-QcH?1CS_arf~1t+>Lf;qR64cmJ@{RW~#@ z9R3J6{QKj>5)A%m?fvo`{%SmCXe`a7S7-05IM;G2t|LePZ-Ysr9)VY9@ALPjW?VCp zvvoCr95AF@<~GpIuMT)G)O+Z82r^^?pF}}hrILIK>`G*>5QAf6tIS$zlg@)&fL_Rj%R z;BX6;=L8SFq1PoBR*s#hMq(0*b>6k{&H#g%w>y*V~q z6(j+|1yFfiW1kn$d&aVeC1>2UR`w}kn$J}Sn~mURBe?F3;MdVC z`=}?Gf%r~-WM!sGXk4^FXZO+XC1=^H*;L+|P37$1FR{CvW`pu(a&FaiG#YX{^Pnbl zJi@^Mug0J*3`5Tf%Vbdc2HW4kpHCiI<9b|AE&6}B2;59_13>5L|NZV!I{x$Uu;1wa zt9VN5|M3_%dViz$*RA)T$%!>z*H^{XO23cUrqS!SMz4!@* z9#U6V1NOlBxfXESrjtLk612rnZTi21L{S>}+4}!Mzu!%t|2ls0;-Jz0SMgNR{~aV6 z<-bw>xBK*0zzB^xzg6nIgT&&x{3mLoPgI;EXiv+j^Ja~8)aajO14pTy7c=S;rl>Na z47ijbeZHf21YAsffp~3KGs!CEcou{bP+>sm2dqJb`k^R#&xQzXkFDSR< z6uh+@>f4<$rDF#v4FtQMPKwf&4YiGUwOe@2GcVf{NNfkOeeO}w#)7o&b{Dz>b^V#& zLEUK0ZEeGL_R((Ry=Aa^Ar^_wN7BvI@f4~E;R5Q zP!ivQ>QQ?yKE4s2-q0%Hz(FIiqY)9*yOj9NtGLtK0}-B>jA$W#cX76}_^&vvUvR)= z20gOMNI>Ocl7u(Ap0?!ZweG-oW}-G7Mf>uosyup>mn~M)M5Jbh3F7>n13gd9+S2++ zrs_J1rq2_k^j+1FF!323flx5b)8n;=B^#-pI)sUdan38&?`zgxqK=G`Ba+r(G9AVo zgjKS^^16$BXNC=sQ0%jc|J|=W;}X;)WB&;HTd4+|IcWUG*Qd?P-I`obab3ACTrlN= z{Q-}VHMJZB-d;%{fk&3=_%#S#z7!MS=|vwq2-rdYC8rj0UT}mTP*R4QYt|J!2aQ84 z^Y20~*MH1`fR|D(3rc}Q)gOQ%2Xm2RRmGWM5`>cgje*2tposgy^#Fhz3C ziW}+hz(Ko#L6G^FV*`X8P$EX^{x#wWegwx`=mm6;SUP3WY#8e(Y;y7njVO!wvYhM) z(hn)6yV!*gs!;bOaw_Fh3OksSGiqL5rI0txJVrsdeVF1%2q%;?+mw>5VX=nA9)`uX zCMTGKgwAQQ9E;6>v5qXJyR#82rt=L{WHFxY|1vBVWJlQERjgsLhQ%I$#kOe`%P4v0 zh@fleRLOHW7tA^`T1AeUKn_Og-W2VoB>Qs=4?UQKtWcR++Nu~#H zRyIpXn(}>4GGvsovV?wVWTsUO3^Y>8$&=P+%sbZ!xY793A?kVqX)t<2EnPy7Xi*O@sV^0c3W!~>u8tF%S1;YR=;`WV#K zKDC_x3xkmty8o?v*gweR|2XI!H|PIW@svLQ7maaq0D(h+-#PZ-FO@wFwAJ|ZcMc$Sr26<@ZXbYY?^#7u~ zAK^1c|35xB>ZSDmql3dH|Hn$6vig6c^*37oR_Fa_6!&dW+!t5ZgW3BLa~BG$4ROc6Tcf{F$e(1H^+e;#YF>#)QYwfBR?4W*ojXSfA^xrK=cbfij^opQ6rI`XM8 zd0(Bqk2ZRBkVZB$!B;w*aT8>CSSFdf0Nr?qVMHvcT~*6o5ZrlMXFe0%wGwx+X33H6 zOWuvFMiPUa=%^DzT!b+z;q_a=6;>9t(dnP9qfTeu;UslsbEF~c8Ylnv&3?>lu~H*( zD?fs6r@Ju{q46cmcowk#Rc@QeL8aCCnI3>S_Wz5+7a9BixZC9aU&&Ko|L2VH{d)jp zqi=&Y8@qoq?S6BN&yhu$Er;Wg8UH!6zcK$e)cnsGYtsRsZGeKDRo7Tds4u{d<@tNS z&17j(hs7<=kB+z-H<>2dZSBd*r!8zds1+;kp+?3Oqnt#l+lfAf?zJ$=lHi+3>P&f=c#1t?U4LfUW5-HA%zk?pR$fayL_=wzB(Y5iz zsX%--it?c>@iPfL3H~R4G(wD*g$%&oX8?N*tqHR0f%Or&?-(lG<=Mb495^fs$+Wlf z^0Q^o{49B+4D{KvCjgk@fz%X=x%0~90*Zk*cHvhg5OdpBEI6~-44gd9i<`AEzZ_sX z5ZFa!q=SvFuqEZg&c2PV_ILl+j_X=O=d$B<{(eiYT-5pdEgRaMzu$8C&z1b=M*VBM zZTT;h?{{bSWT0C)Da5cspRN%ePXw~ppmS+d=g7>2wAVnB%Rx*Lg2Xjtmji|@gs%c7 zoKK1GPFoI5_Ga!nmfI4#k-7{f4=dFIjHZj<6tuW;?3Vno>hAt_m$(m07|k)t+iejZ}cYe z+86C6tH^IkoM~J(V(m_U)JjK~{0+d+M*BQPgAA zRsi1jvYX5XZ7>HOLpHWVaOB!2KQ55a``vv!@+E{Pbgfg(q- zPhxMx#nhA9a_bK4XKHi&#|+0yqVjXl>vq8ppm!8u1Bq>57Ib>xV1gjqSpv~tk0-zF zf&St1k0?k@NtI#r%-Gu?>P0_1F(+Z*FXp4e>Ad*G9`-2!if+GYziFoJO|>~)ZA z>m}HVgiPIn?a|7$M|)&w5yAG#aVZzS_~Mt5l2TPgTe^ND-0Bo=bxSd*MvMA_j<_*e z`EpG5KwT@QZXrDSTMk~2TT~UPVD}(44!Y|b64;21Mr_=h*f>}}v2pOY#KuOc8|trx z?Wkn!MrJH8Gq9Ytd_~?N_1DriRDg1=?jiN(V8jqwK*Xp&OG}r-1?{KuC^mkfMxHhD zY*mh-dlhPHBGdXSl4*}tq-~UrX4q!x)UivL7lWQWs-a3kiY*L9V$73Q88$m1Q zl7_Shk(`ByI(&zidF6a8Bf41`0fJ!lyjvN>HI1&KUn=uNBs79x@}Nz`@-j~ZgKKmh zbHI?~A8EI_kW5oZC*?+Q;J<7y$ozpU>zBIGzAEBBlP(pE9{?fO%_*Ot;f8SEy;$Wb~JEg(9vX% zg|XD^p*SFHa-d7-E4p%zjjP<^Tfh#l9co=(T%zkYNFcRMQHlpL)z)$1t4`$lZgSK= z9!2y{i3NsJ;8KPH7e}|tJK!zhK11Ns=QCN@qGW`}zEuA#${4bQGnGxL<9QQW*RGT$ zyTC8ewVfOXgK=~(Wf>3_D({66*hM!3IvR0x(VaSEpU*E!9V>FU+KAz+ooWz+i}UjQ zCyD$XOW^ziXx|IcKNLHCwjyd!O}S50Tug~IVU%Dl6`1mZC0%t3go~ssH~Tz8pxl!f zAyn`js%Jz=7S$qg;$kwO>RM^+B9oJ3^#K=9__Nq3;X5^X(}3i{8?}hf5l*L|cf5Uk zc>e1Bn>T38pj~!dH*2gT*Gc>VWDHJooH{8#&9m~{i3zYnI?8E_fbA<=j1mMW)4G6- z`K7*6)@O)!41LCpxrA z+6fq}ghV0;*&IM25RsUuh&GRMj)%?^KqX=TzlJph>MGHI`Q<1vQs%C%HmUO{P%I)s z&eWa1XU;SjbCXx{ci2I1f*DHL2kc@|KI9@*aUB^_b|ZR28P^Wbk}(xhd-fSaMB3nH zmD!;c^#xJHh&#prn5p;?Bgn~wi=vwe^m*Xu%J!a9N~#4^jO+n)9BM(4zkr8<5c`QL zF1*2RL1IwwBY06{>#lLq{Dw0d&Rl=ae7OW?K3eUy*8GEu=Kl*esLQ1PD{=q(K|23m z|G2sTaV5{J{C~k1ujcB=Q6#goy0;mz>5zN7E{Y+WHS>HNKQ z7sdnjO&zztzQ*T9W!Qe)&w~1YnQPyoXP*B5;y9iE`{2bvqyMkunW6u$>e{#F;`gnx z0;tCGv6%qPHE$bl0F=JnO;GO=45#s(xzQ-@0gu|dV5hfd_%K~~le}VAHkMP#s1i(r z#6{l&kH!~#pX$L?JBe)<35G4++LM6QvC2Kr@~GX~1L~6R=*Ldy`EJX&^(1}m$uq59 zhKR`)PZCX^JS(C}<~DQP&0Tf#v3jq2ow;F-l)gwV*4Up*b6_V;KsBlZ2aSYk%e2%0(1L3gJ805>D%_W?`Aq)?FzT#wQa>W!p*<3Ep-B1#Y($wWmk8M_pb4#@VgfAfrE-itig5cO{$!+0 z$drm$DJm4bSt_t6Z^@0{g|8ee^+aQwj17z;Dmb&KkUE7OnuBqHy{LQP<>E0`gAlEO zY-13&G0@Sorh5GCTXK20yKBXu_2*c=-`%zDe(iO&9uDe4ss?QIQS-y?3KYzWi+@95 zO%|6nM)ge+H`lUTntE$;;2UI7CA>tN}LQ*%VX0>97VB5MdS@|odRWC^|k*~6#A8kvJ; zh@KCS3sbg+h+{=ykf^2wP4YBv9`nlf^zk2`r_TQ@#=DtgcN>3fH_LhF;fz{bTS$Re zM-Hx@;3=BWWNfaaL6(>iEh!3&cU7gra*3qMv3MkJN~stoO={OCB2;u*oqCq;{W|cu z2X&&i?PBQ@wb2JkFMV%uxs-;nj)JL>zvAwe6&#;dXqFIaBS(?Gvfzk2mXqr-j}g_L z9JFnQI3MUnYZDO-GipW>=Lp)<#h6gnT}S3Kcr2Vc)pd8w$uqq-HZ9WNXXpnEaq2S* ztwNtCR4`+x3}($Egq%rc@kooGwVJx{W{#4tKrY3yr>CHsIFGo6*7!O4s#39?htp>< zy*>qGYk8Ojl*lHe;D=;JIPYRZKm^yrc*?b;WUrL6)k%g5;#Wo zAZj{H@)0&utC_#X@}N1pu~yGg@n5x^|2XdV57Os94!eg<{MSmJx$$4C4Et)ry*54E zD+u!*u?qt z8FDDJ)!FSZ4Y5M6CY#6ja|mtAq<%IRq0z+zE&-B-S(p9_{*A?i`T`_e9D89m^^E`$ zsPEVy5bA2S0D{I);48%U07uAx7_Oz)*Vi3Jhdvh`Vir<^(ZO_IO)gBvZA#i>=#LR^ z%LP0Z1O0Ya?L^r~KvdNhLl| zWEwb#jKyRr!L$dCyWlxE=*Gw5el(At7CEo-(6OKgEi9(XNy_+m-nIsnsm;L)L~KmP zN<#3@^n$ub{jHsa)wYC70bED<`RxB=?_Imw#<2y_`Fiy$FiOr%+{cn7`IYjS^ct6w zs>5qC~#PzHen}Ce%DJrG03O+HwNyY0b3;t&gTsQ?@MDpK8ErplY0)Ffv21 zV9AS#b2iapnv$oV%x7jMIROxe!-pUZ&NS4D*O|TJc4^Ig@wgW9ErUCZzGoC)57{zFQNYkC$(LypRWd_7hYFSW8fveC_X= zzzBiCB*9CY${G|~m-|xhWm5V8fy-%5=KX2&q=++rqcb+xdJ z&71(HCgpx~y`~rD%=|(zz7(>h6asQDP7(C4bSqadicsjUza8_LW!-eOZn!iS&%muZ zU%b@GuKy&X9pJz!tsAb)E&Y`Ty(uz@;a3LKrWwFZ)td#3#sRvLH^44n>3jKSbFh?Y zZoi}R#G5z@#BmLM#52S=oJC8KwSwy3J6ZE7QDN{4G`$CtMAMtv3~S1;$K*x;J|2$| z6`;$442)t9d^68J;AWm&+UF$ZS7dx8o7!&h^)~ngnh}mJIiYZZelq9WC3bjW1K;%)Pfg#L z=&$Y-E*VSBg4r~j+1>8Z5Y*sIXpE%8VJo1rA()$w`TMvGD<+<1+IHU`|hR!WP)Q?O@K6P zME--#LBoT;mz(u(#{N<tKyT6NSGai=hs{lTdmG zy|nOU7sNC$=_@FnO!*4XEAve=24(A%45Zmn?AY{p(A!dEj@}@KDe^C;koqeeE%I$_ z>9ZkCzM2-m)ZBV7@Zvxjn%}}uSuD~xEDtBBD+)THEYsJg-Mip+ioK~Io2f~qIN8kL zojBxV8Om^hEG414n!&rzCV|K6_P;SJT)>ihb(l{l|3wU(&cPS$_xc66Q*^JaNUtBW z(Ygjls=Msz{2Om4(w-&eX-S68OW~?xPRq0;Up-HTLaqpXVrq)jMj&f^Px{Yii2kTbZx@>%kd8JjW8F_X{ zjLeWeglO1B`5WYp%G7ekvcM45zH7dU2nnuCb657PZ0W*!bU`-13R!R}ZmGh0R6%Va zeH&Xw3{;+FiS@hshX7Rj<*N^XQ2j~v=zMFQfXJRUL_c&4qz1@`HIEN=hWJ*<$EBR$ zABjJ|#Q#F8(2?R6zI%eAkN!33iBSQ%X9jnNRmmD7;?J)b{ujD0YN3D>-H2Y6CL7Km z_!RPVB%)0q8eNGzu#_}p=3q-5nxGEKWoC8S;MU(#hK*AOZRmNgWI^>&n3)WMe&3dofdG|Bqg~)wr%V@^b(?g!y6c2 zKNFPk7Bax%C<-w0?;(Vc9;8llm#%0f4KUE_2I_C_dFVCeUfWwO^8Gh+lkf|kO#fGm z==@g#m$LSADn7+M+}MUxb`PiL?<8)gkp~C8e*^Fa;y}zfFhTDGEx?@VS}2_05Xt-a zgvs|0!3>8nzYn+EDV|Ja&q2Csr8fo?L2rt~iG+6s@9-?1rQ=|T-slu|UQDIy97X?( z5zFpmcrz)!!dHX867a0!>#F$iUb(4&+s{=B@cQNgGV|4|?^O^eiWhhq&Yie+Yla`p zH1`wg?)<<1pW*NUBM?@>Qtmjh|IHX=Y%>UL@Tr^sM@k$Xs+sY}eHQ2c=^geC z2bugo{lnvJ{-2FJna!WBco&6Ipm2619S$x_;i@M_T z2!ZyQ!H%}O#2+afkFj@70_@GJCbzIs?(h;3%AGVo#f_$aUPj34idrAw44E0wV?+O} z9@_`vqjN%e+tdy9i>mRMa(18A87!*NL}XQ0ip4q|rzO+nEk;anN+IlBIH8C+oGhK* z35~papE4JYH73q&NJDkL)GSssu^jbJ%A-y?w5n-khzTL!Ryr4^1E zZaZrn{>mGw%#^7(*COHO(q&(D&Vg>l+-b1%oKVo=dZ&fVeroc};5MB-GE|G5mn%u_ znHkE+?iF%((e$n;eP7l&wk$J&Q_iFFl_$~EUT-3Q$+WWR$rPjbuxLUbTF~AKt@WOD zqc#vTZw%8tne%;|3goj11@I< zR3ZQO5BvKW`G3E^mH#*MtStX)ggDg*p#$hu$o;zXYT|!s=O0o+^WduB4E-!4vkf=u zky%7S6mo}v-=o2iS@!kKfQnkGd!m^HWCVjQp1}z^kAvV6c@(MZ=Q`5uy)0{1<4#~($F@0*QQMyL z%RIoMp#)y3S4doQSR1^om?3+8c3gN+=tl&H<~+aXfLPCpt{Urc$PvAPfvISI*EAH) zR?irO)dqukQJ4zTE^s3RDWV_A zH41;iK{D3N>zeab(pc7%5AvxpvJ%q6Wjr3^JJ8;1*T6+eVvf!!5;eYl-J2m!vB%nK z|HvN~NRkO0PV_daJrlY5Dfoa#0eD$^q4F&y1nX$WP|kuSE0Eqi$a#=ny<5I~9l)e2 zQj<9F>FNq9bH1yqKN627rH+whPRxD(6;2&&DT)H6hrwZ|;Hc6pHMAtYdJWt!mhTvIF7Jt6-CBpMe z#2wSW;Roq77smC;eJ*d!VwC1YA|6W`x7tnK7Bpx;`;&}hr)fv#jasDa_Y#3GcLpg~ z*z|Z}!I-aD4674*j1{%S7B?WEw4?x*wr0O|T6Bt6f@M01i2t#$pUelRt8Ar(y={5< zQ+$@>{}G}>B*h+FLISAr|2-ZYX8Hfo*8g`i&${>j=ZIctgs{|oqSfcpuBXHbJ$vb; znotymgN|uJccSVq4i1w7BCD56AcYFW3YMDc6Ep0%liL?-YZr48(sR5Eo+B~(IOI4m z6I&n)6-hmd#hsPM{rNL^jyEGG)ldgEFDB`I;bLIk+zsRdq8jeGplVr%C2~|TESZQ` z;&Hv5tIOv^nBtzQoI@77rI>azt|?3Rk#|BV9YWEVLX_>Y*!4sTuBGlLnI=k|OiEx$ z+)d1Yl8yL$c`FrXc<7(S!^?RW=>2N@7-TAD-8V|c0&FLYOcPPAddArz3v14YC3~j< z>iaB!yx&3vDJXDTcy5EwIOJ#?2MqBH01@#!Ss*$qcbKKBX{?^2wavumSxxA!DgSjy zgec^M?m=HBGbLf?7LBHaT(iAD$*2=S?oIbJjiDs4BZXG!e-4f^_rLptUVkh9ZRA-f z|NW*HeSsX96iM*n9~v!It0eI#S5>RK=?Hqp)8J)D_;O8qi1PP3D7EfjC#$TIJ;0bx z3B~_X33#tx$(^1Ek(yy#kf6wod4=2Q1k$YeydVMXDGFv?Hr*3rXbUofIyht;2EnfQ zJB}t4`e-*IjHe&O5B(+IO2ufWyO@N#@)wIemBVMPJMZW2 zC?(1*Et|>vZkeF{#K^rrQxZqkJ#x|oMH9?8wbxJIW2jphtB%as0yz#Rnw&Z5TUb(V z7W0ZyH;9g`h9`(8f5(vzIV$m9(04cyi*hF@F*!;BqY+dHP9~&(_Y6jGgagbmV%@*R zBcuy;f5u69(^syg-)<14*th+s8JgW7dPZguqzJVC3i{ptVYk=mwTmE0j~{v&z^4}q zqh9=>cu~MmVJRUrg`y2+N~U5(xOF`T+7-0^de?Rzc#fk`-09| z8}i%NX;~b5gk>!Ml|pc)iW2v|J&6M4qitkBUFfF?Qu>uA^(JK#5a{u6(x z{i$3i6)&lv|Hu8qqwM+bXuto&>HiaX9yk5JA0j`JxSA03|0JV^DY)JP4Hz1)$_O4E zxi?d?2GutzadV7v&OfK;@AUez?k|vmwpnBtL{r$;>d$B3RQv>E1bIvmU^w&;xPHaN zS!d7!fb-xIc_j4N>6jz>6Am$(BL74tsdVFL>q$6SaKX|94v9{0F`urojUwWoE^%Yg z1y$MA8Zc#P2qRuCxF}g=rYI)joOu>N##|+P?o)~7?>c?tCfXw*hd4yE;I<$kE}1KU z6+n^^54vm$U1hy|omp5mXB^G0wD#djt61g=ZC9HR)jLHIVM6wpgRfst)@g~u3HbW; zf3hhY93HLJQf@|Eg~m5APgQq2Ni>Xw(bsBr871wwoj^LVFkE4I4!_0APG?FOe_eyt z7)dolS$9{j+FhP?ciD}X8$p`*ZzWm4ZH!3)TeexJxrJwo6bqK;G`6^}G8|e>8ui^7 z2|2psCr)pcjhs`;^62OfY9Okm)m>zpUWutmJ1Lj3`=>~p6u8+9zlClN^?XR8_0N3j zo;I;ibZ#;O9L9GY@x5biN|k?DXmLoemiS~^wl)HAzrt?iQEwbki8ANsVreX>vU`XBk1#+91- zpM&GP{^w|`|JlT|)&Jaw{zr~tBkG?gqwm*qveo}AheO}4`ez+{_>Q$dhP2p6qOJ00 ztNi&!ls{SYR>}@7YBs6ucoHfZ)vlbA&RD4%H2;3R?1Jd(MeDEzkI}!jil43GXKh^C zDt@+#pY5~Zr&;|^#G(T8(JKGnUe^D2zc)D8+W&0idEEM+1@@oSR`mCyJW}G<-c5gm zI5aAxbK+lWs;I<-M#2_835(ETrv|xF5sNbVhdkd|P`)$^#MCW2 zj{M|}I%*_8eR3P#Lh7HMze_6#jAa~!{O_{IvjF0m_&Ozwe?a_}(CZXr@$%jInfv1W z-JA3Ujl)y+J44zb^BFitbcQ(te2P+y1>i)KRB{s~gr^h40A%161wmIiYs3LM0WU5@ zv!~(wg$!f)s}Rmi2Op=H>HP*EV`A21XF_&k08#{EhQkRsjiOUJBNR}`rvij248i4F z*;aX9aTEnIuo?_(MpB? zGtZX)ub8lU%jh37qc6Ud*d~h3JtnS|-X&*tT&t!c0ae}q9v$?v@qZ5b+xS1MnE@_- zHbDO~Bz94hw)B+segR%KO^sKvxG7*JC4^a&oavH!lsFWpfQ|<{V@hTxZUVHue`6_Z zZQ!I>lRTg@L7Fwtoovcx0Zqq%^KhUvber28!!UbJoy@W!U1=O3NhwjcFyBZEtYwL9 z5XdZ&W2<=r-c3TH%?-1r$eEIj7=>m-@)^~sQg!Ckr~2NpbXwG^n*E=t*8WSoRbb9( z7jai14e_f8XTHg^i?^pA-#V{;KYwFhc>zWt*Lmk$3(mBeb%nFEm0ArHxSc!UhgDuU zi$P?jWKW@{>jqIl(F-UV{4oywU=_*Q!Q!0;MRPSG1KINUos6l>PL$&rA~DxKKg)OE z);Fyt;Jib{N&?5Z{@$W+)9920@Q4LD+8_|13n>Xt@&qZF~w$P zHKoG}`S0NPI1~SMa4lqNWaX0~*(JP`KPVAD*kyNviQX-rx z02=Y}5X0)Iadp(#*S;Jwo(ZDqOWE=X`24li>2zApz|SZY>mX+8>=c|?lN2-LiKN*a=;O}L13bB($DT&PSo(^%gygixx@``N>7J~X$tu)*+b3vB;DYGz#*EY zglgg=tYuy%W;tf)VRm(MSK`g=Y8#$rc6IW#bZfslKB}{`*X7hHXSGgwt9XGIH<~)n zA2`QI>!{kyKFrAPe#nw^KedwmlmvNQv-%1#VmIHTj7n=}uOp8u%gg_TDgG8dRrcQp zz2mI>zrW4@u$gBy{cnLiR?~^}mM0SiMc4NaPFh;sJy`3^GTO3oEt#YG>Hrv|$845= zvp2>e4Di2-J}Q4=^*#j`j|dL4uqyu~E1Y_ghj3Xaq`!5^Xl`Y^u6oTFW>f-bRhyymSr=O;vCq$R zfx`(!jMePnDeLW;de-6pbrS|w*?$lAv--coqob|=*G8Vl!T-zssh+wR@Q|7668KZM zqf)1%<*%ypaqp62&DQhi$vjKbe;-8wna@z_-c$K3rvHQf;lUt#{_7uX<3DWVk$I+* z9SBZdTgD&lO5+ZD>+YDn`&?_b{uYmfka>CI$#BO`)17PevAPS#exz1-#4@|1C}Jt| zd6S@V(@bYw(}9O)S~WC{*|6eeA;s%8Q5xyXD0L}{0_;I{0{ZG=OT|T%b2JmO{I5=A z@^z-oSfWTfC)vt#$Lh(|(Jqbxzh1ob2A16rnLt&Kb#XZOc2jZAmyc(@e0+C%DDBK7lm>EtBqPkh zTN#?YU8iv;2D{xyJ_vg?s%Yk|4-u~QYx;CR$FTX1J}gv3Ezh)@iX~sa?tT88+?bUJ z7|S&bvfvg@cAeP>$mAzNXOI_7KtLv)vHV&w_R?0?f$LXvgMu=!0`ajLs=_u_2Fgd_ zJTV?-MWBfiX}^6_CrAmCqOS6P5z*iqT96y8&i} zYboYOix(d(1Yg+PYau!asO^`}!$s8>tPb>jEEpv$Tvi!&5Wv~!KSGAy z3DR5CpT)B^E$fURLmf%0UpK{w>ch11cD65`>I~d2f1g8N2CkI{pT1Jo6pZ8^ zt*aYiT`!93b{^mvZlG(~OznhhN6;{d&uZf<*@KR4tRF5vSh|NPl(cu`FJDi-hrEZ1m$Z=hVGc?ydDQ4UEU zvxsz{?^DFs>ni#Xur3Zcq9F{<2`yBLiB@^QIx`%0Zq)r;)n*Nt`2Twx>iaym*@#e{ zOC_18(FGV)+W9K5&DG$P^N3Zdl%J@GppZ=n%$ zEN%Z4v8c184^Wl;??Jzxwg2r8w)Vdpc^b3-tpwCattO5sMy$rIq#) zl2>V>=0`{;h#L|2tqrz7C|L-4V{r?At$A??`zlmqF z^#6epso#rB;1X%iD^1#0$%6jfr!HTt(9fyjwi(X18P2zYLiSmj|7$^R85y90{~ru` zS^v-dgTXfb&qkh=-6^w~(zb4t%ezsUy%a3xe3z-VKtiYsKm-1qMNL)sb=HSZE1ce) zE9jD+RYE{v{gq}2-6`9EcjtPBtQSkpklj=RYZnmc{|EyZdg%nY@9NG{zNNJRx|GV= z3(4p6pxIx=PPN=mJnU2HfU<6XT0^WFly8c!=}hqAg??nFOCb@1_L;$|wu=%UDIAZn zS1%-g7Au^}xjWXxu%uqB<(y4LCu?J(+d==VKkN_0N9Tm{#4;|_FX|4}lrOp1ja>5> zE^i`K`xu!ihLI*e{$1mYoRKEq+}e_}U)u^9WV?pgJ@6 z^=o@8ia(ZTRrx=Y;(6gydH*xW-v1vQZ}oqhc~;Z^m2Yva;4g^^RzJP+qliS++$loW zmsj&NL>$TctQA5!bPD!lN$Vb&MI=NaxAPd?!@*{oT=@wR{0#%n8K%r=xREjMZK4)eR_A+@ZT46rlADfTXqL|Gh~fI>?{n`PkCHU0wZ=V1zoq%Q7cMgL#Gh zK57MO0OV}g?jMH9%;N223L$@dAHDIHu=vBQai z`vkOw0M~Yfyv7s6q*G9o>t#X0n9N5z-N{55YIVSi05kKaOrBAY2c;(D_mNIUkJIFk zsY*FBT09|X2Ayo`mrm#2P#pP~KS2x^U~dupQ6cz)qw)<#w@(n21=vpvl^-72R;8v3 zH{LzsW{ev|&oaDNl*(%_qU3H~?8(@kavq&5ukuR27YAoa?%yM2b zCSd~|Ix3Rfc3-?t*7E30zpvPUy|gt~a_uQC^NFs?>er<9W$E+34uRR=`G3&cKg`;H zA0PI%=l@MS4|@KuX4e|s%=y2e+q+Hi{Mso+mlt?yPn|wjZf9=knbUIOw>Wc)r0=b6 z|9g6t<^Mip(-DExUqS?^;{W|Ymj53d>~G_LY~*Q>|1S`iO9W~i-ROpXG*Jm$HzQXDivN^1n)s!Lp5+PWpDi^yq)ID9_@ z^@%+V(Er@t%W4Iy>3=Un|9eLV+xYJrdA?oxKPA#VWlQ`Yj`)iu*i!yS{WL)TuV1lp z+5f*))BpY6VLwa%`~8D0{olkRQ%}fU3r=1y+|1Xnn7fzt23ED-V2{fd#+1w?g;no& z?ATg_TN)U27ZJ)4l~F6+a?dvmM>?D38-ER6#vw!8e%Ce_in!;?gg>nNCnI5d2Xt0w zAzL>S8mt`rGZJ!2f&kHsx9}=s5&J2L-NDuymD^TseA*dVws<9%Z+W1=%gOR}z8Hs5 z&3C+H0z29WuU&tx|0E+F^krwqnxbU}wA@fOH;6RWv)-hy6DQ~tOnZ-ggM!@kE4G#6 z?p==4__Y<#zOSbN`A^dIMG|23`G1g={|>kE-)5d~mu*P2+%k>_RRG8}*lGYC@zVhP zj}RRbI+NM=3%Nk0{YU>ed;fR5rT?3Gk& z2V2ur3hd)zH_k3Vqj)mG;bhBx9+dr9YqDiR#ZMFTpZJRvz$@JU?H_F8|7_&>cIAJ8 zmRtJ!u=JPsTj~FCKMm0TTSBh`0{v1RP)Yv>S^ZCcu#Nw=ndjT3{{}6$OyFUeK(Y>7 zPVfkyW$C}U#mpMaFG>qpMgNcY53=%qZ+}bwH}Y8Y|D~NwtjLFF2fR%5GuP-{VF|sL z#}EggeI}_VU?^?3q~dE%V+3xeC|n4^YZ*$t4GQfuY#vtVg5(Tv(FpE6&!w|>*3T-C zZ4Q7x$w)`*cE7EjcA$)5J2UO@Xp^YG%GUUSgvicf=cZ!osA%m>hpUdV$<&;#GO?#j z9ZERyNN@r^o}IVs;N`j0Y=y0*WsAamXB@l=Td>uWc}Zh&mF$dXQwC_{$S?QmZnzE8 z|H8B|RrY_!+5Erz{ey$8{(mFSmi|}J|IKHs$!_mu>2Aa9Wc^GvDKj#E-BSXEJYl@A zEDCj2!Aa0#XM1t@Tn(O*&BC%MmrI@cpe%JrZAfGc)8A>*lKEX56jPxXk`aOw(U0UB zh2mU!0`_ho-J@~1r^8w8xtZ?VoRoW9R9qr4%_2aTO-Zv{Tj_q;CZl=W$!Hd>>+)%6 z>bABpJ$NJflGWuuqu;4c2v{Ni9pv)A?)L^;|L@H__hbLDg89bka#ka8<{fpiD|9Jx z(P4AO(X2TAUZ**hvR)1WXOvty1d5G-x0;*xs_th&GDB`$O2}% z9HUa@Ozm70|4@4{t<^h4kzGYo^-|^2!PDfn_*-GxcCbd8RwrWrx>1(5*N3Y<%gX;#NPQHdT#S!J zPlfz{w11e*|F^%tzm@+t@<{nV*-p|4+?SrOSG_iH)MZMu)wIF4qmwN+{)icRdxsgv z;Us}2>*^~#eXXIiulnt!>`H8>+fL?qGJ@Qjrt-vpl2Nf7(LI|&%DY-<>qMejhrzrW zH$Szds@w859lI7$r9DN#tjnf*Q$nuWCmcW}r!41!(tD~=`<`Y2z#XN6>L94=8l7V!j*2Dr`)0AK(B z`skAKrw{cym4_kB=Vv73=+4SPPUG;DeIVflA>273=}*5iL{C7!*Xt#^Wg~)h>7*N=jhwChHKS7SqgP=ksS--b#LsXK<2b=rySn?bSJKecti_AY44VT(W9 z6SjDx$0q*lMB?Ai|GN!iPG+KjD>_LFE*Di8xudJE3gB%xzM_5_Rt5 z98|tlmZ3~5Bu3Non4W&XCmDHq7rce&)V7fA=PRk+NTP+Y$%p1iW}3Gk#(5>=#n!^s8WvPB~fTG@Q%taTEno zK?8%7QstX`q23A(Y25_N?9Tm!cF*A52{<}D+&^pqFlK5WHPjqM(ElwA=3-#d*jDn> zbV_bW5GS|P9q>Y_g;j+9D#o7OE}>cx98Pj{NiS9@ZW%*t7zE^2{F_55Eh{8gArLl^ ziv(`z%P~!0TOKL6)d z|F@B6x&2>P3-)09e-+EeQed7CF&VU%Rm814D~t4WYcyIzjkgxaHMM+3oX)BFz5yLy zs=K3c*wK6?`L?9muB+5`S;8UP$SduNi)B=HO(^YBwVm~<1NKHZ++$PcUB`2N{?+;g zz?e|L01m`Q_i#@7mRryaU2Wtv|74qNl(NQm(|Fz%Wp8q!TEGPf1o+ooF6gDLkE|dfN?b5p! z7|G0@ncEj1MFE-5kh*#403V_m2^rEq8HXH=!gj4Uzt)SaG32HFx%u#xc`%S2-jltCV~6p*+G!+8wiiW~8~1L&uu%WQ zr!XX}BPNCJp*v2Yf^8Y&|2pyCv;Mz>!Et|k|F@ARL-y5*Ux)?o$_2kv3ob}^YHqKv zKpJ|TyiRiL+t||iH`bB$m&v)`1v@#rdrNi@eHQZ(eLLX*RyCq9ajRV4^{B=KF3k8} z5FwyWJEhS9){X#UFQsXt>pB=G+0agaQ+oAqF~%Aqu~)lB^Ih=#1_lDlYGJ4x`{&QH zs*IK?{uT{<0i&Y>6nb{dm-`5(@Z_HQ^k19hElK3{EFOaC|W6!>yE zJ1pbRWq&gWVI2%4cZeXLew;_BY}cB4Nx3fx0dK#X87WVYxXTE6c#ORy;Ma3)8IWZ? z|HeB1#+G6=J1)P<^&mq4mJGlNJ(w^>N@9-ADH`KDQ8B=b7uL4^7US#M_3SBRb>@#* zljlH?fCVG`j$LxGM500T&&pXc?PFa{&lZhQJ8f1Mx{M^Ch}qnd;fDY)PP+ zv6xoVTAD{I3BuS27m5%LX5Ft?kbt>{NKn+qhQxsyBU9ZKN5EWAenGUObwkjZ@Z9!p z(85CHX!L*20?1g39@)1}$c*~UILqYx(7NqfAc_MwOWV-$lXzQ2M68 zX9T@#Hn{XX^2 ze-ZKDU`FFeNE<)K{-m}ZuuA`Tbaarl|Jpy=#{b{Qv&Q+a2q8)?iK+#-$RCx-?dSP^ z8o5r&D7hZuPN`hRZb;y*yKXZap58zlq&&)gQ!3lmYbrV8UHl(~Furf_h0v|Lp+y0(2fUS<^@W_BKAq=|3XW5aRY-` zN^kF+L_PWUwP7l5vOLSl!tuoc`cND{zVK6(uv)MEEW%m4zqnWMd7S~~$Fe_7(KKh2^Gb2PRa$Brh*khd z#pb8Xg~@um?2_kNU6{US6;O5bpFFfCN5~ogXCn)(0rNk_0=+c}QWUuEyJaEGwAciY ziMy8}_=#Wak{dl~rxB=JYajru7xlMEq1d!EESZogMx4&XQCV)bvTZgoXN(h?s3`Q0 zrQy<%V{MseLF21I z8xjA#R&;7fIDN6~B-(9N%_O09n>-CUJ{^y7i21x8V=#pkr_sMoR%KyUYfCaHQ?qI= z&_7q2i}n;!ziliTGG9}<>7OyC$bS=498NC1De~h0hm&`ckR-3)-XSlR=Y_;eh?9r} zWHSF7n%6%SxgdN@o3PC-HE;u7&UbWy6MPo!)SaIcOCsP8XR!+oE zRuj>gy(0IAAPZ2W3Q6t*O&$bMj9y}`cxub4TC9ear*?MbCfKOKugi!c=%100aSBt3 zy!7+3Osl+~x}(tE#!2`$<_+7i_+-tEnS8i`L(6lSA0x0pzU2euq8%<1=9qi8r%7p)01v1^* zBE2o4e4zn*o&~82;JbF5`$nGn_CMw}qjCSggWh4z|GPIBZ0&zG@horula1!`ty}ph z%l2Swwn!%TMl_2!L+%-*c)c$F!?3iC2etYfcK)W+&|OQtoG_4!j$g6ie+gza)#x#t;p>c4KP+NUEIp9ksMse+#5Z#pt61L zUd!?B(*m>>nR(<^N4QYxsY;R*_v9oeou<7S@XWSl+v3llk~g4`syu z3B+WoW}asG8#o6PHPwIyxe@(wmZ5(uoN;OTuM0Lv{|^oiviTqO5B3kY^nVl2`t-j@ z-?5y_*rn6Zx2{iG-?z7pkLudorLMiGD%+rf%v!Ailh!1}dUCo?V}Ml$0JB$i$L$oQ zrin7z$fBd?;`8=GADqk_V8_xM09<9^T~WUvJWD-5B1-O3>s)z;zEh^T^#C#D*M)<5 zQbNcSX)XP5DKjPF2uuf2jX>fQ`-iapkDsZA|^T_8#Ga5&L>yZ(IS8c2sw84;$k zsf7L>C1GL;maaT*y3r_G5raBFnG0)*4L)66LEoo{v8$^;5~~@d)RJXRj0XD(`^k5T zq5yg_Qlf*172Pinx*%&!z%D57YS3p1Yi%vGSyr0I4Qk`tQoDv0V#m`x94wUoZ(x9Z z$WaG}jKd)4c+_wH{&&#pXYYUe$NL9c`F|77Grf-{9J~xr2wnz+T*nb;FUmF3POd>N zNPya)E&jK&19~!G)-$@Uh3O4-9ehv0ZjBfI&2c{k>DmKSlAXwu+;_ zWOGhKD>iXXd`$yZZGe&yTD`6oWY=y$P_-2sanEThHe$sG$+w_x{<>!{f+HMYjuGpg zp1;!vNoJMAV=#s6Z)k3Me96fr7ySvzUX}a5GC(;+IF0R|O|-Y7U-~}sY8O05cbIV* znOC>zK=cy*Gbtj6=0IZX$U*zU{B?Hmh5<+sm|@1mT!C|W$^x3;0zO5+Ba|W*5t+?N zQ~}-;Qm#{jn4;nvpJJvycIBq59G>;#p$XCUd61_b|BqN?WPs-RfA6T5)&KR6k9%AG zzlrBQ_J%B6QEWd+)hylMllYg#1TR78cq=F zww{5H0_#oGe!!+E2!Pn%N;!+e$*w$Gb9{rO(#U!bL%;P5glHm(33wSvjS%v|E#_13 z-*>v;w=kFk63UtaBoHAA0vw`ltNZ5i>XH+RTF<~4naxND{(g1|d`wxZJHdQU{;R-m zbw~fAd-7lNVmjFq|CpcbCfrLK8bMD`yfF?CYyG9mZll&;x+8eq`b(G3qSjyjPwN@@ zJEWMz47_{umbJPOC4VB1x4PIza8H$^YKAhyn)qzbI{+dn?My|`>mp?AI4^6*|O ztCRjqmrwrHYQ-Vu95K%J9)KDY?EkZ0PCxwo_SfHj?#}#1`dGaG2M5RfO#I*e!O_8b)a5fdkn}AGOr$HcCqhxd5RM}CEDVa$&8Aps$1ZN;UN`l|C|DNW9w>SvE2m$E( z$QR^9@<_3-x~Zt#MPgZxQ zK8!e`f4rP>9Ho-yAG;Wb^v4m0v|;v^ik-| z6~MIy%85#q2x5k)B%J6jA`Ag9W}M8lQrnt5lcL3HQi{`|hja#Z!*1I7!wMH^ctO!kW&%_h`v|qnBlH`ubpg@-{+Lv&4;^c zRP-9X<(@wT=P{d#FLJ&q7n3g@x7+&wQOGIxF zE$9aDDdeUb2Cny+3@eAI-Pg;ZAn=6)6d&vn%2A1zYM46)PSNl}IAx_%e^%PdsXue3 zQonPiQ@>kpN_8U@;L>X}C|mMDcFhObMIWpOqG-W}@2%XV&MxVue<}w^DIw&dO0gjz zp>|M~TSRNgQ3*+48<1KTfM79C0Rvo4;p)%l@SAtObIEFB9V2%NJTTzz%X|~ zJ_KWo0^bZAhI63I@%)T&1t1ipnoMwZ!4zs^uB{4~20WtZM$V;MpUHuzEod62l9)=j z_$+y09)%G~N+{uv6SI10+#Nt-&akhBn2bEm;qX7C-HIf^AVkQQbXcD1 zgb0}#_MScJeAjy37`j0e25IjMykxOA6|6!)l~#~Y34mk}J8L-a7{`JhO4g#*SrbP& z@||5Z7B(eu;LC%kIt;7TCNfqzU|28)%)zZJr4@&w=1laZEiJ2td3GTM=%T4q?FRT7 zS+IA(1q#p&47tK425u2W00xYJIPxVn$%Ai-!WVB(-@Jd@o%t!jA5JJ7!w`1GT6&On zeaNOG0tFLd2vTjrApjc6tz+$~p+fw0>eTijV=z?OgHGDmP#~&}C=n$$SP~#Lx7yiS zaB6U7lI$|GwUDyuNx;2IT2`FY$TA9K!<`ZMSrrFw5&$(rV`nS31n(Q z&Gv)=2`AQmR|q~sA@2eqkC~ZBz_Z69v3-P4C=Ol3lMPxjhiS=10g~BX31sO4rgXIf z{TU7gb++CiA9F*at=Bgg-3}9Qhn9Yb4O4+hHYj&0WDHM2Kt%!ZI}xR%NelG2nv^0z za#>A5+rS{JOnQ_#3x|LX^e(DUGPq9UOtbo z2Lo{N(E?2lNg>E)0|IP}GIGn5V!f=iXaKb;e^ ztpz)YBSE3XT!LYS@(#fs7+RE9{dVqkn8ICEJ^eHkZ4T`_u%V><(4&MY%BHzw7rYx+ z%^aLes5rzU`ST}2#S{$%V(V4d)nm3%K<%fbs#s)VNNR*UsV^|NBekbpH+w-;qpRQ#cDS6*795 z?$*bJ^~<6Al0(gJj~Vgtcn*wZ_MS}$IW`AQo9^ub!@TDtroi3+y0UJq3kne`p}B=P zm3}QotMvQ*VO9k$S=@3HNF=8~O}}Wyuu8~4B!pNpwl63j0^thhBadlXzS=V=XCs;r zp^Ri9n*Ptr-##QlV1y{cj28(BBy{L(f^P;ZjA#i0Ji}ZaaE1lUPavU1*>^Qvw@y~N zb94CXFb+BDXqAL`DYa`C`(bkdcLeLWMkJNustGngG1$040NZ zp)iqyO&x)>N@E2>fbQTd3Z%#h!DRrwYYl3m{lLts$^aDSPAOy=T(jlbVm(Mi@r|gi za{yq(zzv3iCsN|ap6CaH5Z&rSokRuq5>1iIH){A7I+egs4s2H(i$re$7OYtD-I$XZ z1sF06Cpf$tDl6eMAW~-j%{I85O5K>6 zN6gf@Ss{s0Q){1HZomd`d|?5hoBeGKncuxy&pc$qS8_s5|KzWPia_!JXvP?9PWCEGIgZ zfxu{q2E$jqSG^&4sn0t@V?jQYhaEM@Dd*8po*ohO)xc7eA8NbA;m$6&onmjAoGfmq zNN+1M|GKEcg^=6=Y3L3(*-ei+V?uAm=|Y^A#Ca1j&J0I2?WE&UqY;Qo+ykK++PAvuL_42^OR6x($%H99-@+O`zJuZ8d@%)vcWUSUgU0UbrSeoWOaoVg(C{iANswX z0_=z5AC4BZ0B7j88$>DgQRzVRE>yQ(VpD|3LySQ>H@d(4J=6=9bn^Sfui)h{O+R7; zXuEIl^{aLH+s%}cD`ge!vKw!>BOn<;!I8;$Y&Nn|ib*~#Hm%fy&h*fCov~{`C3v%# zakbS)C@H>h(j1V~%#~u{_d9=QMw0abBQxZxF0~w=s6;i-HXp_ALHbfc9zM#{tF4aY zR_47$WJK{X5#&Qql&rE|Dsa`J)X)rOPC+E(IE-x#adNRK4MbMDMoHB~CptjS&?6Le z{E!V#%!x(el=%)L+))=KqJ*A<diJtdo~reztXReK*LXdxQoR0Rd2h!Uv8Uv-ri z1c}Lu7P(yLd=et1j|lKhqQ}vau{b}23F3e#2IMPw9fILhK&X*>sPFYs-~b7Oxv0pJ z`yi{Do`#`R%Fjvc+}fpWDD|K;2tmHQKnMX0P1m7mW!GFZ4(+TjU2RD6TljlY@;|_o z+=?OXs>#C47{B^9PFuTMA}(PPZ)EFYo!lE=1yll#mKXF8tXZ^}Yta;z;{Gs`SY)U* zXv7~T`WpAf!(iv|bDPHDDf>Xe3qpAN1hhGgQF|At_rEhl#mj!L*K2=OBCmE}v~D}4 zjx=J??+l$r(J7q?so6H6$(1}!qny0{{B`K)$cm1giiFxqEmC;;QRlkE&e;y0NNp1MEB>H=d25Uml`|4%3yE{=b|$rj>w$o%R)t2?7Hes*;3YuxtBuU<+(N# z3g)gJ7#SJJT<|z6n`ofRMT|6-wmKxq850x2l#35V5k(AzT%EKMot@n7`igjD=^96K zMNVD{IyB@_Af(5X_CUk?wZ(zNehE4FuhaLxCZjAxP)mv88bZK0)%+`M>Ftz=(?z zTd^-@N-`NCLU=p?T8hx;decav1qZ*s((2C#+~`6hKLor1)(QO5)Hh)Eb;*8yOo63mR#~ zEOk0k%`-Hy6Ng5nu(JX(__n$%z4RB$V&+p%uW4f{P8|`PpzbR6pGEe+zkT?4@$Sdp zKfe3Wj0td6{I|iNm$m=j@AbF#zngfT0coCxN3kT7KgZZd0S*z97Q4TQ^PoBk$|5MS z>QzRnt$r8$MCi48mnJ6O1wOh#0f{7);)ro|1*@|KZc_2Kg3CO2Ct<=k_vd@RKNr;Ov&ReU`Pp8 z$+26=Cq{ais;OY}&;v3(lOqhLW5@>W12FKp#3mXyJvG!qsggDP@

    nxKsrj~ZL+i5lD`OTK*QBz5`L)7vavsQLaJNu8>c)E8V8=zh8eydUl>U|&Aa z?emp(s=xK*=p)ei{hgtq8)y~O@7UIXsQ&|wW+kBuTa#H(lCdo>qD-q`oAVr#hGtI~ z>zm(arSwt_YNP2|xhC(b4DC0hMXe;rqW{_`aH0@5})bG)~6-|L_ z>T<$8BkoNRed*M%i78<`xvOQE>FCCW4(4*#zt)?rLmv!=Bugsk6L3}BhZ>$-m~L5H zbMRe%$NgP`TJ_Qy?A2)}rcUR?G4Q$b_809^gDuEdF>Iqzvgwxn^#|o^9j>C~yQ75? z7(6xZH8T)ilFHK8TGihXpFA$V!Jznrmn7YMO)vtAKs|2GPF4qBW4F6}+#jw(6ECN~ za^UwBc2jqMEUy}1afF?MnS_L$DaO9ZMDnN=o`189KiQ>7^ zhiZ=p%S)>;bwmk)?KH<|e$V zQB>kw1Z%~9l?|}uBqV!RF^cMKh>p+ zNuzRRpt=P5sV>a};>v&5mfKui-zr?K&Hy519gF{L7hiWvOy>wx3(&A=^Sd z72xG|67Wo`Q`^c{2dTqDWBG{>shI=vr+ETpnGCV{8`7HjteCcxm9=KAuR)lc5PN=T zt508Vw#zX=Q%C)n^%u?n0%w9_e#p1aWH-5R02{G8yI3;^Qe_9&X)4A5&$J_{hP$T1 zqqUMvks00Boy9bsJw{eX#;g0^0-ZWY0X8Xd?x9IMYXUA$I)$9?WRlA;K04%1leEDf z`rymuTTib)vQZgj3YN~r&xR`xdFPkk$(FL%n|!^6JGbFxA;*(qw=w6J0^QVUI^~r$ z>}N55**hTDH8|O+lbxG}2ugf$HNenH57uHMxl~|Pg_$SFu7F+t?L>X7>#=%4wT*Zv zIO=6uLm9Zk_FMxf`1sgwee3`Lmpudj|0N7?2_x`~{6q#^!eF_7E@3wQ&l2X=_y4z- zFk=6omN1CVepAk_&>NY6#~3fU55W2Vm$NW*|5p|!<9}yi_;`IketZBQV9_Znt$Hx} z^{UnlWFFC*B zTSS%Jf+=0yoh1tf$fBY@mODYtfbh~C1_=;eGWvvrFPiPa%3I>u3s1!I!R)1E{t!>9A0vtXVt%d20z@_> za&6~A&Rh2L0N)w+?|&#&eOLenZ$I|m0VKKE;*Ji}d`cX`i|^t*>bDvOeu{%}ep|uR za6j!I>6oY+P)i-kLu#!XLz`-N=y}o)7k@7(#%iak4=JK$h`YsG&;C;K=8=J|N`e_a z6S(SVp-FbS$?vKS?vXBFh9CN^ktolEqp#uEW$#lez_N@Pl~M_dU&&ci=8MQ?0bCw4~gnYw8Lp_YE(I%1vkcDaSFXsWOw7e$spMjE~wA)C#4z zQumwRr0(V%cecQ25QvSs!%i5lWOR$n8ZqIkzI9U z6W%12h2^e3b)P_oP6H-ynPjw+_Kvyj-n@mrW?7&r(IN7rPQxQ8E>5s1VcpoL6_bw3 zY^h!AZvyX;oyNwfc04Fr;jyM91j;qwW08i#Bx!yJ7MT$FFN4tO7D~00iQ?XU$|xqW z814LsBjUi;h9URS?Y1Ip-iTQ96%Iht9o6%Kquvu5YQ@S&wOFSjXm9e6l71$jEO?;S z?if{+^FrL_ly&qkV zh#ophL}nLSq{P$rENS{s$g7NpK~6crtcp}ym?ye41yoDJbzdB%hwjAlR5t1YNhzXs zz#N}?3+|!x2~SzmExtr|dT`P=Dqj>R7h{5R8+Qvkif^%Cz!l?JOS-Kh%0P+_La1a+ zStd+KrLknd1qd@cDa_WuzdZfrC*CQceVX>8kSoHVxW%$nZ+ z=ULnO3Ts}>7np6%bLKksWB;KW#s3BOP#X<&W?%ro6%P7kS(cLu8zV(qwN1{R`UYwz z!jQ^PUhs?}aEO!3#mI@?V`RomooLW#|_>`cYm}s@UZ5C9B%%Ro5=wwTmi*k*v~Qx(=#K zjRM&0qp{Al?fN2X@#u)Xug>(A(3wMeQxQs}n$C=raQsF1(LQJuv307y<-H0Qsr1{B z!C=R0Z8+-Ic)l288E>Hl&^i9}keXhE+>h%t5nwd+bOQB_tg`&bq>0(^@+hJZ694Q` z`#``)Y#L;v^nk3fH#pN6tD=C0Lso9LKx<3h#gKL}MVuI`mh59+Ek}b;`989uW|-&+ zrPN-09Fryn8^!)jC+XETL+(7QZ+S_63s*)`jv-l`rBJ@%wppucI5{^?Nq`wW2RRq*47tN} zBesw^Buo)mupupw!d^oLeUCc9Mbye_6$KBR+I=5##*K?B$Xrz$RbSsqP$g74C>ufp z(q)3cGwEzfkVaA;K^cyCGnP zbY(>MR84W3)0EwF2#@rQ)q{uuo{VrR5Dk+t7T43axB;Q)j18|jR1-=73HYW2|EFz#yXARbgo(>C%%(SM)p0|x+z(`mo`)gps zjNMRzYMn}8Bjb;kghMALbF;z_{Qi36B;Rmat?zkx^cQPz@KBs#Cw~>S`>K*7d=h6Q zWhQCd^kk)G%?0AVNl*o$Ry*k$zcv)b3|W{?ilImAXLoQ$r$kT$w8%q{vLZky4U(O6 zWj0=oUN6qwRKD_=GmF<@6Jh8jI;$fvotvBaeLH5A<7bHO9ue|Oss&Z~1sa?YvV5sv$y4Nm1ZyG6jIplbvbBo!$)^2w=$T2r#2cdj+NN_YEVAmo;+ zN;?m^2dRu}^cwUyeR><{fQ`1`jrO9zj}&m$gO{|J6PhR zJV<{W0T-?=RA0X^3|(R>#QVea^=7G;&WfLlul zF>A3anXdLDlW=96x@?J^2m z_Na%#&>B3p2m!qJ8$M760+!Ie+$*GbVz2Xd`VD0xRzDI2IrAV3Lp!4(SIJGPMI#X^ zw`(A;zGh*hLgQGiYR?{>8CaXL5mb9`?zMEvI;PIr$z}Wmk6?|O^Q;$SJiew(!NH!- z8EC2Nhf{Hu2W>vxpE;l_YI}BtPFmpesSC%#$jIJHZopa7Hz7v~H-NuW#ye-pAT@Ns*TKq+A>N`E8hedDv0@t}eHmODZ92c&e>H-UF!5G(C zl+0^Zuf^A(k}u5wRg5#+{geTDS+d2ewbXJnS^`CR5oQ&p*cBedpOX&w9YvYJbFh8~ zPOvGC-@Zrw32rWK-hx0Y@FSo=rO}(l=3&!ha3@W8ID&%+j~)V~V3}JGh#+LSEPjD@ zDDJ5+Qx!96NS5iAmqcx;eN?JP-L^~`vPMWD3~Q%R%||*^Nt>8Lej1hgk!wns9hDT@ zv|0ifH9DWdACj`_yJj08!t;M_3+w#8$+{k89)!}lUjOaFFdwCtoNmq=AXz2dD{Dp& zH{DS7y;Vv-aF(~2u1l=hUa4-mc_*&kVhtda6^Gj@)F;G1T5p`I>@Q-f;DuTkffKV? zZSqhN_q#ve#&q#JYuDpJ*{!J!Z~F$WIc3UH2V~o?r9jH!5*ER7 zj}a+*;ctva%#sCLLv(^(A(Sb{g%xa*Ufjt@`>ea|hDX!kc`#7JC&025!eN15igx9K zYmSn@W;uUJwH6=&pYmaRvB%HeJIMYyjnA=J**7{ojF_p1`}^tM?O*^N(^VNAg3&X)Rnyx?phh)V-lDy#du!@Q2U_V%}nWZ4IT;V=s$c(M)jTX zne3SfhF)ZBb6OS{WTtn&(P!s$88}W(;D6W8qt{ zL@^6&zHQ1sdkHJD4C{aP5-1yW6Yp4morwhwvX`jzSYk#-w5f-!I89mN;^Xrk;*Gja zNxHq=RuBf*ic^Rmjwgxi1ALjTUD>vU2=!lbXq_`_7SbM5Xb6UI}so1RGU;i zi5`cC#j77-pIh0X++#!%z}Z0@4bL-SZfJto4GEol>LSt@M*z@rP*jpmfSo)c=aGz) z>_1ejp~<|z&J2683v>E8OmRiW{C*APDhS-shR8FZ8rA7`p&ziCS*V5?Tm~+g1>KCA zR=V49#MBsS<7Fji^=<4#7iw_{+YTo8)1v=}FY*3|FG07MN{UCiWXO42MvK;@;%ec( zp(ARCP~i-?_Zyh>1JfBCT#QYjb}sOkj#FSrctf*Jnsf*ySg0Uo;LyFO!YQd;8wad& z2*Or6K)2>uOHKk&NaY9amnuEDmPLwc0+)3Fv$IMsll{&TvaBWs>?s=rDP`CaV~~&xS*{+=#1e#$tpp1Rx^#%TH^~X9coToFcw^v7ckWi z`^gO7TdXw794a_ZpP7%WWJ{*Sg@} z_;pqdd_*p1eX+kF>|%YjFwoe*p^X?ifwLAz$Y=Z|0w>%mVhjGlExjXu60B9Zj^Y#KS+xFswzRVi0djR^|1 zE{$3b$dS6PR8V_eW$DWI4c@ag(m|}gBFr!%mp-e(1Jl5mR1p+WbZ46lmUe_P=RF;K zfNDeo`G+}W5C;fgN|Sh#$=9D&kKQBnWGYB~T|;}~B_)6|Wgdx(nTt}p+dq}tg$WT! zGqx%`P#4<-IDqLUt7GQVxfmz5Xmow4SbW1uiH?uT*NX>Zgq6L+j^0f2>FWroN(b$& zIc1~=vJpi~SXexK#|-pB45g~~2x-lWLk)c>^T z!%8Y9BLN{2GP88Gv$9iteqlPW2`RA>WH1GDFOOr2c2RYZ&W7g+77RH5Nzo30K3hCt2XS@Aln%B7vff`wxk2X| zp__Tfx4RB{#&1H;P18c8>W>-Zt+RKu2?+y)QFu{Iomyz5H zR{y4TBUhBG;}AO|U*Ox0Z}(%_9qZ$S+`MIM}iZ!npM zFo0vmn3SduIgmea39JlEVjcUQ$cGW^gL95;c#VAy4J|xVF#T3_m(wa-q@WFNQ>X){ zvRXgng}o!zJ>js5#DFaH?GM3eLsj3pG)qVjfen zje?bi3Yq^a1|%?1Qvl3cZf*Gk)k|(<*z^Armb@*?o%rsl) zRZ)`}9mxU{7a&=0NtP$HdZ}UgiCpPlI?u#vju7dE6_G+QP;0+bQx17+nA$ks_C*`< z5J3t*p{gu{>K>9-O?Opxasrw}m4#~(ufoJNvrBN;Tkd=sB4RSEu)sIUpc&sD7PQ>{Eee9taXtarnl z0yqa8X;spX*Q2h~G2);KO(bZDdlZKL@(O?VqQ0(>rLFK#p*qEj2f4rbUfq&8t<7Or zVWIxqq>^^fbf$VC^&}K>yULB<=skI%_a2pL7dl5AkM+nZm3{{GYj}R^gALkG;ks8<_8*(}Ufp8@3wqg`(EV*PK^ zoND~s{Y7K^u9a=bmV!a#g1SUu^>k)O%mjYSs>e!*^{3T(s+8&UEcthFe z^8z>_mt5iN6djkJ)6E1T#c)m(>F0l=NmU;@RybLXke>;Ch*5jyCOs7AH?T#IE}kL< zp`bNh%Y>YI{0A!f?x!sM+88P&;i+NCGMUFWyOQiwo34c`=%`Ni27Ry=hsUg(A?p!f z+U?V+EC7|pX>cR3MG^_?vG0cur~Fq(7SU`W6%SQB^|pRoTI&KQc^K2WdO$-AUh?$j z_Dq|aR*SQ~CI60N7suIb-nESsZR_?3{4=Bvm)zPmrl~<&TY1YNKI{1*hElBfPf!UX zOuz2Z*WF6k3VWoAVZG^)ieu%otzqm|mM>4fz*(Q)x;ef#N;*zGbiIx(=vQ6u0Zl8I ze}gdPW5e_1{sssKfHwSrH)T(tQOEyJI$}WPOu+ggPZDFq-kbXaoePxUwl3wmh-hld z$%iXlMHc@dI-Z;*p6X7Ah zFB0yKO`tTU31Tj-@F8PN12(~tmM8^TY7BLe-|A&>sLBp`^ajE!bgEoN^i+igIk1^F zy%_U-eSxtPj>5aR!^r0E@A1W?0FNKO$__t#?@OM^vj*SJEhRa2%6KA=1FLeb9iR42 zbiX2qT|7w{da(b>ukSFO#>-(~!wX(qB**BUT>elr^o5%Cpx>AN#q0t3sK|nT8y=VT zgaLqXSi07`o=YVYM8*rh3A|UQomqJkAmwK3<75CT z)oy_@q?Zmo8(vGm+}%63dtnv@ytHZp@5N-aJ5bu@Pr;3w8Q@t(lIvK?Br>gRXEERf z9Fgo)e{RM)>41{}Rj(DkT+f~C&&eE07{f?{bViE6*|m3~C=@pP{RT;w&#|h9lXRpK zTT!#h`<=0Px9W1W2;c`9C#ig~Cbog>oI&hs$*DzpV=&jO(>NR6ns*FxYf}?vZw<F!F=5e3&6e zy*IXR&}OX?0%&bLB%E9mU6tT}$%Cc+RCwyt-=a&r9ZgVO-5mA(NOc`9Ewpr5JH8QG zKKr0vRo6sky5-C}Zollx*IPNdeLU~)1$O_%s)fh)M5(HmL7#fZX4|n~5&petk|$fhbVrOg5G}sB#8_0!u=8eKibM{kHmb z)SkHSn6d(ec8jsqP+Iti?_bV_EV514s6A#e-gtuS!?0FsXd&r`w>Gxb->Vl;mBp=eLGW;WH)TZ+P;zAjF! zE$u>xdm;({@FirPQA9Y(UTSwDSr;kXBX(ph0?9uJ5g5t-Qdv@c9iyD764SX7MrUGY z7&$oH`W8F6BWVaNO|{%WuguCW^1aWDL(X#3}%btgpsI zUzwD>C3$Sc-4luuPV#BnpxcZ!D>-h0npkEYmHY(+7Oa<)kaBDUM>h@petueILz?^e zbTs)<99L8ji5ZIWf2rx_7%;d{wHHP>Jv^Lxezd`@yZt9;)EQwa> zdl_RYwdiSBo<7>zyG|Szq;%&%a_s z{it5TMv;J3Q3lbLfUUBc5*}-<%yx?Fw>?Y!VdkKMyVZmUzJ_jyNkuE?qjm0xk7OS` zz6i3H?04OoL=Q}Cmk^_6FN?K?Ucl0nf?pp?SQnSQKC>XmZgtd(fMgbKCVxx&j%Ip# zEqSkcMk%KL_HcXzv^2N0RDSk+e7b&=(!2S z9=BFHrf%?~P21JfPKO8VYOb(um~Ft@g5UH+*ma7()Yg%2Grl-_wCDmIJ??ZqeKKdX z5`jf;MpK23UW`6)x!xR}T-Hv11~`?XidoLTE(=_sXCG$5u)Iy8*@i_{9#k*C&v%&P zw0gsNHOOV;Z>n|lEZQuzh_(lwt%K*EkaA*0ZQ2^mIDWS{ZgBw6)GY(XrW|dp(7s3+(N;h8m z?^3*Q27-u1fTC$yWbDSHR~bo+>+1{6ml%0kUYL%I#JC$2V}2w&3W66Um5yZJ7j_>3 zrm;d%@S9SAGcGJ9yUxqzA;IJbAgF?UiJVA$#=T` zMtmqZoj?y_UQ=YP4_b3xCXJ}<0pS1#wr?=Mm@P-?;h5+YvHbb)+a{TO1p>NXk5q0+ ze#@|w#+5x>KI<6&2A@s)0(KH2 z#6wNz9|U-(@-JJZ{kz0IeV}SoG5jNK_QvjQOM1jXC<{`(tZ+CY23Itj9BTE#mIer3 zf=k+MRz6@yQT%HhDNg$r*QjkD*QEf%`F7M$y~8E-D^9b+;Tj;% z?egW$@`W)RSjw2{(YyQJ)3Q)`2h2Ho%AO>jaDUnC(ewEq zyflkIVVW0Pmf))a_nR~yk($8kU)Ky5;;OKas;2sol||&~-h!FRjrL*%CL`Ardft4X zt7XR|kKX4;(-RQvXp3mhiqLRKM)D}jZ0RMSO=Aq$*4DJ?^WyuNx7#2V*CkMjr`bGU zAW_(VKp6unn>l@1{)=T%Lq-3@&R9PJOE5=E{_b07Koe8TB`rz8g5pTaBV)Ve<7`}= zkZuT~mmV;aXy?B86#}J3hT7+45CwUqDT_h$Qaj-7@cx}siQT3;0fIJ5>~n8B1=3`g z@ri!&KYGc91(@iz({md=r;!Gf`W$oEbzqwKR%%XnDUqdYlk(au^b6T1w0nSvv3EF7r;u*+v^2|o7-*GkVPRPi2szy3> z{xz836JtwBR9gJm1kMm4G(&q~7@M8K(OI+ayLqZi#JX(pXTq9@OB+-67?fb88=A|qQCha^~3A$;z^mm^SicP>UNdYf2BTG8V*H|VQZ^v9WYT@=QF*TT zwZdYO+DhhQFkzu@a2<|PCv7IWaDm|@ z6m-)la%my!2>V~ORQPu#U)CT=%9IJlwh}hV!+h@&WH^?rXA1F4rzBA<16+t&`E3PO zD9d#xomzxIGtoF4f>bIy)XX=Bk%M&AR`gvA3N20=#=Pg{afdGn_6e9KkICtxUvJSg zHo{2cVu-t-E~yj2_5hlU=#n>YTneU&{=N8XR=i5Hr=3CCXfUKvkQccBOW7d0D&AAD z;A58>jT@DbhLPTvJ$-{i|9dn3T%{dKzW~j|lcS!o;!o-jJL1Hqggk#CjklBJGK@w` zp|q#y)H|G=4Uy?%qP~XzV54{5wnq4HUIjmbli$nMMRoWuW>YfPxUtRowA(uY9=1Ff zl#XYG?bsyd3tksxLMF39wwk^bt!y82m~9b7P))xR<>n-!?4Ozyxi}2=rTWVoJW#G% zQ`=)-TT$e4lwkawF*MNCMG9HiXOFXBf4ZFl8HjolQ2LI;*{X~_0@X(0WM8RUtNOjG zC}I0cu^xT`KbcSv<;ch^aofSa)*0r1{LZw$`AoDEI^<--$s+6qd@ zRa1sPEhQfE+~LyY8G^oQ^eDFd@QDko-HMu8 zPbUCE$uLWam)+@cV?c6v|`Gq+`V6F{F&epNwM~_PZfm zn*H!Jy9+q?acE_&!0JxB1071G@LT4@tQkD+QBY+hm?epQxor#~{U9ak&aHT#Z34JKIsAAkw{AHZY> zUj4m3_kfAXC>M6FKMI5T$m7B(To|sV9tZ+Cqu=I^Iy^?nHJ^6LN)y-BsQL~ zhcmh~n$&wu2DFIrAtEKb--WglQlAX+OB2&umWW+_J{0+v3 zD#ladjvkyHS5%~5{{c*&e=WGXtJKfLz!$xJgPpKiSl|tRsE9CKM+gizGgL0PHs509 zs!o%RvEJ4^|ChAE%IN3q`i@x*AN;^>V&3cOMRqBH%FU8e9il{2O(Ikdk zo+%RTXt6!#+x10O0;qgREJ~&sAx5At(r*u%s|wo%5w` zy)1{;B)e+eBv63aNN9LFUPrnQdZA+Qw$GQl0B7za5*1HGOV!j@huQ@Oft(@ zT}GjYJVK&*PEy`G0A3$}VdY}XK=-w-Tk@1GU537hzQ!OEKd63Zuyr4EaXt`v07O4Y zm?f(xtFZL8xbQ&U+J7^TR1a=0X%+@Y>-58**rw5&Jj7?yq;v00e3*#n4~ZTI6k(Y= z5QsozvHusg>8>zU6)~zyhU=DR?sGJ{%rhqIoq>#2!=`16vCNfj=fV^Jl``Df zqx0^UVOdqqu&0lFfBDO(>^nF?3(c{S7_HQ!k)TyeP@ylJCZdkgDISiz_12EAZw7{|y$omMy=D=H~w(iNS6K#wG?Y?BRnjR@+Vxx-(= z-KEYNj5m~>LN7N&{wXLUNR>oO7~X$TKsj$ssrg%YVb+@&hcE}I$Cw6$l_Kb4!UN9A z>JTVRw1N+Sb!41c8-Eevjq>-Dm5Hl3g36JdeJ<3Q9ptaPe9>sm!G7$Vw@tk#q5o6I zD>dQ~wcqUYuY4w#f5^8;;Z);FqZV>}0HiQ+I$8+QDJ@81Ly%2Wr)M)zOxLBJ?fgci z;D>VI%%)z9ba{9efeQ%^V`i7if0_&Hpf*7=>=h?3HCMjVMOKCfcmx?j1><^0A{?pzh)rkYtgaeM$_l zjoH{v80Dgaa0I)Fnyg@8qJ;E!)?X<1R>bw8>p6O^+6FO9_9_3DVd}Q~k$@JBVpV0> zO>M3GJT_|Sehe26b2=s>nx`UW*2+jHz5|1S3i~{pm=@fp#fadF=FSEAzGBqXT5OG3 z07f|mp4k>G&3HDXla9Pi>KYUq-89O57JT-k-`L(!vcQiS9w6ei0`o+L*w>9z*!xuf z#eMe9HM0(fxvol079oDQKf?!3sCLMhXanB27iI>`DTwx1COLY;vG3*MWnmW>Krk%o zGJz~%lW+;fbHRDwY4%ISe@OITs!Q3XFjoXd>)9u*tg9d{kVhx`z9U_`yLz z8!k!=HZGZ!KvndxzV zW0p4KmMrSWE_NfZLrKF?&v?;f zNMHts$F*0#V$nRVOh}EO`tgu93qL<->B0OuWPy?e@Q}zKJkfPniuS8Rp%;B)Leg;x zOO_G0xQD#v=X|i;7?PEXg$YX}b5{{kTzp4A-5Ar=sS7Q$##N-I_wIW09!bLcie^bq zSF&4?#F8yn;1oJYdy8m*G>T+d!m{J&|8=DRMx92VuCX4CN@`>fAvlIT?6&C7YiH-a z#|aHuS=cN1gv=kIkFXG+^j&-2Wp*F#;!m9dMNSez+iVkFqM$AaNcyp7bhAuHO1p2U zL(rB*VZCrQ|i-Wxi@mR_xDy z+!Ls9L`Vl5#H%koU4tQSjMOpDm$KYedx;>E8H{(WSd~4=#L=jTi&N`bc$IYn6*(Is zc_0B~iA*D`McYb3N>mSLE5Nnv*Z7yo03eRF@U#@Cto87QK#m-yRS)p(A8HlkEq%BRL!Hd{_3lT@YPJqTH#^wtP~IT&~sjEYdIy zClOdBpk|4T;w*Ts^Tla=J@|pP*ngxcF08mLfqd76ncHR*JA6i^NHP^- z-D)7_vd;r<{)`v@JNriowRp-3IKEt)@Aypkhu_wPBfnrrQk8-n`}rsC;cgx)@+fI6 zIczXvL*JBD2^+#U@}<>9h)lhu`K;*;kDwOSlR)h4dntrctX5qMFeG8*a(Lw_{U6BN z*1l%Z&U3gom&f<#*8?v%Bgq_<#RB$N<{5kjR#Ic)P|%d9Oyvl#5B2G@dU9D?xt1*n z3z5*zJK-MzJ|549@7sGF9^o*SBua{M*E7VG>^+i5lFANiaPnWZvun)Jo5d<~_5fUT z0~O2kC5Jmb34O#mM#d5Rc+)cHH45iO0ZsgTtY-;_w)=GpV+q_xNcSgv61La1YIKP> zUrYo{c#WB3#&kVmDe-jP9+sPs?xiWV>TYAIlZH030^5DBiBFnW0yS>F(VT)!4a+ zlx69MM`9*|@#sLljQyq;d;(^Hg9PYz&0l?0R9sy8p5uSaVn_fw99&I4v!N|A*x7Fs zEZqenA)KrVT8#FJuxj-Tjv^aaEuO=4MdDyjb zxIZqZzH!Um7;d1(n8MCrj#%ZzSDR_;_VwqTik2(sO5KU;rYKPDr>ObFfPXc>HNwJ( zXq=V7HvCzKW1p~?{x{gX82xM}iALWArk@q2mV(mh&xHk3tkTg@=fn%ICSrF%o@jzb zgQWbN_2hf)S77|b#nEe?(j6N^Rg+6@*xJ`&Q|9O0m%nnBsjk+| z&b-ILeYzFxjVm|0Q5~FkyEaV*sOKz3qwNS5p_?Ry`$;R{HJ1n-dzT#g)+&Y0ck7I9 zw+VM7qcNBhyg+`v^^dieE$956Nu0%6DCs;=nY+_ptY#&tD9VG=VGm&bbPkDGwK0>W&%}Rh`=^xuv=qx<51a@P;M;RujvB z2nVi)F-Sx54c(75cz(T35eK?GCAob4Z;^&E0{$~H^MQx!tlt&!OEfacaj~X&4+7s` zE8iF#?l(N|&KEG@5)XX2HRw}?NEUT!{DmDb#`W)yE9@Vd}(ygJH3k2D{6-qB!8f+&;; z_Ss{)>`r6Q0sQ4@@%WK>Eywd4S4)KTX>Pf)83;DZaN=6LzrXJQJ`{+W_?}bc4&WYN z^9e$JExH#WCr?_i%}Od-g7ZYzvGCYLPRvl=9rmRSNq8zv%U)jTDWQw$YP9=DK)x=U6W=2 zh7yNe@D~9eaQTZx+l!&}V6>c9pJIsLt*zSj-I1+7aX*l3ZW`PNDDF;7Y2+Qo4zoGYl zzFKb1j!>i79RCaMgdgtZZ)B5GSG?5a<-JV6EU3=vi=iLm>zm!pNX z&^c8lr_!UYqYY=UcGN?n5iiKi4?SEM6W<4gBqQ`W48DETi7soME(z(6S*CVPQGJnn z9$nt1mBK_z*g96by0y{q^Q~=aE^UU&#E8yvGQPSwF0C^oc|ETb=zZpXxcgXIEh9jI zSCi*$ODl^ipHjm$bGXdt%P-YQv-?HY;DGvmKgcB4nBR{T8U z#@j!%Dn_(BX#VWABjzfX1%`ev29+V#i%)dY_cFdI^=RDzk1sBTCV)yeY%4ChNP>5K z!IA5)S>4F{K3MXsq28bgSkiG)J64X`v;I0IFRg%$k?5|YSnGqr!U|n^9XbiDsf!>?C}MPd|j~i zvubRD74_*OxW5`Jj(}9rf!dZuoQ|SJbLKroib|2EwHnAVZ|A<*l4zU^Mj_NqQTRxO zz}Y}2hM-3d8?D-m8@O|OlCV_ptCboZMJ-tDe%~adGRsE)numlQ?r)(fv;*7)svyUf z(-hn<$6%P0l|Ms4_`sm6V{BJ`EKdErCdD~zaLuTp2aQ$O0p^nt_p54GVQbporicEik@qdo{iMV4IiX7i+_6y31s+e4HEeLoeJXxzxC{h( z8`$Mi1+;7s9MXVY_#4;)SpXb4L4qiao*_Q&+FXY zMq?6jv;;!)cQpy}-?D&7d%|nEW0wkz%tv)+D8;g>rYdm9*&tl1%+v}eodj)^+_P~R znU_=U(bAI1_J;2q?x%Kt81^d*dRr1al>Zd=dz-`4>3;RG3PV_(rO)x9%o=2qiq0!i zejA|3rWFf7_sXb1NkUvq0`#FgKbnv2%R(psilRc;%L+ta0>fiOb_U?#jK@x`bgSCs zr4BiME_!<2)!FJu71+SvLP;A;#?zN!tArNkW2Kq}>%~7~0BFn>;U=DZB*KQWW+`X_ z1kWjq(aY4F<3gz{Ak?fW^3c7oyJ!myS%=_{QteatpL)c}ywAaySmqvE z^Paz?$s{!hU@R(!8b)C8hlpr)FL_a8o5IW*!{fWnj_B$-kxCE zeXDssT8B{w|X(?E%B4YU>-+r^hYdsV-rW0iU-oS;riNAT#{ zI?+zRd%$6tlpzL1I8_b=X;uW5SnplXRpJj4%x5mhUg0PIEcx-nQ^+pKO=Z*4U=_K; zk{_orPL|(c^kir*FMo-l9L|EjB{oz=3TEl@$#}Rvlmq^yf_Xm<=5amvschV~MG^BV zL&&=9wj9AxrPwL@V|LciE7M2kGFvjGWYDB&i{Y`xqWB4*5%!5RRac}7m*v7shJn!Q z?L0^#Jv3(MD{Vlhec+K(t8?#!cU z-q1`>M7NP~9NtlYk83GDu;rzG%SQ-w@np~e=@O9!yos=m8N{3aOmwp8Xm#i3Z6jnZ z*nNPGL1OXZt3(Yam`&1c)Lx0kJ=S6s+`DqkJ3BG{oHouM?LW;049_1x?tC8X$bG(Z{kC*cVIhzXLb1Uz2Acr#F= z@7)O--pUyx0ilC%q)SZ5&-7?U=!m^r=;rbI%tE6n)UOJ)!Gr#|8ubngi*UlC74K4YD zz&cAhj zS(ogM5_~zGjQ6LvFJS}YOegEKMUGT`+yg-`%1~hb(>1g(@?M}@U6~P#yQmp-{V5Ai zxGuxKeG=N~3x$h9+b~SFoRho`W8L{J#3J*K1l|OmT21r-mr{MqmFwn(O|>Tj_>t;( z71r*M+L*PDTUSiPeE9Z?lR~MyO_CXlM62+yufvb~Akhi8NRuHac4KEg@L!l^n(wV; zTN%)X%w2jf#%x_fxjYAR2{AM(#3g8{XJ1Ou@DaEf@Dm=4!9VGJpQSyGA`VPK-7UXW zLA*`0?RW=rNF^}_wqIat8h?kd{mJ9oHUD#jf&xEwF@J+p_(u4~N@6a>Ge%(;Q^VDs zf5)%p%HeUta3EsWAdW{dK_SNu-4XUea4*W)a=3S4m?B)ubn zr1xe`b8-!BauqUa#69x2uBo)FT^J2GKM8Gzgy+S)raSd_)Zv8xhr4@ih$+ji2iZFR?)dY=Eh^UgO1Gi$z+`PQ7=r*%~Ks@nUye)~Q+ zM?Xv#114eP$Qb^b~UKly&6Bp2k90!R(_a&Vt;g$u%{(n@8U<&HgGMh%;BKF z^RzHW2u*KY?>*p`DQ(vffW1De1+j@SRXW9Q@fjXluAMoew4@C17W*cAC|9q}HZsv< zR1j@{tjBf+6_w8-dp!zb$Pdm$SCK6Z2N5>RM`{I-CkdAoeZ^CuF{cP<& zfHTfbOfzLivV@|D=nDI4q&!j`%Y#wf3x!#Qx!plbL%50{6^eoqM`DF0_Su3!gJ={E zL>Ve9dH~;JdY4FPBhkB;fPFYZIro}W&KeTyJ%7xH7PTUx9d@WHf}M;qSXz^1z}5To zv^n9x>Lv!jLQCuB&*^L{8B+@8cBzEiDmegj;F| zRN{PHV0?uP7Wlvm^`t?q0r7nU0^^gz`kip35^8Xar>n+vAgA`=>H2~=<6q?{h zd7OX@F;LIF7mm$lW((IAZTjMUvkAOo1G(a_T1RQXTC5` zY>Sc#nxj!#dc)eFb8UtXud^_LhCb+{r@gewZUV!E5@=lL2tmK9irwV!T-)d~4mInHds2ZcE?%?az& zOvUNXb<#8YV3=4DDJ~flY%dO^i>f#_ z*IW&b1eCTZu9+2D5X*4BG5bjYvWm)pM}o8dqScxG4sRK!Rziz^94%Hr?S`K8YzbOC z`=K<)wA1l~j4sBn$EK0KK$n?J>iB#3X-0ogN2Ws_pwT?sBT0s(<#fycJ)=Mu7inA^ z*D6qgIq5ex^fHyPfF;? zO8{wnjgov1F$5`55}n9>QTXOGSR?0|QQyI7q!z^EB5_;5IgA-Yj<*$#( zgpUJqR?ML@x_aH=-^p%Bu2t_c8`H2N1t{4OP%K3aP}4giul=f^6mX-Yx?`R%05993 zY;cU#_BkqJ(|a+Pe5d6M#GHp?X;cQ9{$h=~*T+op@b3hnj%f-BkOrBJ9&hFsv zP@p0^Tj!lVGPDfJ*)RuLHwdAQYFGWz4an&d9NHB_Wa<;MXd`g%GR<}JmBuIn_9L)r z-`}M;LLAFgegX@TnZ^qT6uVQBG(j&Ae`R$ciS7zb(Z;^@x4TK$a~oPo*c;v+P&mw> zJt{DZ3~?bbQFDv2Mq-Uh`NO}^wilx~QnBa2YFLqn<5YpD*B`>jz<(KE4sT<^<>!>x z6v)GS=3sBS*U#e8p|o&$<^e2#qj!rnlj8n9oY8d7p>tY<4rQBrLk3nP^bC$CA-jZ7 zx1T?U`)r~Otr1Dk;cAr)H)4Rsxs0Q!@4v?R-CJD9`@jiM?ifiq~G=Qk&Q>zOl^0q<_3z^0?O>ur*>sjlG zMe5ATJU`Xf!%`Nz(}m8Y$*q$3`(e-6_bE*E(+v>S7d*=t7awKpQO<-=}U&M1i@vR&oER1}Y zS!SFb;37tQ<6ZelC%Y`*Uyt=6&B2$!T=F&m*r9_bw^-9h!H`gKM?OkL;R1F!8;tI# zQrV^LhF9^v4rSAJ&Xfk49St%B(gh3pDFCymu!XPTD^YgMra+{JLqY~qp+1eDZnYy5 zLE}j*Tj+AKH*b*i=>fD}kfIa}X+8X?^0`E4lb@_6C{4@XwbY55P10X{OoapK7N z?X72H$JLY55xHxr6QPg=nb!GByKALgFtk0=q^q};$?Ce@tWs|80o+hLT4ZuX;#lts z)qr+V=6-=a<8%4eSChe3jbA6Rp9oe&P(!f4(|X?ji)no6k!mfPo+G%jyQ`M-K<)=# zpZm-S(c&ds4vc*xODWIp@biFU;Z-m@63t2pvhHAGtnLwNff>S3k{4!)qSuaU`iuLZ zR<(3+KF{(6q-C`nhH}U|&Nh>`yin492q_w_=T0In6ViX~G~*pf4^!NyW6 z7vlm^u}@Y!go|d17vVqIN9@P!7&az85_xN%(Bm|ASpV`KH{ko->KP`wBr`Px%A_|p zs#qyg>es9&HVUNy^(d8^)f0~q1{Qvbx2GZmT$w7q(BkW`;s%}z<`MB#5s z+C}TkOA0KTpbW{2Qml&wt4E+z$ERYn@v5-!d3`P8G#6-zU{prkaDG~7gx@eFbg;?I z^%9O(8kw2Bn$aD!v_{p7Y9mC*{!p9qIUAz_yoD}j?@zGxu;hL!Unr* zd)qgVG7L}w7^-eJe6c+g{U0l5T+^k%B!bs9TIb3Xr@ZnGmi)=|-v|Ln9<}u{0hUhB z!HNE*=~AC|f&1SEXBcTDZI&4{ZXeQhQfmVOsVZ;_qY=E6%Ohj|ls}Q6JFL=T10*NI z=lf>JLy07R#$Uu-zCKyw-<)Vg(LyC(q&u4~zW%CF@9C7AQL-COD0_0PY-%RvNazo; zDz6m&p6{_5RDER*92Vy#!z-xw(Ww}hTj6u4pAF57~Xxt%a*?I(Eif{MUDHz zku3lv@>BY6hI|Age_0Ag8G$_~GF_o)jUTR#VGqdE!-vXEKLS+EhYH496+AHbqCN9A zTX5rt>4tGUrG(%u@&0#m3pk9;<(<9pQ+NPXR%wYyX+>`xUe{R)ex-XF*pT%-MpKG$ z9WLqEsaD8Q&==UK!2D(1>ZxAqp&~rm5OM-Kf~M(_HMB5!1nic1XSYld8DE5qU(si5 zG*=utaZN=-}t8K!(htcuiycw9ilRoW98xr^K<)?A_e7Br0} z6WkiqOrsdLr7UW7J&a@kF42ZZ(jb8oL_HD54S!HIJo2J8Cn-r;2y11~(n(_E{qhY{ z<;z*=cGGMrr1FT^K>Qi-mHq_9ic`VqoUEky{ug9GloRIb5GqWO_ zuhd()v{+3*|Giuqe;uAr_hn;ryX+8Nq&r1A)c)fmArzU>Wi*WP``P8??_FG@U>#!x zf+DT;!_*b)(75HVqm8EnWgr7i*a=}vt6c=T0+_w;+C8^hs-8PXv;eU|V|bws$DNpyzVN{}XZWI= z`XOk9t>xKP)aR|K>s76kf`U#uaO{pIUKTX|1N&0>MJX{8BTU=04Np4~;I^C~*B=g7 zFtLDN2%@|D-0Ty?x=EH3zPW>pmwB|U25OHgT+~C8rCISIuF~Dg65ihNy+#0q@%!Hf zdd9RZ(%5hB{NaDqSN&eyJzaf&ZPI>(Vk%H+HMMs5df(p(ez|@=jCTUVKH$ypjT-5Q z*#FcMB_}#NkZ|KRgpPfL$XuS2>ol;n8W5cHrNdz zD?`;T%H=C|6>wb0UQ9N$b*@ODyjxnn_ZnO{jAJ2nBB=l&a6X(aEpt2{W0rQVG!u_I zS!`se`T_+SOR{&XS{diwNll0W0p;F53g1IaD~^0ufAJJ%I?GM#`Pj!w5$-s+ zUetJtAK-tKkz-?==pGP%Z(s?@ISe%Pr8Tmy_KDMy*1SGKKQ7=tP|^i83TisW?1LH_ zlI-8;^?9;Z9B(}*Z}osEOe_<~^KRsl+|Tlzaq04JKNdxEZtnSoTlrrIcw{N%e-QAU zhoFITUgj63n^Op*k^rt_JqHEErDy+xl5MZY7SAdB2d~g`t#I`Z{R@3&`~=+|eoGAU zcZzH@`VuRacIW5%FIFgPueZmZAvOtSoHTf z;dyoxyq%nIKNkF}e7U*!xOx4&9{*XF??=qW#M_?v;Xwa8Np3cjDt9pJF-BDVcA0OV zBYWomT1^yg9?@l~Tuy)PTTiukqZTB}Ib**l4V7;uI=rsw5%FV}2W#l=-dgWL{S_eq z>H&EL()K!o;MOeD{J~I`nfd)-qtd9Wr8E5DBVDuBjr}nr;>E)gN4)C3{aYlV+JQ$x zF+6I#X<63k3revdhN)!s&+lPi-2dNq6Hoeq-9)UD+t2K?ug>P~ZoQ|kEzcL-uNNr+ zWUsH~8RtiPW7q7ZUhnXmBByoqj&WP!iQ+a)tdcY&jPtWN5 zk2y9kuKqP5X~(CU^1qjoak)=6C+#?21T;UwBF2SwhoOP(DI1~}Jc z;-2ArfwU~fvo<7rVKnoOr!qH=E&}-Q4(EEQQ%RJOBBg2j_5&Nq!ZHk9J;6(vdd|zZ ztKn-VK&;q>IZ5o!jK~*$HC0$AA^i#enJCT1CeQaH`nQqyfp~rjvg&S&DAv{91gtm~ zHh6I*74TqQvZ~a1tX8Ur0y0Uk*7Ay7mdOn-jh#v@FO~W41LJnNm50L}Ewt~oC51)` zv%aM(4WpwpWU`e?@$r!ECkf=o4$E;ODT~U*W8Ci#Cs^Npft)*q zRsrS!`>K1Tg$a|J1EUWom3~;sU;f`bxn0CO+(u^uwbc7??+toe|6NdB0T)#N>y{G{ z#{VD6iF|4QDJPP@_}Q0g9DHqL&fW+r+`o`Nf05n1(szKZjLmN4@=09Yclx#d*^f&% zGaAarmKYMiS0a6>Kr<8;!IkB_x2xmy{)yAz=Ur4)7CAqGwme!oTCiVVp+>7|n?Wg1 z40Kd-N^A3nY)a+9?k?I+h$IfRcsj#;aYnR`DeqDa45kVyX(z^8sB z$|&xbA{wog1vS^=PUEH*9sMhSC#y?npfMvok}Y@1RX;nSq1URC5!AY^HpD7YnH{RQ z7*+Sd70iBIa=Qe2NQH7pz!F_Vl4ntJG@T!l5!kuRUhwkQ(Fjhj#Vn*93 z&S$a`sMzbLd^=1OuKvaZ-h5(Lk-^&lV_ACdD}p31ITnsxFSgDfwSPDPR!$AICjwv> z5RoN&dPm(!F4*Q$8E#O6v|VlRJ^mT`3!!53CjXFAkVu7_2$~_d3FIbixJ> zyx}=q$Hr|B%|K(UNFu5hk&xr5q7mmDinD`61WREF3cTVGBue85 z+UdSPQo6I)ZksAgDO}A?X48h#DBQ^-YO3@@?LJJ28MX$ZjwMX9t#NVwmr3!_h8gyO zBWR`D?C8jEgcuC7Dzf%tan&T*3hTn?l)DdBt;zH)e@;nVTsAu&iHn~=Xv zNKABLsz37^i)4=h#dmg5c-)UHWx*d6QdXR@4XpPh&=bP2Fcn=enz!PG6`r8S!t(is zHJ!V3He<<>p{J7HJO?QUEi3Sj*MjxiR=(Z@zgAxZy4>$b$Io_$;V+<_RF#Ruue6wm z6_)=)fLD)(k819R38lMh45`$FWAzAxX2Y7WJ=$^cybw;}iip87p@b1PRj)QzC?b!u zr_)@@B}$bfS)(UQSL5{Uy&*<^yAOr0k)OzC8sq8CXC0-QCzj1ag*{@Esj)(c@F>eE7af?S`h-c~%`*LO1H69?Xn~7;s+o1Y&de4VAKRZ4~=0?!%k9vHa7qbaC{2!v=0h%paCw1*}6B1R`|m{pi|1U zudNs}Dp={gM=mifk+?3DQ+o!ziN~HqeF!w1OqdtyegGk13~ewA{bMXDboeaw9|N2e zhX4FSF)-Mzi%ozYD}SO%7+#)QjjDw>T<3sQcPcUH7q7BJTTzxH#KhQQ8#+N9EA@&0 zu|7_6i$wQ2mRAku4}RXtfje;;2U+_Fk7OFKQY&`G?_AT}T(=GWdmN4==Kmnz)hM|_ zRmPfv()T4QrQ3`N2~d&@j?9gl9YKWX`+5ex0JWGagxo3U@x1q; zY=ZElyIGqFgR|RQF*NOs*WrVLV(&Y&Uyz7QNkfjp*D$&p6)TlVTFkT*Tta6lP>1qB zZhZWXwdsc)H(bhiEIccUnd>9J8D&K3Ly-z9=-pa-{uv$N-D zj|taiNxH71L>Aav-y;MA`pA$RS7k)q2&$6!;pJXsDmx0s_nf>!+A1#SiL#CY%kQGf z=k&$|G~&RT?Lfhh$R}8jVE6Eos6gpwn$QG-q5lsCJXs3u<39}eZ{$6&5jRufSr;}2 zg7M(GzILuv&p?m_FmC7x7}W+w$%ZgCau}I%K^Z5t%;&tAp3M=N=F(we%8EcM<9ZS^*!-fZqq3=Dcf-Si! z>NLNr?E*SMKOKP$48PO-a%c=NZ0;{+%yj}~HO=6QWQQS4$Qoh{e#Xu$js}4aRGvo} zI5|I0N#u39JJR2+g39tWmkCV*#DEo<)l9zW!>x+2G5neftZNFQ*hd+jczjSP(c>2; z4J$G1XeG~3x}w#66?V_Le~K}n!;0FQi!^U(tMZUr;q9w zy}l7=AQS^XZJxPfYwLAei(#f;?uZv^Cdwtz9lnYR#MZ-5gy$oRtG}8!f6S`(4k=Cg zCdn4vMc>mbN!F$#Ug98Miz{9kXxJ@_gL|v1AO4R64sUj`i!Mo70U=H+MQZPYQwXCX zV?5;1Y~Z=OBoWsw<@fZ(~;-SS?NCqV>8 zSSDL_=|`O`IFIfeA)tk4*u zY664>NN}B#O9hHbZrETRa)aoin?rhuJc+sky)zcTyubYi0yovHFAKB$h02fd4+P#8 z#RY`G8HFOM)f`R&|CxWu2EjM!YR;8L=~;QRB5OYtjM}>@BN7(HZKp(6JwQ>IY%aHSr}b>CUiLFvxh9i}=%8d^-y4WZw;QM0fo{ArWtwr@~QF zUaW;NPWah`nz)?{mU;w{LGW8zTCS=#!4tt809PJqlUgE?XUoOP#4X;;o;S#H%;q)! zn_}5?3I4X404czG14WB0Ij_3>DozN(erE{gc^s>;T_Rd7lF-5QgRS6px1gTV3Nn2I zYOb$Dmcu_lr@Xcz4P#E@(ln4cQRa3d^^IB+=Js3veiEcK!a+JUpaxM~G4(8+@H#=` z**!AHVL^sd)jU(0%&+G=JO=R{TVy{1@SCw{k9oL!5qhRhf#|)vl|g9z0k;Ff4%n}z zb`)xasbcTgJLQ6<3-ua;gvmBx{!Vz6X>)P zA;GgYsF3^^v{KHd(U?VM?>i_)?{6m66M2a_PKs^MJEvxy{D!=(P0|sa;L>$(q=CrH ze6z=DMlb?;!G===K89sqr(#b~PUCnCXq5^bB6&nbH^nzAO{Ec8BetWIru_SdT5Dvy z{Mx8}WDF50!ZwlZ77L|ztg2%{L}VAJfoE5o1lOnDd&SPe=?8LcN{p8wT|h(znw?Eb zK;zaw`;VL-*#MwIa-+S?!*Xuj^KuX{9&stzLni?u7``l^0shE@9<(^#!woFhMi!!d zaUaW3IvEnActQY%ymb2Tp$7e|kTVb@HtgSRno-`3Ol`ii;+35?Jl&#>&(KL>J`M&0 zD3=>7aNf-DhS2yVn=szvofPa`Ao!_`W->d}Ut;I|`8)5Fc zH=_XbXB=2mTc~xbp!KBb;q|a>8=GT8L^gt#3BZMxqZg0@R{;Lrr2m}dFj);oggIPZ z^u?qY_d9+^*=B@Z5FX!@mjP1XHjkHvoLtU7j0#*?+1hb; zpq}mEP?3ab2l5}Q1iy_0fYaGQ%J`e{M5qhMoq^O6HrZ=`hqMI-m;|7@PG459ciXM1 zKHmQt{Asm0@%zjDwYuo7o|KAn&7ga??3_pTwF`M||8}#(JxGD?#f{z6rZ|Y-mG3(g zQjX6If4A3T2MdlYg(-WjtkP|LFe}^MHDOy9%8uE3JIn-N% z3q7OFI=)i<`FuJlR*tQ^u$7&ief2qWvHCH+TKnCrNAQtvcH`WmR{xc^Q}nfBhw)h3 zW7Tc@$v|6uniqY`5i)L?JfWMztbB@yc4>@EkArlq9U@d-3~Lg!{>pmgTH3(o=&Tvp zW?1`08ZW&09)=KN*H@b#qQqNS55oLC{xAs<7+<=5GEd|hfefPVQbv?bjXt z@B9!gvu43qG($E6bkWX+cRQ^w%4((U4~V-;L7S8E=lsuOb^As0U^VKdV@QNiyyr_k z`5Cea+P^`u#oEZ>)t}WPUMlFxIC>JAn4F}Q!R^yq)vG*$l`lXE9E@H4yY&V4 z$e~Q9nycM#bU4@%S?c2iTDGo?Kn=?uTAVtlZ%m;_ZJh(bU!2LCriHhd!N0#TKa2pw zKf87%Dnn=C`yYQXdl<_7E$b%G}PYobN zAp?ebZ2rWA57!eqm|>V(dwLMl!)OByqXjePNt4x(-vgcM7!Nxm!^4SKaH?&c*qWF4 zhndqc@2T>O!?-!)75+q zsaUpkT*;_ZH)~~!34GiQDkC4YV5Jm5{x7 zWmvFlqBX}MFtzoX*Cj}N6EUQuwB^KvgDb}d5s5i5o_d;SY88Ef>?(_<@{zwS<$jgi z-6r(V8t*Gz1AWwx(KgM-g_YesU)mRB`PpC-uGwrJbkfzXTmo~>cm^3D@L`U66iZ$@ zgvYK~E1#U8Gz%LtRQ*%)8dxco1HRdGVl7*>-kIPh;}u+s8xg~A6Ms#-fbd%9dJs6p zL_x`8=$q?vx-e1OdcxWxfCcw-F1x*@C~Fq+0DFTwxHT?Fi2Zva4ahEWo?FwBl=Oca zgX_V*lo_iQD{z8mGaW@Zc=xHb>W%gxI}UY6(i0S+9wMfWKr{k=ywjvZ+{-##XV2?| z?*a)R1m7%(8Lj>Ip&Mi)bgaPjBMRtR+xdMiSO-!;%GSqFb9_Jl)qgQ?kNz0!x2$|) z)9;u8|6t&Um#Pm7@FpqF|6t%4PZ@bgjK_)Y-Rf6T}Aj z#i{#a?C**j=>?U?OGwyIXcb}czjh$N4 zUz-~~FYljk7`d3~CnJG8gd(7pHF*{^U~2O0rrwRJTc#12H=3)yb0BQU-JkY0A@6Aa zl=;VKOG=KS&U)$&TPFs_v|6wsLgq2HMPDc(*-$5HL;g@x`6&7d(6&w+Cski=bSW1IQ6CUq*2lY~W#j0U;yjQ>+_PrB zshoCS1bW~(5ACDuoq^N?8^FE*k~?dU!=@!KX$*K|^BMT+Z58>ds=|0cm_M9KS+{+p z`ELjHN;IP~`KL`BZ5#9uRZ>CqVXnmI>@wXUd^lznhM1yRk?WhM6E%}hq-x*NB}S+s zl4B1N05YztS>sBSG>Jt+QSt<&kSL1*&VoOM)-Xdpl~B6!q`I;g=kIvWdh?lh#EZpW zJSCHPE2tZm(K~i_^nVquyH*tV^nFG*Z_+01-uvAGelp0a(Xp+UvzI4>1wpl2xfIFD zV$KBJ1jk^MXwX#Z`$!9X#&W|w4NI)TR!M^7Cl{#*x*bz_K~(5+;2_Kq9|xDK;fc)ieM0w^bxvup1DM>}R=< z&V#xt?T;x0^n_aHASJP|?>R{H&qMD!G3_=jEAvoyAIP zLxulharH+tzk5n^1S4Q%-T6rbKaB<s~HmrLsOrTkrFYm_@l3kl@F zvjNs7rcGR6T4CT@>{HQflV)`f;rYOiC3&?@HHlE&F&P8*tPbFzuuS>)%hq?;LVQ;2*?l(i0EP7{S zob4wshW~iWkS7K=qP)PXFNR)jfJ#1?J{C6nHTJ}80zZZA@6qFT&YxB*FYr}L(`JCZ zo2nD>cMli=8O1mj`lM7bJdJ+7N$q5=d?I#p%h~zLIR-B(u|;RJr?!h~)=2te54#OA zDTOMtrLMcufplOf)CAVJl6I`Wd$iq01|eE-!uf(4xPpJWP*w3`0>=5rKFcw5s_nyP zyo1%;=b;v@)x|&k)hU3~1k~Xo9+rakTlM92e=l-Tr45}myKo_XhF$|G*P^d>CX4nt ztO7C4O{4#Eqvy1jBg^2x1BeN16`#EE+0p&OIB$=<&aqZ=^RQ1tt)<=9%A>pC#39v{ zC2$RTW3*ozd%zb+@WmJ%#M1Lxp_(rC7>x8X1z6EmH$qhwRDmCyqC)viS%wVFpaR~t z#25`L(sYTa_G^jAR(7M5vm9)ukke8rkEpZQTM2R0sAq81AGHQ5To-j@H{#x8W`N74 z-q9D69Zh-MQe-{#ib)a&lc`~>JI`LRJ*oY7l{V$QeN>>9bbl}Z^gt#{fkww$@iAhh zF5`>ak5T9&iP0bSk<1NJU9-OC?C7zPiQ!29Rd-cuBw6zHJ9ME2$D`|{KhBzOM@uBt z)MI4ZoAlngIULs5^)3~Og*bQmVR-XMM(TI!?jdNzy=#ux3rVbpyKoTih=VMt5ON>g zJqzCOaDTKS6BSQ7$ZP(8uEQol>h(nWj1tTkvxHR{1uriAQMn8eT`3#G>!#|Bx2k@@ zMmduT${o^Rn_34_CW;*duC?wl??P62%jAp;IwpJoS#U?s*BVK)5>?#R#9#sHUg4{)lNh?kgApWD=dMQ%2SV)YZ5lfoDM(eS%_e2&1$6$KeF6jApuK0R49xc~c z@@L`mNq@L^ag9y@!XdfNf2j;bHv7nqihP~EiMYBX|hFyQ@&?uv>y z2{#os+`Nis2Fr>{W(%ZAf_c)sfAupZ6LYjnIhKisR_2Qj->gV*LXy*k)5+LJgE<5o zto}h9afos!bMh0kv`z8Sfb0^7NO&!903P4MmV4SBAUQtYZhIAK+T5J1X)<&Tzk)A3ymqrtB+#py%|Ss zkhDwYY$dU!`ub4{l?SbCvt>1zL`3muX=W-0SFO0ZWe! zE<_V$7mc)H%AcCwJM4JtSjm-yU4V_D3+teX!IdgptmI9Z+Qb5BSHTW_ku}^k z9QL+E`U9cstu5=1-t5%b`<+Gap2&CVtuJdLpGlK7ncJnz{zj>wLZT86WdZG@pItZ-*+^t)Z}#S`+auisYE6uOA8ty{%QjW6MPEv`{uNTb zk*Rmtq?<{J%$u+`J14u1tp%X=MME501MPq_$|yA9Hd4_rKg@Dw?p!OS_`3NPi=e?}t ztc|Uk47y+$Xe+e2Fep)UW~sJJLwJe0L6vN3n&lpgQ$P4T5+%Y&T-Nt9jA4B+89kDL z2})W+PJ4iat{kvL*vRydLnH*#SW&&jHM;GlxLKm_COcI(I_#8LbutcYG27|O{TR&2 z`H-(Ri;GhNkmIqik=IT;4TTGer>Pf`kiy^2kpGz4s~`?xC@q#waDG0SWi&qpSvo?{ zqRgipE+0_%L7v(k5_dAI@x`b62~Ko&w1gV6O(+OI^oG{ssS=h0m&}1>@5Cw4!z^jp~NugOzH%jqAplEk9r8 zT*BKgS@Tz>ymWdyQkj&Q{PR24ov66j)LuVVdUp2yv6hU6PY@3BM&*I;qw>UHf&@|gtCIK{QD7tI%dM2U z){pP$WDm=^O5kf$A&zMI!jA|!_U7h`_vMl!(#vZ*q6W6&<~vh)M(^iq5L4*;g@Aw( z&Y!o%vb1Cq)tz66Mq6bv&9%ie6!eYOde&7xeoE7;(7-B)v`TU`Sz#E+qmR&24xD-Z z<-b9cDggMDG22qsMV?0KhJXHAne^X#;yAfj4LdJrXTt76k^Ak?45~0?6Q3ULgfQnt zqg5asI7l|m9OEJtr3H>Gz-vUTz0|Q{7XN2+R`wx>32eRkMd5d)8v`JElx^8nK__cd zlVod%goMKCfV@WvgD&gKW}VOM8zj5h2)eRBqL3mr`S(3FG)!G=u&|y) z_Si*wx&$UP!&ax$r{4w(0LlUm1K7YJmvK9iCR}whNK$)-B`U-EyCw!x3^+s{fvGj@ z&!yIzLx}>IfdpJ$Ft*G)K8y$$jszEt!JwiD6LR*vCUBa4m6Uzf1|b4M!3XHiK0#|C zN2OTEmqJ7a8}sGoQXZ<}r8qarKr)dwI&sBa$VrJ;xkXFxr55b{(Pmmv$jUO_u5FiO zKiSSbx4ps~_DJY@65(A+L?`dtj6Bh~S|Wc5a6!u@`!i#GBY^Xix ze$@h=G$7_Lc6X7o!h=t!)16wBK}CxIktX1hpV@xSaU;Wp=BS8{5_NURODb|&UO+;s zgP|?(0l;&AJFYGpxq(GL79(md2W!-Ejq(1po4($0N=BkdxI$V5>mnU+79wTX|mE0y%`U;{9oy;#Ul6nF{vqP?)n zl?yMIGHTm|6ezE#rl}a^Es0`ndI<-^UL{AWZ5H4sBXbzP?JHCGE2SCEBmze70wL>n zVPO!=6ZG;5%ktQ^>*wI!ky+A2$WbS_6qv;VBRqmL%S8w2Ck-XWvXi*g=Z9HOC&adc zRX{dICOr*6vuK7yc^@v%_9Eyji$4d$f8)$;yO}-0_A!$jlKwFX3&DMdI#=t>aH)zmHf9lF6oX+JgR4wlJp7@8GZ-9Bv~j+F52B0Cr?x zViGLQ~q1W_amEY0b;uhQP|5+Cy)vvIbndB z2n}oVBCU?)s^?(iam5R6Cm}LAC!;6j{jrq2SUVg@%GB}>*n=)m2uUh+1 z55Dik)MIwCDM-P=k6Ha_OR1QQ6Fiy+aBbtIVGHC_4ty6L@{sdfHR1f^GT-VJ+i$9^ zqt-DF1YGY9>TY>)kE4z}*R$MpQJ)v=GhaY1P>FpDPjNybXM5FCf09K#p0UhRdOid=7DlIwAi>%$O`w@i@$|CV-b2~ zkoJHR62LR8O#L_~m+qi`n8TO1+QTNz{Bf+@VR>}f+Ml_XDUilF8d}N7u ziNT0CL}~;FJS~GXmzxxxHR_L#>oh1uLntV;AE4Kq=}on3r^AJ%%;y)1?I=c!R8%cw z2d7vPzon>;Ls^Vc?9{QG<9t4=^yl_$Kw4M0^`LSE-|pGwlC6DY^VYp{^F)Su$tR;E zu&hIV?6YRe|8%jN0iGUc2Z0)=V$B^`L5rBPTa(=rs%G9=ja}*v-#;Gv z&0`2wEvA-NMnM~_UJ4t*5e6zKN+v9@HhwcFd#DEhV?3hjJ_krPlPX_}%^l7b#A~@O))h zXc~SdvnVxPK$I{q{>0*q7K3)q1lB+~upWq}q29j*vkhNw$b-nV(9oR6+|s#V$ogIT zsb$pOCwZGLzdeNw8WP6{BjL`}O_OC$ih{YSs1s{v1d3X0lr=lIm!q~*iv_)Z7>`ZE z>QwThiHL%7UdNGo7nTJg+EdLN@@8nPor6#!c7=(v!Zb5fm>h*?kpYclc@*A^6+SqL z&!FtmBW(!n@isWSBFW8Da|vk_Uc>z1ctCH z)?|epR-xW$v_iL-!HT3gI}juimmcW@EgNH)hAF=Eb{jH;(EY26mx^BTa8tPw)g`Ap zU4L4OO^l!)G&(EGh_XhY#jpK5;4G)Y=A$s+CIY+KP}Cxe$-64MY88TLG8lo`HLYBG zSC3wfBm`;rDoaQqEu3AD8Z*BmHWw+10^Wiwc2Ezhf68^-|bK}VDGN5hX!vhA!> zl4VtHtd7EkptRC*3n*K8ze1P{21TXX#wQO~WUMsFVons-5~Jx8hUfW|jj%BjW=L5l zKi5NDo!jS5Z|$3PpHbsD19VtF6OtQPa}pgP!$e&h9TxjYygf1g-l_r!mS^E1ljHA3 z>+@&@EAd~{nzOaN^oMnd55lT=T7XwNN+oM;%TV+cUpH}&3x6ulLIcNJigqoj5QcB+ zoQd}g8Y-6$#I?Q`yMZ-*|ZNdux1APBzj^k_x()Ie4MpI#j`yM&SX%FQt*?9mWS9Jhfi3y0bM zDjZ75K|=ND^EUY+8r_smTex_oC}eSCJ_9qXbuQd1^U<7BJ{Bf$a37{L$=e2uXE;Fy zB@+L9lbrm14aO)ZR99ro_oW^8%?5_pF%ZCU{+N$AMULC8z(lBfys#m7eaf@Bio*w% zSmgzLeH)eIGcO0TdsyBeaKNxGvH2)WgjG7EL3v+=Z|auQr%_2Qja9 zur6%{Iq)%Z07D=wJvnlb|3?UZzyGX`|358)u>T5!bylQzT>Dw~{^!|I?fuWuv!_pv z@8kdP;)AJ1RJJ>hv49smPNzC*EK2(`WUoWk!DJ$>O)%W%QYZF!_}xo(hpi}WU5PoF+MdivtU(eaBH$Aci4 zaL)Oo0e}7^c=~K2o*(t?TzhYidT)=OcwQqqlfrQok1b-Ay)~1W&K{}MNSm&jG2&bb zL+?};%OUG@>RG#MeZ9TA*FL;GZ|^HDa*N1#hvX0R&aUb9}oP&k`Z+TYp)mPi(XzP$`_U97&7FJ zWuvi!St>>1du@a>WFLOTx$M>sSqCMIqmH+yOL^fERc2%zWI1FIZeb>UK;K zzxlIqp6m}zpIfudDkH2pp^YeDJl4o(Hwki9ubhe??0Tm4kaf(z%~)tWa@eXH+l-^@ z7gp9L ztl^rAVK?EfIN`r9G&ec4v4Z&ajOTnV{f`pE&CJ>yj$)RQqUVH&ypltbk~zBJw5=OP z*qi2jEX~_a%#I=Tehd-TorK$(k+u!FU6w9!RlqGy_T__as|oTxd!FGWZS?EJylP8) z&9|k3UsGAwU#ykZ(+LD$JbL;9M|?A3WugKyBE|0Y9i>+C%e%KV6Rh4^){I6q+`@@L z1;&&~9*8Azr`9};2?O`E^KvZMJPj$xG@3MG?TLX|tEzeh>%Pjep}_=rE@UAOiR!N+ zDb$m%s9;GM$5<_e&XJOc_SVKeOw#;{=OIc9)?i&RfWJ9|?j~B5hE?6sFGA*G4PiB6 z(Pece|GQ0C zTs@juB3uv3eF-zLG^d$p%zD^tb@1B-tMX9%t%ToQrz)^cWCs|gtma9Rnxl2YnAGtC zsA_#=s<-L&PT;g`ZzUkf&FTr;OgraX8;rF@FlmKhkG&H-lxi_Z##*4qAc-ik?ibSq z)woHS1cegfoJcb7jmZk(nKWVX{Pua9g7q-^DtNIx2nLO zJQp%cDY68hQV&L?nG(;7o>PTxnZIqElmpfuAOgwLMEr&0kjQRQuZH~D^Nc>I@6X#u zA^|GuTMmLzt4W4ou|hrh4r7u$hjD%3z$7)*7mZG$Yp)u`g_^%d^iv5vsFu~FZPt%t zDcGB03OK&zuZD)GRXa9#;LWu%MObgiEeQ1CSC5%+96cq~prLwori_enCs>Kw*|367 zhwO6TANh}Zs@<{uT^E*NA3C+p9YjQsW((j0^3Cd*=MiXJBJ9P{IOqAIH!kI3j6Mbh z!jb>z$UpLo!lKoQC@ElPKeyJ1EqS$UABQd<0LwXjL6Y7m$=2SRNR z#7}d(Vafwt2`&R>q?Ey;)VUn+D&Q=L(=ya&0=RcBuJ8qKE-rEKvo!3zPkTSW{h@5a zN>Ce@1x9777H4Tl3;7|v5|<)B`~p#Lq|>6F$HPmW1i(<~Ww%fd<0$7zTd-3jN-s)e z0Zbcnffx+>T5IAsmU-|@nzi64w`jw=v$jeQnT?s#L3p6@CsVTY3K|)1d7KP|Be?%4 zB@?h9CnbzE41-LU!y%2QzBkf* zk>+B^#Bgj~Mu>Fx=Y!`1kFkxImul9+z9{MlZr(Dn$y`VYb$yK+a^X-v;^tdG$rA+yB12}IhF(W!0L zzO4lEUYE2oxd23j=8KjzAZcDROPC2rrgXBE*q03*Oug7JXXu8_X}aCE%b?Mb+Bl5A zv%Ia|KfUu#KjpmoM*81KUtR{oZ^BM%VX8`z(Qd6tqZAl6DB4!Xs>D>|Jme=_N?~q= zvij9$ukS1T8ncw zg~T4RiwZ}+3-m3i=Gns%hHW@pQ*iW#xZDUhr!=U!Jcyybdy99-G9HQdjc;-Cpp@jn zb!DD*)2``Kcl$2y+GuIUX!>@RkJ8fC1%J79dFhBNd-ZH3%vPqk48QAGt;4YWn{prf^6gaXu=*;ue(}3QW8BtO7L07Qi>(2mP(;Y!Fm&_ zWf-x#Wzwp)XL>fLZ;smG=$7rUj!GKa*{B!U*z`HuN8ENke&k`83n@Q->~uF8rm)p? zJFKIA{}_dv0pX;%?QD#U790Zw-4x4Oq-_V*9eEo}SNh!s*Oh_Vu}w^Mvk|tyxQiGU z=dEYcl*+Xs_4+`-^CWRUjXg% z{X_4;W_9x6pQ_+~Q-I%XCEjiPf3;8)*~5%W>mP=$dH?<7*^_$x$0vh(|KHtwkkpKA zVukO-Ut#T4rTtet-ayNVeM`NnxU6>(CE<`c1!lduD0s*VKJ*x)C(#?5{1_wa4R%?s zs>j9%bIp<{frHB)Lw`)TG+gcStd+FwF`h;Lk*8%QhwQ^cDxUQ4*J`*2d>`si*G$!D z!bX=OAIl*I(|&kZN>S;Ft`B-BCyj{hDdVU$e2DpjVjcqb+>v!=BbtMC|W zo~f;2mL)#yQq)HSR;d9!bz5VCF(-RN9UTpaP22XFO{#%&s}0%~sM~HSENaPd*zE8v z!3d}UDsx2ZXKS3TuyKABG%{ApXu?)6+uxEfYFuQSd%@kHYAJKIS&d5Me=G`? znZWq=ERQauSWMmUOD7uEgj&LPZ_Zv-8=bv*ZNJFN?PgyAB z%kDena8`r$*h6~Zw6pzGd(mSLg~}TJP=BG*TEiFk(o=*#sIT-hYhKLGuyi@@p7lol zBt+=cBem(R={=H8195)-)+U9P`jO%f@86%bL`Jm2t+A&L(mHL@`Lad+zm5L4X-=SZ z`F{pSgPQ*L_~_An{r@}pd>#FdXPE@qdP%QlGGZ8jE=lJ#zH7`LWRsNyA|r(*AFdMPD>(XR*9^m>qS2*@Q9%o!R+l#m->{3 z)c90GIvWL_8zb_$xo#D*bO>P!!c1yq)ur;fYpK3zdbLbiLmO&1)i)W=dRhZ;8*5nt zyF2V^{kYQdn$^dy09$!2vIN#j%<`t-x?$8Sp>-Rs0k%!Bgin}AlA(>(`+bzl;?Ger z`+>TshE)aTr|q@Z24yKR5ntGxm2pfun0C5Qm)ehx++;Y^Hp4a0V&3mIMs9nIXdtRB zed#LzA31(w-*61yz9(XzKM$8SEBMsHYhMtVFKiG?y}xSe_ErWl-;Ks~2NKohJfAv` z*?X+loBS zraGyEUrN!7^^dF8y3!8Pg3p!h+x^!rT)IV1UPM{%65HGuyme+8cVaZT^gyF{d4o(B zHwLk$PZ(%o`srm~6hU9g7zKW!e&?TlhRIZ`{Vv2}O}E~KXzxmOUDovucn_LYMp&}$ z+E*gEbdNf5L3Pd@{(-M3zR%OSo8*E~eko;YZG2BzN@T%{Qu<`hxD{6**nkz(0UNS5 zMyiT_u4Wr*P|dfRBd>}GrXW#wK<%@5l_rJwRJeFSbIng_0&0EbGB!V=Er%Tq27@}D zcBIqevQYE>5e|xM?!4x^(6JXrxdS4Wr=mS=p~f2=H(Nj7Y3siEUGC2?J*{YUG+4RU zyGCTCB3DC<2-0kUIFvh;AgYNC){7Fg_UmMd9+NYd0R_$<*A69!V3xAZx!`#)i;}66 z?FDbtY@sVpVR9*Y)JEy@L&&`JT=+$vZ|6V%^6u5?%Ln_DC;GYMR#2WrDi z@R0QqcGPv|G4O6PO>L3uXK~lM)p{B@S~iz6Ss`S>dVjBM8cq6{{p-J~efXY7>b)m| zVQhLGZ1O28*iq|{HGC}}QI};~H((PrE1=}9a5jkSRhnU@2U~ItbHO~Q&jbTIU%o$i z_aCS4U78KDmQIZx?E1R7!eBN}L-zRbh0G-y}bEy zeDemz8@o$5KUs_QX2+IZYcan`k5&<@dhS}qj=J%@RxuH{Zlmb7E!KJ|^7VG}?_r=8 zd8opGHV4iU)h%IDdA-^|tFx@lp?h_*OYE6UW352esGO8>*E>cXb3!TZlY!RJQ)Ut)I_x|6z z_}t9@>)v6Mg$%WM#{|-NT6!1Zzl}%i>wJjSck+My;NO;4{9h~Rb?1BW#Y0{j5bq-5 zwFM2ogMil-sCFGKxMn+lPr;L49~&EoPP$*-BzDpS?J3=4eGKJJ!V0aaU?*XPONP5m zSYbmT{wI-*Z!Z6BUJ-De{P*lxz5d^$r}y&T-F&`^{HMeJ8WDZFW7#{1)LX-|*^)p# zO7mn``qoi%%ctEz(A@IDt1)xTTVvu{;p=Vp^zJwWu3199J*2Fq-Cc*jHEPqZf^>_} zw=E*qTBXoGYT#Pc8-E+GGofWo)SBC9`Y#9Av&Z69Ay@3%p8!8+yR55=^N zpq=JguT#{OIdny-+n0yfIk`3~5q2uWUrRH6U*u_#25CHG?_W`<5T!Xf(-SW1DqA_{ z#>-xA^1YG#r#O%5vsMsXC;uHitH1wv^5pTOd-?A!KDT@SQB6YEdht1+aA82P-Ssxp>)$^#WRRnsw8vA?CZRzukZ=8sjaNphtjTM$8!_T{@l0!}AfXHJZX!o)5UL+X zvJjO@ksHM4Z!BiG05s(cT_s^fic^E1u0t*x3m^BNum0Ia{zI`FC_le0JKc9BlYXu?z?LN)1rk)9V=cU&dE z-edo(23MsCT%yi+I{jMPUn=MF$3yl&e2OF`Y*u4dAlesyGfi-56-exDVajE`diR=2 zBx3Q#Wa_gs%!$s_QDkSfDDBU>C@ppxi%FWBcGW6tK5pIGwe3FZ-2LbG^0^WJ&t#_W zfF1Dv!P7yL|3A9-|J=#vOYr|~{Q7&Y0=AT&`#*5WI#s`azv1zx%(6J*ghw&!yq^gr z`I-`7VJaj}HshC68@@3NJ2`t}tf+?3>Jrsw=Vz+ov8h$5SZtQ3fe;~Z=T4PzuK4%Y z-v9OI-=B$i9!--p7dM<^-T8m=sGk3GFnIFp{`}v?XOErnq7XS%wxew(qr$k1Vyw!V z@!*0_h4ejDxmQw=J;~%u#4&^OO?I6qnI5uS#Jq?u1qMkw-}5B&_E;jOR8n_8!z4-~ zq}b?x>H1VxI!&-AfW$I^h53Z<`LEAEo)>8@ygl{`%cihDy*g)MluOT_Mnxb0OW=F{ z`0sfi|Es^4P5ba4{inQ4`gWi(4}jxgK(=?_%d5;g@W=eZJMfD+{5#E~sdw<7-X8lC z&!e=I?9J;_>G@fn{#68p=SQL7eQKVkfA#!J8Kj};fAzz?!Ds#Xe|LKF`o~j$9^QD4 z)#v~C>CvNF{m-Lk$AkO(|4u%8jpE*(ccN>8>idmLs^+i1{-BE?|GJM)6o&o&G%9B0 z7}YmjLs&QJQtfLg3dzc>T9+E@QJd1r6iULA&@EC8Z5qXw(}eL#iEz5!eb0MCa;an= z>I+*5tRng%7otC%^)g_1ea-p$^`W|GuFC~4L=!e#ZNL-r8e}G-90T+oN|vaS;IK&Y zOJiNqV%ETe&raBf7PYQ*8Kgb;Gt<1Mi-2}({uB1<9g8I6u}rny(-tQ;7DzWS2nb2q z_=s;31rKJhE4JRaG{v$_1#I8*egRFD@r0DmBoNfe$W3^aL zaAd}gfXrq*4@DwEO=R8G@E~d8*a!_M?Es4Sl%;r#ED%S>&-{Tu@Q)f$T$E!${%8Zs znprjesg6l0_2XTMpokEzN)Z%z!LpHl(=^%uwe5xYac#;_!76G3yNo#FIy!}^lqG73 znaI(Cp(^Rl`2|(96cVl7Bb_XIv}U<*V$!m39$eAOS;_VzU-+u5I~z5Vzl^$`_ue5~ zIEaexm;d|cR9{a^s#7|xPUmp3# zPyWsG&I_Iw@N<+jem-O$&eD)+m4=_sDAgakkK1uUA@t_`-gHjOGc@eVU(Na})gyun@P&C(# zKxK_UiIjwnEEks`8O96Ku*gd>^1c80umASGV+RL6rG*$C9C+WcHx=bk)t&|UcwvU2 z^fGkFuDE1e$~1_OGU={J7NCgQSf7@ii4r#9mw-;VV}T*Wg3$tL_~EMc`d%IC30tJ4 z!QLV*bE8zZQK{C75ggcWRGf)7j6kK-Lyg$JxD*Ls6egJSq~vkDU_mT+0s@0sWEbct zDyWQiEVKgu0}Q6nNQ5%Z(+iPcVjkgU>d^4YGMEu^CuNRRba4#2GLbjo6u`Fevfft! zCyE?LhqWafGEoG6*9S})Tq*HMvZ*MTEXv@**bNY}LzWTDL7GUcMm`gU+rY7;*P~a+ z-83?A+{%ho!YH_JvorznL)Ng0mW!0E&WD5*bpYa6(M}6_QHDrjiBSPWKjZPlo_;NV zk%AbZ&}00E8YH^ojq%yA2vepFTL7KZnb=9FZjIY;F|G&cceRW#7Qi?{W8_)769Cv8 z$T<`RkK)SK7_Y6}bF(?d?Q9Ci+o!;ZxMJ}k?vQQ__aass69uqLb)(fKT(u4WgydJfnN)2Y0dNYi@4;W%Tj5=^Gd%&j zX0L?|@(3TCvuk$xiO;iGuxsy{_29$Q|9jQHu))Yk93ytk@Y*)DUbW0R7MS3Gq+1cP zYb8FRBnM+SPHsI4mB~)57)UZZ#h%KF#0xy>ijy;52_YoPj8sYLZ&fa_?}pzxGks?z z8Gz{c(X$sLT2iCtxdzJofR-3_@!11<%|@NhNTK>?RZwUps$Z)u+YHt1v05V>2M#}+ z)g9;~{c)U*`*RNLqW?M#N_v&1Y#^Bu7J!R`))R(=6wEnLMWB&wQIb8?89`(wf_0Ls*~A-dtcDd&#E0J~ zI!2-5^RLxqo5&)4I!ttl-h5!(DC4DBiNQY3QXfioY7mp+D^8!-hrR1Wo`Y~yA|uE} z5(@PADt8x3*Aq5c5?VDv>*UQKCWYNaUn$A;Q1f zEMo+Cr6`pQAuw0pwVYhD(Qls>&KkhEXSQNUsn{q`S_WM{;2B;E>BQ=wmav?Iq~0N_ zmAU7qO1o}?LV(6H-K&|%g>w>VFS2qRM{-uTrSyO%qe&EKD)B+IqG^%omvIrnWn#@j z3X2lXU6O`c#8JOHrU4X~fq}$PfV)j5?Q1rA@#yJ``Z`oDL|BW7iLJ;ux`h#T(uuht zOqivcmBrj+pFgYjbod&;t#wY9!RODuybN@uS|1arHs2?xQ1Q;l@}iN2TH%p+Os7*@ zotv{2R(LLeL9$8Ar?9m|BD7mJQ#jCTF375&Sg%w@20E7`buXg0IWo2mm_RF06(}l6 z3Zd0$LLy8fi4pK>*Uq8%_GJV-Pp6QQn(ZQ^RI9+de&zi2y0QSs_bJN2cM8X$sO%m zB!PryX{eZZE&?)oEi?&h8BgEss0YmX0z?bPzQEa}EJ{@JN!_{q9<^ZsB>++@xlXKv z8=$YIEDd)>*T^7mkc_#`Q*Tn!cNc{Dk3ki!wS~VPdTzafVvz|8emmGG8Djdp!3oiEOy}os&FgL!%tUaphRjtOI#NHa2T4iv zR@@1A0t^A8UKL-#J27%@O^1=AwTq1^*`{I82!zfwiv&pZ@j_Y50wE>O7b<%mCb0xP zJYlCQ<((cDwZrkk6<&Q~phCulqhd4A^Oi`VO3Q1LnyWcRjb#azssxy7Hd7GXwE;g$ zmSOjc0I8CVE`!k|HT{Yp+H9jsG0A@yxU4471Ym0^HL1Qv#wE_A6;kuAU7H$_vhl?D z?3~onP)wo(WOm!QO{&QS56M@Khd}!_RF+61Xgg~AEf4|imoWl3*=jbw6=Stx2Y{?U z&jdPL9JZvbARF5NmawU|8o`?nt@&Py`1+B{@6nTtR8rN`imWDOMJ>dd{&N)x`l-RaAMR(bTx0Ee2w)Ap77U=p! z`z%1@z&zvfkVS^_Fh-Fs7`I!)>Qx@&o%a^0ei|OT8CN2Uqj{vlE-E6OLt>WwwGHj&a-q@T z-)gz=BOYx6w_^MwD`jImC*ukZhwNtIrh!5D^eFD(sU$h7pBD-$A>)r@K~sKKH}b1$ zJGJ(_inmI~4_S3=!_mo4kzUzQA-tgUxiid1+oYZko)0RZ+-xri!1k?dCgoYSk#-ZNv?>azC0_F|uosY+i{=pM9U^L<~wFX@l&r8uSv!B>Us7 zB!uSPL7C@3PR$6*hmu{=ki<5H#ApHKMV9rIS_uIzjtj9c?u0qd;GTn7F-nkzJFFPh zah*s}xGZJVS%8 zS{CIZ>>eukslV~KWmz1N0VK^sk^72BJn!IrlnwD9gTMx2E%D8!;QB3yR@WC|aZTdz zcO(x77`{YLu&<&x40s;m4)UN7xjgWh=bZ~-BVKJV_YPuS?OKmWv^{~rJV|Nnl;DUJX_Jpg$7?R)?L literal 0 HcmV?d00001 diff --git a/charts/fleet-agent/102.2.5+up0.8.5/Chart.yaml b/charts/fleet-agent/102.2.5+up0.8.5/Chart.yaml new file mode 100644 index 0000000000..3221dd4e82 --- /dev/null +++ b/charts/fleet-agent/102.2.5+up0.8.5/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/certified: rancher + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.16.0-0 < 1.28.0-0' + catalog.cattle.io/namespace: cattle-fleet-system + catalog.cattle.io/os: linux + catalog.cattle.io/permits-os: linux,windows + catalog.cattle.io/rancher-version: '>= 2.7.0-0 < 2.8.0-0' + catalog.cattle.io/release-name: fleet-agent +apiVersion: v2 +appVersion: 0.8.5 +description: Fleet Manager Agent - GitOps at Scale +icon: https://charts.rancher.io/assets/logos/fleet.svg +name: fleet-agent +version: 102.2.5+up0.8.5 diff --git a/charts/fleet-agent/102.2.5+up0.8.5/README.md b/charts/fleet-agent/102.2.5+up0.8.5/README.md new file mode 100644 index 0000000000..2c5724dcef --- /dev/null +++ b/charts/fleet-agent/102.2.5+up0.8.5/README.md @@ -0,0 +1,8 @@ +## Fleet Agent Helm Chart + +Every Fleet-managed downstream cluster will run an agent that communicates back to the Fleet controller. This agent is just another set of Kubernetes controllers running in the downstream cluster. + +Standalone Fleet users use this chart for agent-initiated registration. For more details see [agent-initiated registration](https://fleet.rancher.io/cluster-registration#agent-initiated). +Fleet in Rancher does not use this chart, but creates the agent deployments programmatically. + +The Fleet documentation is centralized in the [doc website](https://fleet.rancher.io/). \ No newline at end of file diff --git a/charts/fleet-agent/102.2.5+up0.8.5/templates/_helpers.tpl b/charts/fleet-agent/102.2.5+up0.8.5/templates/_helpers.tpl new file mode 100644 index 0000000000..6cd96c3ace --- /dev/null +++ b/charts/fleet-agent/102.2.5+up0.8.5/templates/_helpers.tpl @@ -0,0 +1,22 @@ +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- else -}} +{{- "" -}} +{{- end -}} +{{- end -}} + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +kubernetes.io/os: linux +{{- end -}} \ No newline at end of file diff --git a/charts/fleet-agent/102.2.5+up0.8.5/templates/configmap.yaml b/charts/fleet-agent/102.2.5+up0.8.5/templates/configmap.yaml new file mode 100644 index 0000000000..ce61a87568 --- /dev/null +++ b/charts/fleet-agent/102.2.5+up0.8.5/templates/configmap.yaml @@ -0,0 +1,12 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: fleet-agent +data: + config: |- + { + {{ if .Values.labels }} + "labels":{{toJson .Values.labels}}, + {{ end }} + "clientID":"{{.Values.clientID}}" + } diff --git a/charts/fleet-agent/102.2.5+up0.8.5/templates/deployment.yaml b/charts/fleet-agent/102.2.5+up0.8.5/templates/deployment.yaml new file mode 100644 index 0000000000..582eed608d --- /dev/null +++ b/charts/fleet-agent/102.2.5+up0.8.5/templates/deployment.yaml @@ -0,0 +1,51 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: fleet-agent +spec: + selector: + matchLabels: + app: fleet-agent + template: + metadata: + labels: + app: fleet-agent + spec: + containers: + - env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: '{{ template "system_default_registry" . }}{{.Values.image.repository}}:{{.Values.image.tag}}' + name: fleet-agent + command: + - fleetagent + {{- if .Values.debug }} + - --debug + - --debug-level + - {{ quote .Values.debugLevel }} + {{- else }} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + privileged: false + capabilities: + drop: + - ALL + {{- end }} + serviceAccountName: fleet-agent + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.fleetAgent.nodeSelector }} +{{ toYaml .Values.fleetAgent.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.fleetAgent.tolerations }} +{{ toYaml .Values.fleetAgent.tolerations | indent 8 }} +{{- end }} +{{- if not .Values.debug }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 +{{- end }} diff --git a/charts/fleet-agent/102.2.5+up0.8.5/templates/network_policy_allow_all.yaml b/charts/fleet-agent/102.2.5+up0.8.5/templates/network_policy_allow_all.yaml new file mode 100644 index 0000000000..a72109a062 --- /dev/null +++ b/charts/fleet-agent/102.2.5+up0.8.5/templates/network_policy_allow_all.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: default-allow-all + namespace: {{ .Values.internal.systemNamespace }} +spec: + podSelector: {} + ingress: + - {} + egress: + - {} + policyTypes: + - Ingress + - Egress diff --git a/charts/fleet-agent/102.2.5+up0.8.5/templates/patch_default_serviceaccount.yaml b/charts/fleet-agent/102.2.5+up0.8.5/templates/patch_default_serviceaccount.yaml new file mode 100644 index 0000000000..aad4eea415 --- /dev/null +++ b/charts/fleet-agent/102.2.5+up0.8.5/templates/patch_default_serviceaccount.yaml @@ -0,0 +1,28 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: patch-fleet-sa + annotations: + "helm.sh/hook": post-install, post-upgrade + "helm.sh/hook-delete-policy": hook-succeeded, before-hook-creation +spec: + template: + spec: + serviceAccountName: fleet-agent + restartPolicy: Never + containers: + - name: sa + image: "{{ template "system_default_registry" . }}{{ .Values.global.kubectl.repository }}:{{ .Values.global.kubectl.tag }}" + imagePullPolicy: {{ .Values.global.imagePullPolicy }} + command: ["kubectl", "patch", "serviceaccount", "default", "-p", "{\"automountServiceAccountToken\": false}"] + args: ["-n", {{ .Values.internal.systemNamespace }}] + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.kubectl.nodeSelector }} +{{ toYaml .Values.kubectl.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.kubectl.tolerations }} +{{ toYaml .Values.kubectl.tolerations | indent 8 }} +{{- end }} + backoffLimit: 1 diff --git a/charts/fleet-agent/102.2.5+up0.8.5/templates/rbac.yaml b/charts/fleet-agent/102.2.5+up0.8.5/templates/rbac.yaml new file mode 100644 index 0000000000..805949bf2c --- /dev/null +++ b/charts/fleet-agent/102.2.5+up0.8.5/templates/rbac.yaml @@ -0,0 +1,25 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: fleet-agent-system-fleet-agent-role +rules: +- apiGroups: + - '*' + resources: + - '*' + verbs: + - '*' + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: fleet-agent-system-fleet-agent-role-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: fleet-agent-system-fleet-agent-role +subjects: +- kind: ServiceAccount + name: fleet-agent + namespace: {{.Release.Namespace}} diff --git a/charts/fleet-agent/102.2.5+up0.8.5/templates/secret.yaml b/charts/fleet-agent/102.2.5+up0.8.5/templates/secret.yaml new file mode 100644 index 0000000000..4715882047 --- /dev/null +++ b/charts/fleet-agent/102.2.5+up0.8.5/templates/secret.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +data: + systemRegistrationNamespace: "{{b64enc .Values.systemRegistrationNamespace}}" + clusterNamespace: "{{b64enc .Values.clusterNamespace}}" + token: "{{b64enc .Values.token}}" + apiServerURL: "{{b64enc .Values.apiServerURL}}" + apiServerCA: "{{b64enc .Values.apiServerCA}}" +kind: Secret +metadata: + name: fleet-agent-bootstrap diff --git a/charts/fleet-agent/102.2.5+up0.8.5/templates/serviceaccount.yaml b/charts/fleet-agent/102.2.5+up0.8.5/templates/serviceaccount.yaml new file mode 100644 index 0000000000..73e27f0be9 --- /dev/null +++ b/charts/fleet-agent/102.2.5+up0.8.5/templates/serviceaccount.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: fleet-agent diff --git a/charts/fleet-agent/102.2.5+up0.8.5/templates/validate.yaml b/charts/fleet-agent/102.2.5+up0.8.5/templates/validate.yaml new file mode 100644 index 0000000000..d53ff1c508 --- /dev/null +++ b/charts/fleet-agent/102.2.5+up0.8.5/templates/validate.yaml @@ -0,0 +1,11 @@ +{{if ne .Release.Namespace .Values.internal.systemNamespace }} +{{ fail (printf "This chart must be installed in the namespace %s as the release name fleet-agent" .Values.internal.systemNamespace) }} +{{end}} + +{{if ne .Release.Name .Values.internal.managedReleaseName }} +{{ fail (printf "This chart must be installed in the namespace %s as the release name fleet-agent" .Values.internal.managedReleaseName) }} +{{end}} + +{{if not .Values.apiServerURL }} +{{ fail "apiServerURL is required to be set, and most likely also apiServerCA" }} +{{end}} diff --git a/charts/fleet-agent/102.2.5+up0.8.5/values.yaml b/charts/fleet-agent/102.2.5+up0.8.5/values.yaml new file mode 100644 index 0000000000..120a18b7c4 --- /dev/null +++ b/charts/fleet-agent/102.2.5+up0.8.5/values.yaml @@ -0,0 +1,63 @@ +image: + os: "windows,linux" + repository: rancher/fleet-agent + tag: v0.8.5 + +# The public URL of the Kubernetes API server running the Fleet Manager must be set here +# Example: https://example.com:6443 +apiServerURL: "" + +# The the pem encoded value of the CA of the Kubernetes API server running the Fleet Manager. +# If left empty it is assumed this Kubernetes API TLS is signed by a well known CA. +apiServerCA: "" + +# The cluster registration value +token: "" + +# Labels to add to the cluster upon registration only. They are not added after the fact. +#labels: +# foo: bar + +# The client ID of the cluster to associate with +clientID: "" + +# The namespace of the cluster we are register with +clusterNamespace: "" + +# The namespace containing the clusters registration secrets +systemRegistrationNamespace: cattle-fleet-clusters-system + +# Please do not change the below setting unless you really know what you are doing +internal: + systemNamespace: cattle-fleet-system + managedReleaseName: fleet-agent + +# The nodeSelector and tolerations for the agent deployment +fleetAgent: + ## Node labels for pod assignment + ## Ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + ## List of node taints to tolerate (requires Kubernetes >= 1.6) + tolerations: [] +kubectl: + ## Node labels for pod assignment + ## Ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + ## List of node taints to tolerate (requires Kubernetes >= 1.6) + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + operator: "Equal" + value: "true" + effect: NoSchedule + +global: + cattle: + systemDefaultRegistry: "" + kubectl: + repository: rancher/kubectl + tag: v1.21.5 + +debug: false +debugLevel: 0 diff --git a/charts/fleet-crd/102.2.5+up0.8.5/Chart.yaml b/charts/fleet-crd/102.2.5+up0.8.5/Chart.yaml new file mode 100644 index 0000000000..1d7dbab477 --- /dev/null +++ b/charts/fleet-crd/102.2.5+up0.8.5/Chart.yaml @@ -0,0 +1,13 @@ +annotations: + catalog.cattle.io/certified: rancher + catalog.cattle.io/hidden: "true" + catalog.cattle.io/namespace: cattle-fleet-system + catalog.cattle.io/os: linux + catalog.cattle.io/permits-os: linux,windows + catalog.cattle.io/release-name: fleet-crd +apiVersion: v2 +appVersion: 0.8.5 +description: Fleet Manager CustomResourceDefinitions +icon: https://charts.rancher.io/assets/logos/fleet.svg +name: fleet-crd +version: 102.2.5+up0.8.5 diff --git a/charts/fleet-crd/102.2.5+up0.8.5/README.md b/charts/fleet-crd/102.2.5+up0.8.5/README.md new file mode 100644 index 0000000000..2452ab2f1f --- /dev/null +++ b/charts/fleet-crd/102.2.5+up0.8.5/README.md @@ -0,0 +1,5 @@ +# Fleet CRD Helm Chart + +Fleet Manager CustomResourceDefinitions Helm chart is a requirement for the Fleet Helm Chart. + +The Fleet documentation is centralized in the [doc website](https://fleet.rancher.io/). \ No newline at end of file diff --git a/charts/fleet-crd/102.2.5+up0.8.5/templates/crds.yaml b/charts/fleet-crd/102.2.5+up0.8.5/templates/crds.yaml new file mode 100644 index 0000000000..9bda897477 --- /dev/null +++ b/charts/fleet-crd/102.2.5+up0.8.5/templates/crds.yaml @@ -0,0 +1,3453 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: bundles.fleet.cattle.io +spec: + group: fleet.cattle.io + names: + kind: Bundle + plural: bundles + singular: bundle + preserveUnknownFields: false + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.display.readyClusters + name: BundleDeployments-Ready + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].message + name: Status + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + properties: + spec: + properties: + correctDrift: + properties: + enabled: + type: boolean + force: + type: boolean + keepFailHistory: + type: boolean + type: object + defaultNamespace: + nullable: true + type: string + dependsOn: + items: + properties: + name: + nullable: true + type: string + selector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + type: object + nullable: true + type: array + diff: + nullable: true + properties: + comparePatches: + items: + properties: + apiVersion: + nullable: true + type: string + jsonPointers: + items: + nullable: true + type: string + nullable: true + type: array + kind: + nullable: true + type: string + name: + nullable: true + type: string + namespace: + nullable: true + type: string + operations: + items: + properties: + op: + nullable: true + type: string + path: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + type: object + nullable: true + type: array + type: object + forceSyncGeneration: + type: integer + helm: + nullable: true + properties: + atomic: + type: boolean + chart: + nullable: true + type: string + disablePreProcess: + type: boolean + force: + type: boolean + maxHistory: + type: integer + releaseName: + maxLength: 53 + nullable: true + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + repo: + nullable: true + type: string + takeOwnership: + type: boolean + timeoutSeconds: + type: integer + values: + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + valuesFiles: + items: + nullable: true + type: string + nullable: true + type: array + valuesFrom: + items: + properties: + configMapKeyRef: + nullable: true + properties: + key: + nullable: true + type: string + name: + nullable: true + type: string + namespace: + nullable: true + type: string + type: object + secretKeyRef: + nullable: true + properties: + key: + nullable: true + type: string + name: + nullable: true + type: string + namespace: + nullable: true + type: string + type: object + type: object + nullable: true + type: array + version: + nullable: true + type: string + waitForJobs: + type: boolean + type: object + ignore: + properties: + conditions: + items: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + nullable: true + type: array + type: object + keepResources: + type: boolean + kustomize: + nullable: true + properties: + dir: + nullable: true + type: string + type: object + namespace: + nullable: true + type: string + namespaceAnnotations: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + namespaceLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + paused: + type: boolean + resources: + items: + properties: + content: + nullable: true + type: string + encoding: + nullable: true + type: string + name: + nullable: true + type: string + type: object + nullable: true + type: array + rolloutStrategy: + nullable: true + properties: + autoPartitionSize: + nullable: true + type: string + maxUnavailable: + nullable: true + type: string + maxUnavailablePartitions: + nullable: true + type: string + partitions: + items: + properties: + clusterGroup: + nullable: true + type: string + clusterGroupSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + clusterName: + nullable: true + type: string + clusterSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + maxUnavailable: + nullable: true + type: string + name: + nullable: true + type: string + type: object + nullable: true + type: array + type: object + serviceAccount: + nullable: true + type: string + targetRestrictions: + items: + properties: + clusterGroup: + nullable: true + type: string + clusterGroupSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + clusterName: + nullable: true + type: string + clusterSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + name: + nullable: true + type: string + type: object + nullable: true + type: array + targets: + items: + properties: + clusterGroup: + nullable: true + type: string + clusterGroupSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + clusterName: + nullable: true + type: string + clusterSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + correctDrift: + properties: + enabled: + type: boolean + force: + type: boolean + keepFailHistory: + type: boolean + type: object + defaultNamespace: + nullable: true + type: string + diff: + nullable: true + properties: + comparePatches: + items: + properties: + apiVersion: + nullable: true + type: string + jsonPointers: + items: + nullable: true + type: string + nullable: true + type: array + kind: + nullable: true + type: string + name: + nullable: true + type: string + namespace: + nullable: true + type: string + operations: + items: + properties: + op: + nullable: true + type: string + path: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + type: object + nullable: true + type: array + type: object + doNotDeploy: + type: boolean + forceSyncGeneration: + type: integer + helm: + nullable: true + properties: + atomic: + type: boolean + chart: + nullable: true + type: string + disablePreProcess: + type: boolean + force: + type: boolean + maxHistory: + type: integer + releaseName: + nullable: true + type: string + repo: + nullable: true + type: string + takeOwnership: + type: boolean + timeoutSeconds: + type: integer + values: + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + valuesFiles: + items: + nullable: true + type: string + nullable: true + type: array + valuesFrom: + items: + properties: + configMapKeyRef: + nullable: true + properties: + key: + nullable: true + type: string + name: + nullable: true + type: string + namespace: + nullable: true + type: string + type: object + secretKeyRef: + nullable: true + properties: + key: + nullable: true + type: string + name: + nullable: true + type: string + namespace: + nullable: true + type: string + type: object + type: object + nullable: true + type: array + version: + nullable: true + type: string + waitForJobs: + type: boolean + type: object + ignore: + properties: + conditions: + items: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + nullable: true + type: array + type: object + keepResources: + type: boolean + kustomize: + nullable: true + properties: + dir: + nullable: true + type: string + type: object + name: + nullable: true + type: string + namespace: + nullable: true + type: string + namespaceAnnotations: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + namespaceLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + serviceAccount: + nullable: true + type: string + yaml: + nullable: true + properties: + overlays: + items: + nullable: true + type: string + nullable: true + type: array + type: object + type: object + nullable: true + type: array + yaml: + nullable: true + properties: + overlays: + items: + nullable: true + type: string + nullable: true + type: array + type: object + type: object + status: + properties: + conditions: + items: + properties: + lastTransitionTime: + nullable: true + type: string + lastUpdateTime: + nullable: true + type: string + message: + nullable: true + type: string + reason: + nullable: true + type: string + status: + nullable: true + type: string + type: + nullable: true + type: string + type: object + nullable: true + type: array + display: + properties: + readyClusters: + nullable: true + type: string + state: + nullable: true + type: string + type: object + maxNew: + type: integer + maxUnavailable: + type: integer + maxUnavailablePartitions: + type: integer + newlyCreated: + type: integer + observedGeneration: + type: integer + partitions: + items: + properties: + count: + type: integer + maxUnavailable: + type: integer + name: + nullable: true + type: string + summary: + properties: + desiredReady: + type: integer + errApplied: + type: integer + modified: + type: integer + nonReadyResources: + items: + properties: + bundleState: + nullable: true + type: string + message: + nullable: true + type: string + modifiedStatus: + items: + properties: + apiVersion: + nullable: true + type: string + delete: + type: boolean + kind: + nullable: true + type: string + missing: + type: boolean + name: + nullable: true + type: string + namespace: + nullable: true + type: string + patch: + nullable: true + type: string + type: object + nullable: true + type: array + name: + nullable: true + type: string + nonReadyStatus: + items: + properties: + apiVersion: + nullable: true + type: string + kind: + nullable: true + type: string + name: + nullable: true + type: string + namespace: + nullable: true + type: string + summary: + properties: + error: + type: boolean + message: + items: + nullable: true + type: string + nullable: true + type: array + state: + nullable: true + type: string + transitioning: + type: boolean + type: object + uid: + nullable: true + type: string + type: object + nullable: true + type: array + type: object + nullable: true + type: array + notReady: + type: integer + outOfSync: + type: integer + pending: + type: integer + ready: + type: integer + waitApplied: + type: integer + type: object + unavailable: + type: integer + type: object + nullable: true + type: array + resourceKey: + items: + properties: + apiVersion: + nullable: true + type: string + kind: + nullable: true + type: string + name: + nullable: true + type: string + namespace: + nullable: true + type: string + type: object + nullable: true + type: array + summary: + properties: + desiredReady: + type: integer + errApplied: + type: integer + modified: + type: integer + nonReadyResources: + items: + properties: + bundleState: + nullable: true + type: string + message: + nullable: true + type: string + modifiedStatus: + items: + properties: + apiVersion: + nullable: true + type: string + delete: + type: boolean + kind: + nullable: true + type: string + missing: + type: boolean + name: + nullable: true + type: string + namespace: + nullable: true + type: string + patch: + nullable: true + type: string + type: object + nullable: true + type: array + name: + nullable: true + type: string + nonReadyStatus: + items: + properties: + apiVersion: + nullable: true + type: string + kind: + nullable: true + type: string + name: + nullable: true + type: string + namespace: + nullable: true + type: string + summary: + properties: + error: + type: boolean + message: + items: + nullable: true + type: string + nullable: true + type: array + state: + nullable: true + type: string + transitioning: + type: boolean + type: object + uid: + nullable: true + type: string + type: object + nullable: true + type: array + type: object + nullable: true + type: array + notReady: + type: integer + outOfSync: + type: integer + pending: + type: integer + ready: + type: integer + waitApplied: + type: integer + type: object + unavailable: + type: integer + unavailablePartitions: + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: bundledeployments.fleet.cattle.io +spec: + group: fleet.cattle.io + names: + kind: BundleDeployment + plural: bundledeployments + singular: bundledeployment + preserveUnknownFields: false + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.display.deployed + name: Deployed + type: string + - jsonPath: .status.display.monitored + name: Monitored + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].message + name: Status + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + properties: + spec: + properties: + correctDrift: + properties: + enabled: + type: boolean + force: + type: boolean + keepFailHistory: + type: boolean + type: object + dependsOn: + items: + properties: + name: + nullable: true + type: string + selector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + type: object + nullable: true + type: array + deploymentID: + nullable: true + type: string + options: + properties: + correctDrift: + properties: + enabled: + type: boolean + force: + type: boolean + keepFailHistory: + type: boolean + type: object + defaultNamespace: + nullable: true + type: string + diff: + nullable: true + properties: + comparePatches: + items: + properties: + apiVersion: + nullable: true + type: string + jsonPointers: + items: + nullable: true + type: string + nullable: true + type: array + kind: + nullable: true + type: string + name: + nullable: true + type: string + namespace: + nullable: true + type: string + operations: + items: + properties: + op: + nullable: true + type: string + path: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + type: object + nullable: true + type: array + type: object + forceSyncGeneration: + type: integer + helm: + nullable: true + properties: + atomic: + type: boolean + chart: + nullable: true + type: string + disablePreProcess: + type: boolean + force: + type: boolean + maxHistory: + type: integer + releaseName: + maxLength: 53 + nullable: true + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + repo: + nullable: true + type: string + takeOwnership: + type: boolean + timeoutSeconds: + type: integer + values: + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + valuesFiles: + items: + nullable: true + type: string + nullable: true + type: array + valuesFrom: + items: + properties: + configMapKeyRef: + nullable: true + properties: + key: + nullable: true + type: string + name: + nullable: true + type: string + namespace: + nullable: true + type: string + type: object + secretKeyRef: + nullable: true + properties: + key: + nullable: true + type: string + name: + nullable: true + type: string + namespace: + nullable: true + type: string + type: object + type: object + nullable: true + type: array + version: + nullable: true + type: string + waitForJobs: + type: boolean + type: object + ignore: + properties: + conditions: + items: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + nullable: true + type: array + type: object + keepResources: + type: boolean + kustomize: + nullable: true + properties: + dir: + nullable: true + type: string + type: object + namespace: + nullable: true + type: string + namespaceAnnotations: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + namespaceLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + serviceAccount: + nullable: true + type: string + yaml: + nullable: true + properties: + overlays: + items: + nullable: true + type: string + nullable: true + type: array + type: object + type: object + paused: + type: boolean + stagedDeploymentID: + nullable: true + type: string + stagedOptions: + properties: + correctDrift: + properties: + enabled: + type: boolean + force: + type: boolean + keepFailHistory: + type: boolean + type: object + defaultNamespace: + nullable: true + type: string + diff: + nullable: true + properties: + comparePatches: + items: + properties: + apiVersion: + nullable: true + type: string + jsonPointers: + items: + nullable: true + type: string + nullable: true + type: array + kind: + nullable: true + type: string + name: + nullable: true + type: string + namespace: + nullable: true + type: string + operations: + items: + properties: + op: + nullable: true + type: string + path: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + type: object + nullable: true + type: array + type: object + forceSyncGeneration: + type: integer + helm: + nullable: true + properties: + atomic: + type: boolean + chart: + nullable: true + type: string + disablePreProcess: + type: boolean + force: + type: boolean + maxHistory: + type: integer + releaseName: + nullable: true + type: string + repo: + nullable: true + type: string + takeOwnership: + type: boolean + timeoutSeconds: + type: integer + values: + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + valuesFiles: + items: + nullable: true + type: string + nullable: true + type: array + valuesFrom: + items: + properties: + configMapKeyRef: + nullable: true + properties: + key: + nullable: true + type: string + name: + nullable: true + type: string + namespace: + nullable: true + type: string + type: object + secretKeyRef: + nullable: true + properties: + key: + nullable: true + type: string + name: + nullable: true + type: string + namespace: + nullable: true + type: string + type: object + type: object + nullable: true + type: array + version: + nullable: true + type: string + waitForJobs: + type: boolean + type: object + ignore: + properties: + conditions: + items: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + nullable: true + type: array + type: object + keepResources: + type: boolean + kustomize: + nullable: true + properties: + dir: + nullable: true + type: string + type: object + namespace: + nullable: true + type: string + namespaceAnnotations: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + namespaceLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + serviceAccount: + nullable: true + type: string + yaml: + nullable: true + properties: + overlays: + items: + nullable: true + type: string + nullable: true + type: array + type: object + type: object + type: object + status: + properties: + appliedDeploymentID: + nullable: true + type: string + conditions: + items: + properties: + lastTransitionTime: + nullable: true + type: string + lastUpdateTime: + nullable: true + type: string + message: + nullable: true + type: string + reason: + nullable: true + type: string + status: + nullable: true + type: string + type: + nullable: true + type: string + type: object + nullable: true + type: array + display: + properties: + deployed: + nullable: true + type: string + monitored: + nullable: true + type: string + state: + nullable: true + type: string + type: object + modifiedStatus: + items: + properties: + apiVersion: + nullable: true + type: string + delete: + type: boolean + kind: + nullable: true + type: string + missing: + type: boolean + name: + nullable: true + type: string + namespace: + nullable: true + type: string + patch: + nullable: true + type: string + type: object + nullable: true + type: array + nonModified: + type: boolean + nonReadyStatus: + items: + properties: + apiVersion: + nullable: true + type: string + kind: + nullable: true + type: string + name: + nullable: true + type: string + namespace: + nullable: true + type: string + summary: + properties: + error: + type: boolean + message: + items: + nullable: true + type: string + nullable: true + type: array + state: + nullable: true + type: string + transitioning: + type: boolean + type: object + uid: + nullable: true + type: string + type: object + nullable: true + type: array + ready: + type: boolean + release: + nullable: true + type: string + resources: + items: + properties: + apiVersion: + nullable: true + type: string + createdAt: + nullable: true + type: string + kind: + nullable: true + type: string + name: + nullable: true + type: string + namespace: + nullable: true + type: string + type: object + nullable: true + type: array + syncGeneration: + nullable: true + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: bundlenamespacemappings.fleet.cattle.io +spec: + group: fleet.cattle.io + names: + kind: BundleNamespaceMapping + plural: bundlenamespacemappings + singular: bundlenamespacemapping + preserveUnknownFields: false + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + properties: + bundleSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + namespaceSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: clustergroups.fleet.cattle.io +spec: + group: fleet.cattle.io + names: + categories: + - fleet + kind: ClusterGroup + plural: clustergroups + singular: clustergroup + preserveUnknownFields: false + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.display.readyClusters + name: Clusters-Ready + type: string + - jsonPath: .status.display.readyBundles + name: Bundles-Ready + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].message + name: Status + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + properties: + spec: + properties: + selector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + type: object + status: + properties: + clusterCount: + type: integer + conditions: + items: + properties: + lastTransitionTime: + nullable: true + type: string + lastUpdateTime: + nullable: true + type: string + message: + nullable: true + type: string + reason: + nullable: true + type: string + status: + nullable: true + type: string + type: + nullable: true + type: string + type: object + nullable: true + type: array + display: + properties: + readyBundles: + nullable: true + type: string + readyClusters: + nullable: true + type: string + state: + nullable: true + type: string + type: object + nonReadyClusterCount: + type: integer + nonReadyClusters: + items: + nullable: true + type: string + nullable: true + type: array + resourceCounts: + properties: + desiredReady: + type: integer + missing: + type: integer + modified: + type: integer + notReady: + type: integer + orphaned: + type: integer + ready: + type: integer + unknown: + type: integer + waitApplied: + type: integer + type: object + summary: + properties: + desiredReady: + type: integer + errApplied: + type: integer + modified: + type: integer + nonReadyResources: + items: + properties: + bundleState: + nullable: true + type: string + message: + nullable: true + type: string + modifiedStatus: + items: + properties: + apiVersion: + nullable: true + type: string + delete: + type: boolean + kind: + nullable: true + type: string + missing: + type: boolean + name: + nullable: true + type: string + namespace: + nullable: true + type: string + patch: + nullable: true + type: string + type: object + nullable: true + type: array + name: + nullable: true + type: string + nonReadyStatus: + items: + properties: + apiVersion: + nullable: true + type: string + kind: + nullable: true + type: string + name: + nullable: true + type: string + namespace: + nullable: true + type: string + summary: + properties: + error: + type: boolean + message: + items: + nullable: true + type: string + nullable: true + type: array + state: + nullable: true + type: string + transitioning: + type: boolean + type: object + uid: + nullable: true + type: string + type: object + nullable: true + type: array + type: object + nullable: true + type: array + notReady: + type: integer + outOfSync: + type: integer + pending: + type: integer + ready: + type: integer + waitApplied: + type: integer + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: clusters.fleet.cattle.io +spec: + group: fleet.cattle.io + names: + kind: Cluster + plural: clusters + singular: cluster + preserveUnknownFields: false + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.display.readyBundles + name: Bundles-Ready + type: string + - jsonPath: .status.display.readyNodes + name: Nodes-Ready + type: string + - jsonPath: .status.display.sampleNode + name: Sample-Node + type: string + - jsonPath: .status.agent.lastSeen + name: Last-Seen + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].message + name: Status + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + properties: + metadata: + properties: + name: + maxLength: 63 + pattern: ^[-a-z0-9]+$ + type: string + type: object + spec: + properties: + agentAffinity: + nullable: true + properties: + nodeAffinity: + nullable: true + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + enum: + - In + - NotIn + - Exists + - DoesNotExist + - Gt + - Lt + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchFields: + items: + properties: + key: + nullable: true + type: string + operator: + enum: + - In + - NotIn + - Exists + - DoesNotExist + - Gt + - Lt + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + type: object + weight: + type: integer + type: object + nullable: true + type: array + requiredDuringSchedulingIgnoredDuringExecution: + nullable: true + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + enum: + - In + - NotIn + - Exists + - DoesNotExist + - Gt + - Lt + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchFields: + items: + properties: + key: + nullable: true + type: string + operator: + enum: + - In + - NotIn + - Exists + - DoesNotExist + - Gt + - Lt + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + type: object + nullable: true + type: array + type: object + type: object + podAffinity: + nullable: true + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + enum: + - In + - NotIn + - Exists + - DoesNotExist + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + namespaceSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + enum: + - In + - NotIn + - Exists + - DoesNotExist + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + namespaces: + items: + nullable: true + type: string + nullable: true + type: array + topologyKey: + nullable: true + type: string + type: object + weight: + type: integer + type: object + nullable: true + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + enum: + - In + - NotIn + - Exists + - DoesNotExist + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + namespaceSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + enum: + - In + - NotIn + - Exists + - DoesNotExist + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + namespaces: + items: + nullable: true + type: string + nullable: true + type: array + topologyKey: + nullable: true + type: string + type: object + nullable: true + type: array + type: object + podAntiAffinity: + nullable: true + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + enum: + - In + - NotIn + - Exists + - DoesNotExist + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + namespaceSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + enum: + - In + - NotIn + - Exists + - DoesNotExist + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + namespaces: + items: + nullable: true + type: string + nullable: true + type: array + topologyKey: + nullable: true + type: string + type: object + weight: + type: integer + type: object + nullable: true + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + enum: + - In + - NotIn + - Exists + - DoesNotExist + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + namespaceSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + enum: + - In + - NotIn + - Exists + - DoesNotExist + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + namespaces: + items: + nullable: true + type: string + nullable: true + type: array + topologyKey: + nullable: true + type: string + type: object + nullable: true + type: array + type: object + type: object + agentEnvVars: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + valueFrom: + nullable: true + properties: + configMapKeyRef: + nullable: true + properties: + key: + nullable: true + type: string + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + fieldRef: + nullable: true + properties: + apiVersion: + nullable: true + type: string + fieldPath: + nullable: true + type: string + type: object + resourceFieldRef: + nullable: true + properties: + containerName: + nullable: true + type: string + divisor: + nullable: true + type: string + resource: + nullable: true + type: string + type: object + secretKeyRef: + nullable: true + properties: + key: + nullable: true + type: string + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + type: object + type: object + nullable: true + type: array + agentNamespace: + nullable: true + type: string + agentResources: + nullable: true + properties: + claims: + items: + properties: + name: + nullable: true + type: string + type: object + nullable: true + type: array + limits: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + requests: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + agentTolerations: + items: + properties: + effect: + nullable: true + type: string + key: + nullable: true + type: string + operator: + nullable: true + type: string + tolerationSeconds: + maximum: 86400 + nullable: true + type: integer + value: + nullable: true + type: string + type: object + nullable: true + type: array + clientID: + nullable: true + type: string + kubeConfigSecret: + nullable: true + type: string + paused: + type: boolean + privateRepoURL: + nullable: true + type: string + redeployAgentGeneration: + type: integer + templateValues: + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + status: + properties: + agent: + properties: + lastSeen: + nullable: true + type: string + namespace: + nullable: true + type: string + nonReadyNodeNames: + items: + nullable: true + type: string + nullable: true + type: array + nonReadyNodes: + type: integer + readyNodeNames: + items: + nullable: true + type: string + nullable: true + type: array + readyNodes: + type: integer + type: object + agentAffinityHash: + nullable: true + type: string + agentConfigChanged: + type: boolean + agentDeployedGeneration: + nullable: true + type: integer + agentEnvVarsHash: + nullable: true + type: string + agentMigrated: + type: boolean + agentNamespaceMigrated: + type: boolean + agentPrivateRepoURL: + nullable: true + type: string + agentResourcesHash: + nullable: true + type: string + agentTolerationsHash: + nullable: true + type: string + apiServerCAHash: + nullable: true + type: string + apiServerURL: + nullable: true + type: string + cattleNamespaceMigrated: + type: boolean + conditions: + items: + properties: + lastTransitionTime: + nullable: true + type: string + lastUpdateTime: + nullable: true + type: string + message: + nullable: true + type: string + reason: + nullable: true + type: string + status: + nullable: true + type: string + type: + nullable: true + type: string + type: object + nullable: true + type: array + desiredReadyGitRepos: + type: integer + display: + properties: + readyBundles: + nullable: true + type: string + readyNodes: + nullable: true + type: string + sampleNode: + nullable: true + type: string + state: + nullable: true + type: string + type: object + namespace: + nullable: true + type: string + readyGitRepos: + type: integer + resourceCounts: + properties: + desiredReady: + type: integer + missing: + type: integer + modified: + type: integer + notReady: + type: integer + orphaned: + type: integer + ready: + type: integer + unknown: + type: integer + waitApplied: + type: integer + type: object + summary: + properties: + desiredReady: + type: integer + errApplied: + type: integer + modified: + type: integer + nonReadyResources: + items: + properties: + bundleState: + nullable: true + type: string + message: + nullable: true + type: string + modifiedStatus: + items: + properties: + apiVersion: + nullable: true + type: string + delete: + type: boolean + kind: + nullable: true + type: string + missing: + type: boolean + name: + nullable: true + type: string + namespace: + nullable: true + type: string + patch: + nullable: true + type: string + type: object + nullable: true + type: array + name: + nullable: true + type: string + nonReadyStatus: + items: + properties: + apiVersion: + nullable: true + type: string + kind: + nullable: true + type: string + name: + nullable: true + type: string + namespace: + nullable: true + type: string + summary: + properties: + error: + type: boolean + message: + items: + nullable: true + type: string + nullable: true + type: array + state: + nullable: true + type: string + transitioning: + type: boolean + type: object + uid: + nullable: true + type: string + type: object + nullable: true + type: array + type: object + nullable: true + type: array + notReady: + type: integer + outOfSync: + type: integer + pending: + type: integer + ready: + type: integer + waitApplied: + type: integer + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: clusterregistrationtokens.fleet.cattle.io +spec: + group: fleet.cattle.io + names: + kind: ClusterRegistrationToken + plural: clusterregistrationtokens + singular: clusterregistrationtoken + preserveUnknownFields: false + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.secretName + name: Secret-Name + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + properties: + metadata: + properties: + name: + maxLength: 63 + pattern: ^[-a-z0-9]+$ + type: string + type: object + spec: + properties: + ttl: + nullable: true + type: string + type: object + status: + properties: + expires: + nullable: true + type: string + secretName: + nullable: true + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: gitrepos.fleet.cattle.io +spec: + group: fleet.cattle.io + names: + categories: + - fleet + kind: GitRepo + plural: gitrepos + singular: gitrepo + preserveUnknownFields: false + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.repo + name: Repo + type: string + - jsonPath: .status.commit + name: Commit + type: string + - jsonPath: .status.display.readyBundleDeployments + name: BundleDeployments-Ready + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].message + name: Status + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + properties: + spec: + properties: + branch: + nullable: true + type: string + caBundle: + nullable: true + type: string + clientSecretName: + nullable: true + type: string + correctDrift: + properties: + enabled: + type: boolean + force: + type: boolean + keepFailHistory: + type: boolean + type: object + forceSyncGeneration: + type: integer + helmRepoURLRegex: + nullable: true + type: string + helmSecretName: + nullable: true + type: string + helmSecretNameForPaths: + nullable: true + type: string + imageScanCommit: + properties: + authorEmail: + nullable: true + type: string + authorName: + nullable: true + type: string + messageTemplate: + nullable: true + type: string + type: object + imageScanInterval: + nullable: true + type: string + insecureSkipTLSVerify: + type: boolean + keepResources: + type: boolean + paths: + items: + nullable: true + type: string + nullable: true + type: array + paused: + type: boolean + pollingInterval: + nullable: true + type: string + repo: + nullable: true + type: string + revision: + nullable: true + type: string + serviceAccount: + nullable: true + type: string + targetNamespace: + nullable: true + type: string + targets: + items: + properties: + clusterGroup: + nullable: true + type: string + clusterGroupSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + clusterName: + nullable: true + type: string + clusterSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + name: + nullable: true + type: string + type: object + nullable: true + type: array + type: object + status: + properties: + commit: + nullable: true + type: string + conditions: + items: + properties: + lastTransitionTime: + nullable: true + type: string + lastUpdateTime: + nullable: true + type: string + message: + nullable: true + type: string + reason: + nullable: true + type: string + status: + nullable: true + type: string + type: + nullable: true + type: string + type: object + nullable: true + type: array + desiredReadyClusters: + type: integer + display: + properties: + error: + type: boolean + message: + nullable: true + type: string + readyBundleDeployments: + nullable: true + type: string + state: + nullable: true + type: string + type: object + gitJobStatus: + nullable: true + type: string + lastSyncedImageScanTime: + nullable: true + type: string + observedGeneration: + type: integer + readyClusters: + type: integer + resourceCounts: + properties: + desiredReady: + type: integer + missing: + type: integer + modified: + type: integer + notReady: + type: integer + orphaned: + type: integer + ready: + type: integer + unknown: + type: integer + waitApplied: + type: integer + type: object + resourceErrors: + items: + nullable: true + type: string + nullable: true + type: array + resources: + items: + properties: + apiVersion: + nullable: true + type: string + error: + type: boolean + id: + nullable: true + type: string + incompleteState: + type: boolean + kind: + nullable: true + type: string + message: + nullable: true + type: string + name: + nullable: true + type: string + namespace: + nullable: true + type: string + perClusterState: + items: + properties: + clusterId: + nullable: true + type: string + error: + type: boolean + message: + nullable: true + type: string + patch: + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + state: + nullable: true + type: string + transitioning: + type: boolean + type: object + nullable: true + type: array + state: + nullable: true + type: string + transitioning: + type: boolean + type: + nullable: true + type: string + type: object + nullable: true + type: array + summary: + properties: + desiredReady: + type: integer + errApplied: + type: integer + modified: + type: integer + nonReadyResources: + items: + properties: + bundleState: + nullable: true + type: string + message: + nullable: true + type: string + modifiedStatus: + items: + properties: + apiVersion: + nullable: true + type: string + delete: + type: boolean + kind: + nullable: true + type: string + missing: + type: boolean + name: + nullable: true + type: string + namespace: + nullable: true + type: string + patch: + nullable: true + type: string + type: object + nullable: true + type: array + name: + nullable: true + type: string + nonReadyStatus: + items: + properties: + apiVersion: + nullable: true + type: string + kind: + nullable: true + type: string + name: + nullable: true + type: string + namespace: + nullable: true + type: string + summary: + properties: + error: + type: boolean + message: + items: + nullable: true + type: string + nullable: true + type: array + state: + nullable: true + type: string + transitioning: + type: boolean + type: object + uid: + nullable: true + type: string + type: object + nullable: true + type: array + type: object + nullable: true + type: array + notReady: + type: integer + outOfSync: + type: integer + pending: + type: integer + ready: + type: integer + waitApplied: + type: integer + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: clusterregistrations.fleet.cattle.io +spec: + group: fleet.cattle.io + names: + kind: ClusterRegistration + plural: clusterregistrations + singular: clusterregistration + preserveUnknownFields: false + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.clusterName + name: Cluster-Name + type: string + - jsonPath: .spec.clusterLabels + name: Labels + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + properties: + spec: + properties: + clientID: + nullable: true + type: string + clientRandom: + nullable: true + type: string + clusterLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + status: + properties: + clusterName: + nullable: true + type: string + granted: + type: boolean + type: object + type: object + served: true + storage: true + subresources: + status: {} + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: gitreporestrictions.fleet.cattle.io +spec: + group: fleet.cattle.io + names: + kind: GitRepoRestriction + plural: gitreporestrictions + singular: gitreporestriction + preserveUnknownFields: false + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .defaultServiceAccount + name: Default-ServiceAccount + type: string + - jsonPath: .allowedServiceAccounts + name: Allowed-ServiceAccounts + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + properties: + allowedClientSecretNames: + items: + nullable: true + type: string + nullable: true + type: array + allowedRepoPatterns: + items: + nullable: true + type: string + nullable: true + type: array + allowedServiceAccounts: + items: + nullable: true + type: string + nullable: true + type: array + allowedTargetNamespaces: + items: + nullable: true + type: string + nullable: true + type: array + defaultClientSecretName: + nullable: true + type: string + defaultServiceAccount: + nullable: true + type: string + type: object + served: true + storage: true + subresources: + status: {} + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: contents.fleet.cattle.io +spec: + group: fleet.cattle.io + names: + kind: Content + plural: contents + singular: content + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + properties: + content: + nullable: true + type: string + type: object + served: true + storage: true + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: imagescans.fleet.cattle.io +spec: + group: fleet.cattle.io + names: + categories: + - fleet + kind: ImageScan + plural: imagescans + singular: imagescan + preserveUnknownFields: false + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.image + name: Repository + type: string + - jsonPath: .status.latestTag + name: Latest + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + properties: + spec: + properties: + gitrepoName: + nullable: true + type: string + image: + nullable: true + type: string + interval: + nullable: true + type: string + policy: + properties: + alphabetical: + nullable: true + properties: + order: + nullable: true + type: string + type: object + semver: + nullable: true + properties: + range: + nullable: true + type: string + type: object + type: object + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + type: object + suspend: + type: boolean + tagName: + nullable: true + type: string + type: object + status: + properties: + canonicalImageName: + nullable: true + type: string + conditions: + items: + properties: + lastTransitionTime: + nullable: true + type: string + lastUpdateTime: + nullable: true + type: string + message: + nullable: true + type: string + reason: + nullable: true + type: string + status: + nullable: true + type: string + type: + nullable: true + type: string + type: object + nullable: true + type: array + lastScanTime: + nullable: true + type: string + latestDigest: + nullable: true + type: string + latestImage: + nullable: true + type: string + latestTag: + nullable: true + type: string + observedGeneration: + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/fleet-crd/102.2.5+up0.8.5/templates/gitjobs-crds.yaml b/charts/fleet-crd/102.2.5+up0.8.5/templates/gitjobs-crds.yaml new file mode 100644 index 0000000000..bf6fb789e0 --- /dev/null +++ b/charts/fleet-crd/102.2.5+up0.8.5/templates/gitjobs-crds.yaml @@ -0,0 +1,7714 @@ +{{- if .Capabilities.APIVersions.Has "apiextensions.k8s.io/v1" -}} +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: gitjobs.gitjob.cattle.io +spec: + group: gitjob.cattle.io + names: + kind: GitJob + plural: gitjobs + singular: gitjob + preserveUnknownFields: false + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.git.repo + name: REPO + type: string + - jsonPath: .spec.git.branch + name: BRANCH + type: string + - jsonPath: .status.commit + name: COMMIT + type: string + - jsonPath: .status.jobStatus + name: JOBSTATUS + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + properties: + spec: + properties: + forceUpdateGeneration: + type: integer + git: + properties: + branch: + nullable: true + type: string + caBundle: + nullable: true + type: string + clientSecretName: + nullable: true + type: string + insecureSkipTLSVerify: + type: boolean + onTag: + nullable: true + type: string + provider: + nullable: true + type: string + repo: + nullable: true + type: string + revision: + nullable: true + type: string + type: object + jobSpec: + properties: + activeDeadlineSeconds: + nullable: true + type: integer + backoffLimit: + nullable: true + type: integer + completionMode: + nullable: true + type: string + completions: + nullable: true + type: integer + manualSelector: + nullable: true + type: boolean + parallelism: + nullable: true + type: integer + podFailurePolicy: + nullable: true + properties: + rules: + items: + properties: + action: + nullable: true + type: string + onExitCodes: + nullable: true + properties: + containerName: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + type: integer + nullable: true + type: array + type: object + onPodConditions: + items: + properties: + status: + nullable: true + type: string + type: + nullable: true + type: string + type: object + nullable: true + type: array + type: object + nullable: true + type: array + type: object + selector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + suspend: + nullable: true + type: boolean + template: + properties: + metadata: + properties: + annotations: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + creationTimestamp: + nullable: true + type: string + deletionGracePeriodSeconds: + nullable: true + type: integer + deletionTimestamp: + nullable: true + type: string + finalizers: + items: + nullable: true + type: string + nullable: true + type: array + generateName: + nullable: true + type: string + generation: + type: integer + labels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + managedFields: + items: + properties: + apiVersion: + nullable: true + type: string + fieldsType: + nullable: true + type: string + fieldsV1: + nullable: true + type: object + manager: + nullable: true + type: string + operation: + nullable: true + type: string + subresource: + nullable: true + type: string + time: + nullable: true + type: string + type: object + nullable: true + type: array + name: + nullable: true + type: string + namespace: + nullable: true + type: string + ownerReferences: + items: + properties: + apiVersion: + nullable: true + type: string + blockOwnerDeletion: + nullable: true + type: boolean + controller: + nullable: true + type: boolean + kind: + nullable: true + type: string + name: + nullable: true + type: string + uid: + nullable: true + type: string + type: object + nullable: true + type: array + resourceVersion: + nullable: true + type: string + selfLink: + nullable: true + type: string + uid: + nullable: true + type: string + type: object + spec: + properties: + activeDeadlineSeconds: + nullable: true + type: integer + affinity: + nullable: true + properties: + nodeAffinity: + nullable: true + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchFields: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + type: object + weight: + type: integer + type: object + nullable: true + type: array + requiredDuringSchedulingIgnoredDuringExecution: + nullable: true + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchFields: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + type: object + nullable: true + type: array + type: object + type: object + podAffinity: + nullable: true + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + namespaceSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + namespaces: + items: + nullable: true + type: string + nullable: true + type: array + topologyKey: + nullable: true + type: string + type: object + weight: + type: integer + type: object + nullable: true + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + namespaceSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + namespaces: + items: + nullable: true + type: string + nullable: true + type: array + topologyKey: + nullable: true + type: string + type: object + nullable: true + type: array + type: object + podAntiAffinity: + nullable: true + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + namespaceSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + namespaces: + items: + nullable: true + type: string + nullable: true + type: array + topologyKey: + nullable: true + type: string + type: object + weight: + type: integer + type: object + nullable: true + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + namespaceSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + namespaces: + items: + nullable: true + type: string + nullable: true + type: array + topologyKey: + nullable: true + type: string + type: object + nullable: true + type: array + type: object + type: object + automountServiceAccountToken: + nullable: true + type: boolean + containers: + items: + properties: + args: + items: + nullable: true + type: string + nullable: true + type: array + command: + items: + nullable: true + type: string + nullable: true + type: array + env: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + valueFrom: + nullable: true + properties: + configMapKeyRef: + nullable: true + properties: + key: + nullable: true + type: string + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + fieldRef: + nullable: true + properties: + apiVersion: + nullable: true + type: string + fieldPath: + nullable: true + type: string + type: object + resourceFieldRef: + nullable: true + properties: + containerName: + nullable: true + type: string + divisor: + nullable: true + type: string + resource: + nullable: true + type: string + type: object + secretKeyRef: + nullable: true + properties: + key: + nullable: true + type: string + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + type: object + type: object + nullable: true + type: array + envFrom: + items: + properties: + configMapRef: + nullable: true + properties: + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + prefix: + nullable: true + type: string + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + type: object + nullable: true + type: array + image: + nullable: true + type: string + imagePullPolicy: + nullable: true + type: string + lifecycle: + nullable: true + properties: + postStart: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + nullable: true + type: string + scheme: + nullable: true + type: string + type: object + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + nullable: true + type: string + type: object + type: object + preStop: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + nullable: true + type: string + scheme: + nullable: true + type: string + type: object + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + nullable: true + type: string + type: object + type: object + type: object + livenessProbe: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + failureThreshold: + type: integer + grpc: + nullable: true + properties: + port: + type: integer + service: + nullable: true + type: string + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + nullable: true + type: string + scheme: + nullable: true + type: string + type: object + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + nullable: true + type: string + type: object + terminationGracePeriodSeconds: + nullable: true + type: integer + timeoutSeconds: + type: integer + type: object + name: + nullable: true + type: string + ports: + items: + properties: + containerPort: + type: integer + hostIP: + nullable: true + type: string + hostPort: + type: integer + name: + nullable: true + type: string + protocol: + nullable: true + type: string + type: object + nullable: true + type: array + readinessProbe: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + failureThreshold: + type: integer + grpc: + nullable: true + properties: + port: + type: integer + service: + nullable: true + type: string + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + nullable: true + type: string + scheme: + nullable: true + type: string + type: object + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + nullable: true + type: string + type: object + terminationGracePeriodSeconds: + nullable: true + type: integer + timeoutSeconds: + type: integer + type: object + resizePolicy: + items: + properties: + resourceName: + nullable: true + type: string + restartPolicy: + nullable: true + type: string + type: object + nullable: true + type: array + resources: + properties: + claims: + items: + properties: + name: + nullable: true + type: string + type: object + nullable: true + type: array + limits: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + requests: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + securityContext: + nullable: true + properties: + allowPrivilegeEscalation: + nullable: true + type: boolean + capabilities: + nullable: true + properties: + add: + items: + nullable: true + type: string + nullable: true + type: array + drop: + items: + nullable: true + type: string + nullable: true + type: array + type: object + privileged: + nullable: true + type: boolean + procMount: + nullable: true + type: string + readOnlyRootFilesystem: + nullable: true + type: boolean + runAsGroup: + nullable: true + type: integer + runAsNonRoot: + nullable: true + type: boolean + runAsUser: + nullable: true + type: integer + seLinuxOptions: + nullable: true + properties: + level: + nullable: true + type: string + role: + nullable: true + type: string + type: + nullable: true + type: string + user: + nullable: true + type: string + type: object + seccompProfile: + nullable: true + properties: + localhostProfile: + nullable: true + type: string + type: + nullable: true + type: string + type: object + windowsOptions: + nullable: true + properties: + gmsaCredentialSpec: + nullable: true + type: string + gmsaCredentialSpecName: + nullable: true + type: string + hostProcess: + nullable: true + type: boolean + runAsUserName: + nullable: true + type: string + type: object + type: object + startupProbe: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + failureThreshold: + type: integer + grpc: + nullable: true + properties: + port: + type: integer + service: + nullable: true + type: string + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + nullable: true + type: string + scheme: + nullable: true + type: string + type: object + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + nullable: true + type: string + type: object + terminationGracePeriodSeconds: + nullable: true + type: integer + timeoutSeconds: + type: integer + type: object + stdin: + type: boolean + stdinOnce: + type: boolean + terminationMessagePath: + nullable: true + type: string + terminationMessagePolicy: + nullable: true + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + nullable: true + type: string + name: + nullable: true + type: string + type: object + nullable: true + type: array + volumeMounts: + items: + properties: + mountPath: + nullable: true + type: string + mountPropagation: + nullable: true + type: string + name: + nullable: true + type: string + readOnly: + type: boolean + subPath: + nullable: true + type: string + subPathExpr: + nullable: true + type: string + type: object + nullable: true + type: array + workingDir: + nullable: true + type: string + type: object + nullable: true + type: array + dnsConfig: + nullable: true + properties: + nameservers: + items: + nullable: true + type: string + nullable: true + type: array + options: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + searches: + items: + nullable: true + type: string + nullable: true + type: array + type: object + dnsPolicy: + nullable: true + type: string + enableServiceLinks: + nullable: true + type: boolean + ephemeralContainers: + items: + properties: + args: + items: + nullable: true + type: string + nullable: true + type: array + command: + items: + nullable: true + type: string + nullable: true + type: array + env: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + valueFrom: + nullable: true + properties: + configMapKeyRef: + nullable: true + properties: + key: + nullable: true + type: string + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + fieldRef: + nullable: true + properties: + apiVersion: + nullable: true + type: string + fieldPath: + nullable: true + type: string + type: object + resourceFieldRef: + nullable: true + properties: + containerName: + nullable: true + type: string + divisor: + nullable: true + type: string + resource: + nullable: true + type: string + type: object + secretKeyRef: + nullable: true + properties: + key: + nullable: true + type: string + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + type: object + type: object + nullable: true + type: array + envFrom: + items: + properties: + configMapRef: + nullable: true + properties: + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + prefix: + nullable: true + type: string + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + type: object + nullable: true + type: array + image: + nullable: true + type: string + imagePullPolicy: + nullable: true + type: string + lifecycle: + nullable: true + properties: + postStart: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + nullable: true + type: string + scheme: + nullable: true + type: string + type: object + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + nullable: true + type: string + type: object + type: object + preStop: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + nullable: true + type: string + scheme: + nullable: true + type: string + type: object + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + nullable: true + type: string + type: object + type: object + type: object + livenessProbe: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + failureThreshold: + type: integer + grpc: + nullable: true + properties: + port: + type: integer + service: + nullable: true + type: string + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + nullable: true + type: string + scheme: + nullable: true + type: string + type: object + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + nullable: true + type: string + type: object + terminationGracePeriodSeconds: + nullable: true + type: integer + timeoutSeconds: + type: integer + type: object + name: + nullable: true + type: string + ports: + items: + properties: + containerPort: + type: integer + hostIP: + nullable: true + type: string + hostPort: + type: integer + name: + nullable: true + type: string + protocol: + nullable: true + type: string + type: object + nullable: true + type: array + readinessProbe: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + failureThreshold: + type: integer + grpc: + nullable: true + properties: + port: + type: integer + service: + nullable: true + type: string + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + nullable: true + type: string + scheme: + nullable: true + type: string + type: object + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + nullable: true + type: string + type: object + terminationGracePeriodSeconds: + nullable: true + type: integer + timeoutSeconds: + type: integer + type: object + resizePolicy: + items: + properties: + resourceName: + nullable: true + type: string + restartPolicy: + nullable: true + type: string + type: object + nullable: true + type: array + resources: + properties: + claims: + items: + properties: + name: + nullable: true + type: string + type: object + nullable: true + type: array + limits: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + requests: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + securityContext: + nullable: true + properties: + allowPrivilegeEscalation: + nullable: true + type: boolean + capabilities: + nullable: true + properties: + add: + items: + nullable: true + type: string + nullable: true + type: array + drop: + items: + nullable: true + type: string + nullable: true + type: array + type: object + privileged: + nullable: true + type: boolean + procMount: + nullable: true + type: string + readOnlyRootFilesystem: + nullable: true + type: boolean + runAsGroup: + nullable: true + type: integer + runAsNonRoot: + nullable: true + type: boolean + runAsUser: + nullable: true + type: integer + seLinuxOptions: + nullable: true + properties: + level: + nullable: true + type: string + role: + nullable: true + type: string + type: + nullable: true + type: string + user: + nullable: true + type: string + type: object + seccompProfile: + nullable: true + properties: + localhostProfile: + nullable: true + type: string + type: + nullable: true + type: string + type: object + windowsOptions: + nullable: true + properties: + gmsaCredentialSpec: + nullable: true + type: string + gmsaCredentialSpecName: + nullable: true + type: string + hostProcess: + nullable: true + type: boolean + runAsUserName: + nullable: true + type: string + type: object + type: object + startupProbe: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + failureThreshold: + type: integer + grpc: + nullable: true + properties: + port: + type: integer + service: + nullable: true + type: string + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + nullable: true + type: string + scheme: + nullable: true + type: string + type: object + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + nullable: true + type: string + type: object + terminationGracePeriodSeconds: + nullable: true + type: integer + timeoutSeconds: + type: integer + type: object + stdin: + type: boolean + stdinOnce: + type: boolean + targetContainerName: + nullable: true + type: string + terminationMessagePath: + nullable: true + type: string + terminationMessagePolicy: + nullable: true + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + nullable: true + type: string + name: + nullable: true + type: string + type: object + nullable: true + type: array + volumeMounts: + items: + properties: + mountPath: + nullable: true + type: string + mountPropagation: + nullable: true + type: string + name: + nullable: true + type: string + readOnly: + type: boolean + subPath: + nullable: true + type: string + subPathExpr: + nullable: true + type: string + type: object + nullable: true + type: array + workingDir: + nullable: true + type: string + type: object + nullable: true + type: array + hostAliases: + items: + properties: + hostnames: + items: + nullable: true + type: string + nullable: true + type: array + ip: + nullable: true + type: string + type: object + nullable: true + type: array + hostIPC: + type: boolean + hostNetwork: + type: boolean + hostPID: + type: boolean + hostUsers: + nullable: true + type: boolean + hostname: + nullable: true + type: string + imagePullSecrets: + items: + properties: + name: + nullable: true + type: string + type: object + nullable: true + type: array + initContainers: + items: + properties: + args: + items: + nullable: true + type: string + nullable: true + type: array + command: + items: + nullable: true + type: string + nullable: true + type: array + env: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + valueFrom: + nullable: true + properties: + configMapKeyRef: + nullable: true + properties: + key: + nullable: true + type: string + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + fieldRef: + nullable: true + properties: + apiVersion: + nullable: true + type: string + fieldPath: + nullable: true + type: string + type: object + resourceFieldRef: + nullable: true + properties: + containerName: + nullable: true + type: string + divisor: + nullable: true + type: string + resource: + nullable: true + type: string + type: object + secretKeyRef: + nullable: true + properties: + key: + nullable: true + type: string + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + type: object + type: object + nullable: true + type: array + envFrom: + items: + properties: + configMapRef: + nullable: true + properties: + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + prefix: + nullable: true + type: string + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + type: object + nullable: true + type: array + image: + nullable: true + type: string + imagePullPolicy: + nullable: true + type: string + lifecycle: + nullable: true + properties: + postStart: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + nullable: true + type: string + scheme: + nullable: true + type: string + type: object + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + nullable: true + type: string + type: object + type: object + preStop: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + nullable: true + type: string + scheme: + nullable: true + type: string + type: object + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + nullable: true + type: string + type: object + type: object + type: object + livenessProbe: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + failureThreshold: + type: integer + grpc: + nullable: true + properties: + port: + type: integer + service: + nullable: true + type: string + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + nullable: true + type: string + scheme: + nullable: true + type: string + type: object + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + nullable: true + type: string + type: object + terminationGracePeriodSeconds: + nullable: true + type: integer + timeoutSeconds: + type: integer + type: object + name: + nullable: true + type: string + ports: + items: + properties: + containerPort: + type: integer + hostIP: + nullable: true + type: string + hostPort: + type: integer + name: + nullable: true + type: string + protocol: + nullable: true + type: string + type: object + nullable: true + type: array + readinessProbe: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + failureThreshold: + type: integer + grpc: + nullable: true + properties: + port: + type: integer + service: + nullable: true + type: string + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + nullable: true + type: string + scheme: + nullable: true + type: string + type: object + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + nullable: true + type: string + type: object + terminationGracePeriodSeconds: + nullable: true + type: integer + timeoutSeconds: + type: integer + type: object + resizePolicy: + items: + properties: + resourceName: + nullable: true + type: string + restartPolicy: + nullable: true + type: string + type: object + nullable: true + type: array + resources: + properties: + claims: + items: + properties: + name: + nullable: true + type: string + type: object + nullable: true + type: array + limits: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + requests: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + securityContext: + nullable: true + properties: + allowPrivilegeEscalation: + nullable: true + type: boolean + capabilities: + nullable: true + properties: + add: + items: + nullable: true + type: string + nullable: true + type: array + drop: + items: + nullable: true + type: string + nullable: true + type: array + type: object + privileged: + nullable: true + type: boolean + procMount: + nullable: true + type: string + readOnlyRootFilesystem: + nullable: true + type: boolean + runAsGroup: + nullable: true + type: integer + runAsNonRoot: + nullable: true + type: boolean + runAsUser: + nullable: true + type: integer + seLinuxOptions: + nullable: true + properties: + level: + nullable: true + type: string + role: + nullable: true + type: string + type: + nullable: true + type: string + user: + nullable: true + type: string + type: object + seccompProfile: + nullable: true + properties: + localhostProfile: + nullable: true + type: string + type: + nullable: true + type: string + type: object + windowsOptions: + nullable: true + properties: + gmsaCredentialSpec: + nullable: true + type: string + gmsaCredentialSpecName: + nullable: true + type: string + hostProcess: + nullable: true + type: boolean + runAsUserName: + nullable: true + type: string + type: object + type: object + startupProbe: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + failureThreshold: + type: integer + grpc: + nullable: true + properties: + port: + type: integer + service: + nullable: true + type: string + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + nullable: true + type: string + scheme: + nullable: true + type: string + type: object + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + nullable: true + type: string + type: object + terminationGracePeriodSeconds: + nullable: true + type: integer + timeoutSeconds: + type: integer + type: object + stdin: + type: boolean + stdinOnce: + type: boolean + terminationMessagePath: + nullable: true + type: string + terminationMessagePolicy: + nullable: true + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + nullable: true + type: string + name: + nullable: true + type: string + type: object + nullable: true + type: array + volumeMounts: + items: + properties: + mountPath: + nullable: true + type: string + mountPropagation: + nullable: true + type: string + name: + nullable: true + type: string + readOnly: + type: boolean + subPath: + nullable: true + type: string + subPathExpr: + nullable: true + type: string + type: object + nullable: true + type: array + workingDir: + nullable: true + type: string + type: object + nullable: true + type: array + nodeName: + nullable: true + type: string + nodeSelector: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + os: + nullable: true + properties: + name: + nullable: true + type: string + type: object + overhead: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + preemptionPolicy: + nullable: true + type: string + priority: + nullable: true + type: integer + priorityClassName: + nullable: true + type: string + readinessGates: + items: + properties: + conditionType: + nullable: true + type: string + type: object + nullable: true + type: array + resourceClaims: + items: + properties: + name: + nullable: true + type: string + source: + properties: + resourceClaimName: + nullable: true + type: string + resourceClaimTemplateName: + nullable: true + type: string + type: object + type: object + nullable: true + type: array + restartPolicy: + nullable: true + type: string + runtimeClassName: + nullable: true + type: string + schedulerName: + nullable: true + type: string + schedulingGates: + items: + properties: + name: + nullable: true + type: string + type: object + nullable: true + type: array + securityContext: + nullable: true + properties: + fsGroup: + nullable: true + type: integer + fsGroupChangePolicy: + nullable: true + type: string + runAsGroup: + nullable: true + type: integer + runAsNonRoot: + nullable: true + type: boolean + runAsUser: + nullable: true + type: integer + seLinuxOptions: + nullable: true + properties: + level: + nullable: true + type: string + role: + nullable: true + type: string + type: + nullable: true + type: string + user: + nullable: true + type: string + type: object + seccompProfile: + nullable: true + properties: + localhostProfile: + nullable: true + type: string + type: + nullable: true + type: string + type: object + supplementalGroups: + items: + type: integer + nullable: true + type: array + sysctls: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + windowsOptions: + nullable: true + properties: + gmsaCredentialSpec: + nullable: true + type: string + gmsaCredentialSpecName: + nullable: true + type: string + hostProcess: + nullable: true + type: boolean + runAsUserName: + nullable: true + type: string + type: object + type: object + serviceAccount: + nullable: true + type: string + serviceAccountName: + nullable: true + type: string + setHostnameAsFQDN: + nullable: true + type: boolean + shareProcessNamespace: + nullable: true + type: boolean + subdomain: + nullable: true + type: string + terminationGracePeriodSeconds: + nullable: true + type: integer + tolerations: + items: + properties: + effect: + nullable: true + type: string + key: + nullable: true + type: string + operator: + nullable: true + type: string + tolerationSeconds: + nullable: true + type: integer + value: + nullable: true + type: string + type: object + nullable: true + type: array + topologySpreadConstraints: + items: + properties: + labelSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + matchLabelKeys: + items: + nullable: true + type: string + nullable: true + type: array + maxSkew: + type: integer + minDomains: + nullable: true + type: integer + nodeAffinityPolicy: + nullable: true + type: string + nodeTaintsPolicy: + nullable: true + type: string + topologyKey: + nullable: true + type: string + whenUnsatisfiable: + nullable: true + type: string + type: object + nullable: true + type: array + volumes: + items: + properties: + awsElasticBlockStore: + nullable: true + properties: + fsType: + nullable: true + type: string + partition: + type: integer + readOnly: + type: boolean + volumeID: + nullable: true + type: string + type: object + azureDisk: + nullable: true + properties: + cachingMode: + nullable: true + type: string + diskName: + nullable: true + type: string + diskURI: + nullable: true + type: string + fsType: + nullable: true + type: string + kind: + nullable: true + type: string + readOnly: + nullable: true + type: boolean + type: object + azureFile: + nullable: true + properties: + readOnly: + type: boolean + secretName: + nullable: true + type: string + shareName: + nullable: true + type: string + type: object + cephfs: + nullable: true + properties: + monitors: + items: + nullable: true + type: string + nullable: true + type: array + path: + nullable: true + type: string + readOnly: + type: boolean + secretFile: + nullable: true + type: string + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + type: object + user: + nullable: true + type: string + type: object + cinder: + nullable: true + properties: + fsType: + nullable: true + type: string + readOnly: + type: boolean + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + type: object + volumeID: + nullable: true + type: string + type: object + configMap: + nullable: true + properties: + defaultMode: + nullable: true + type: integer + items: + items: + properties: + key: + nullable: true + type: string + mode: + nullable: true + type: integer + path: + nullable: true + type: string + type: object + nullable: true + type: array + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + csi: + nullable: true + properties: + driver: + nullable: true + type: string + fsType: + nullable: true + type: string + nodePublishSecretRef: + nullable: true + properties: + name: + nullable: true + type: string + type: object + readOnly: + nullable: true + type: boolean + volumeAttributes: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + downwardAPI: + nullable: true + properties: + defaultMode: + nullable: true + type: integer + items: + items: + properties: + fieldRef: + nullable: true + properties: + apiVersion: + nullable: true + type: string + fieldPath: + nullable: true + type: string + type: object + mode: + nullable: true + type: integer + path: + nullable: true + type: string + resourceFieldRef: + nullable: true + properties: + containerName: + nullable: true + type: string + divisor: + nullable: true + type: string + resource: + nullable: true + type: string + type: object + type: object + nullable: true + type: array + type: object + emptyDir: + nullable: true + properties: + medium: + nullable: true + type: string + sizeLimit: + nullable: true + type: string + type: object + ephemeral: + nullable: true + properties: + volumeClaimTemplate: + nullable: true + properties: + metadata: + properties: + annotations: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + creationTimestamp: + nullable: true + type: string + deletionGracePeriodSeconds: + nullable: true + type: integer + deletionTimestamp: + nullable: true + type: string + finalizers: + items: + nullable: true + type: string + nullable: true + type: array + generateName: + nullable: true + type: string + generation: + type: integer + labels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + managedFields: + items: + properties: + apiVersion: + nullable: true + type: string + fieldsType: + nullable: true + type: string + fieldsV1: + nullable: true + type: object + manager: + nullable: true + type: string + operation: + nullable: true + type: string + subresource: + nullable: true + type: string + time: + nullable: true + type: string + type: object + nullable: true + type: array + name: + nullable: true + type: string + namespace: + nullable: true + type: string + ownerReferences: + items: + properties: + apiVersion: + nullable: true + type: string + blockOwnerDeletion: + nullable: true + type: boolean + controller: + nullable: true + type: boolean + kind: + nullable: true + type: string + name: + nullable: true + type: string + uid: + nullable: true + type: string + type: object + nullable: true + type: array + resourceVersion: + nullable: true + type: string + selfLink: + nullable: true + type: string + uid: + nullable: true + type: string + type: object + spec: + properties: + accessModes: + items: + nullable: true + type: string + nullable: true + type: array + dataSource: + nullable: true + properties: + apiGroup: + nullable: true + type: string + kind: + nullable: true + type: string + name: + nullable: true + type: string + type: object + dataSourceRef: + nullable: true + properties: + apiGroup: + nullable: true + type: string + kind: + nullable: true + type: string + name: + nullable: true + type: string + namespace: + nullable: true + type: string + type: object + resources: + properties: + claims: + items: + properties: + name: + nullable: true + type: string + type: object + nullable: true + type: array + limits: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + requests: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + selector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + storageClassName: + nullable: true + type: string + volumeMode: + nullable: true + type: string + volumeName: + nullable: true + type: string + type: object + type: object + type: object + fc: + nullable: true + properties: + fsType: + nullable: true + type: string + lun: + nullable: true + type: integer + readOnly: + type: boolean + targetWWNs: + items: + nullable: true + type: string + nullable: true + type: array + wwids: + items: + nullable: true + type: string + nullable: true + type: array + type: object + flexVolume: + nullable: true + properties: + driver: + nullable: true + type: string + fsType: + nullable: true + type: string + options: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + readOnly: + type: boolean + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + type: object + type: object + flocker: + nullable: true + properties: + datasetName: + nullable: true + type: string + datasetUUID: + nullable: true + type: string + type: object + gcePersistentDisk: + nullable: true + properties: + fsType: + nullable: true + type: string + partition: + type: integer + pdName: + nullable: true + type: string + readOnly: + type: boolean + type: object + gitRepo: + nullable: true + properties: + directory: + nullable: true + type: string + repository: + nullable: true + type: string + revision: + nullable: true + type: string + type: object + glusterfs: + nullable: true + properties: + endpoints: + nullable: true + type: string + path: + nullable: true + type: string + readOnly: + type: boolean + type: object + hostPath: + nullable: true + properties: + path: + nullable: true + type: string + type: + nullable: true + type: string + type: object + iscsi: + nullable: true + properties: + chapAuthDiscovery: + type: boolean + chapAuthSession: + type: boolean + fsType: + nullable: true + type: string + initiatorName: + nullable: true + type: string + iqn: + nullable: true + type: string + iscsiInterface: + nullable: true + type: string + lun: + type: integer + portals: + items: + nullable: true + type: string + nullable: true + type: array + readOnly: + type: boolean + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + type: object + targetPortal: + nullable: true + type: string + type: object + name: + nullable: true + type: string + nfs: + nullable: true + properties: + path: + nullable: true + type: string + readOnly: + type: boolean + server: + nullable: true + type: string + type: object + persistentVolumeClaim: + nullable: true + properties: + claimName: + nullable: true + type: string + readOnly: + type: boolean + type: object + photonPersistentDisk: + nullable: true + properties: + fsType: + nullable: true + type: string + pdID: + nullable: true + type: string + type: object + portworxVolume: + nullable: true + properties: + fsType: + nullable: true + type: string + readOnly: + type: boolean + volumeID: + nullable: true + type: string + type: object + projected: + nullable: true + properties: + defaultMode: + nullable: true + type: integer + sources: + items: + properties: + configMap: + nullable: true + properties: + items: + items: + properties: + key: + nullable: true + type: string + mode: + nullable: true + type: integer + path: + nullable: true + type: string + type: object + nullable: true + type: array + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + downwardAPI: + nullable: true + properties: + items: + items: + properties: + fieldRef: + nullable: true + properties: + apiVersion: + nullable: true + type: string + fieldPath: + nullable: true + type: string + type: object + mode: + nullable: true + type: integer + path: + nullable: true + type: string + resourceFieldRef: + nullable: true + properties: + containerName: + nullable: true + type: string + divisor: + nullable: true + type: string + resource: + nullable: true + type: string + type: object + type: object + nullable: true + type: array + type: object + secret: + nullable: true + properties: + items: + items: + properties: + key: + nullable: true + type: string + mode: + nullable: true + type: integer + path: + nullable: true + type: string + type: object + nullable: true + type: array + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + serviceAccountToken: + nullable: true + properties: + audience: + nullable: true + type: string + expirationSeconds: + nullable: true + type: integer + path: + nullable: true + type: string + type: object + type: object + nullable: true + type: array + type: object + quobyte: + nullable: true + properties: + group: + nullable: true + type: string + readOnly: + type: boolean + registry: + nullable: true + type: string + tenant: + nullable: true + type: string + user: + nullable: true + type: string + volume: + nullable: true + type: string + type: object + rbd: + nullable: true + properties: + fsType: + nullable: true + type: string + image: + nullable: true + type: string + keyring: + nullable: true + type: string + monitors: + items: + nullable: true + type: string + nullable: true + type: array + pool: + nullable: true + type: string + readOnly: + type: boolean + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + type: object + user: + nullable: true + type: string + type: object + scaleIO: + nullable: true + properties: + fsType: + nullable: true + type: string + gateway: + nullable: true + type: string + protectionDomain: + nullable: true + type: string + readOnly: + type: boolean + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + type: object + sslEnabled: + type: boolean + storageMode: + nullable: true + type: string + storagePool: + nullable: true + type: string + system: + nullable: true + type: string + volumeName: + nullable: true + type: string + type: object + secret: + nullable: true + properties: + defaultMode: + nullable: true + type: integer + items: + items: + properties: + key: + nullable: true + type: string + mode: + nullable: true + type: integer + path: + nullable: true + type: string + type: object + nullable: true + type: array + optional: + nullable: true + type: boolean + secretName: + nullable: true + type: string + type: object + storageos: + nullable: true + properties: + fsType: + nullable: true + type: string + readOnly: + type: boolean + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + type: object + volumeName: + nullable: true + type: string + volumeNamespace: + nullable: true + type: string + type: object + vsphereVolume: + nullable: true + properties: + fsType: + nullable: true + type: string + storagePolicyID: + nullable: true + type: string + storagePolicyName: + nullable: true + type: string + volumePath: + nullable: true + type: string + type: object + type: object + nullable: true + type: array + type: object + type: object + ttlSecondsAfterFinished: + nullable: true + type: integer + type: object + syncInterval: + type: integer + type: object + status: + properties: + commit: + nullable: true + type: string + conditions: + items: + properties: + lastTransitionTime: + nullable: true + type: string + lastUpdateTime: + nullable: true + type: string + message: + nullable: true + type: string + reason: + nullable: true + type: string + status: + nullable: true + type: string + type: + nullable: true + type: string + type: object + nullable: true + type: array + event: + nullable: true + type: string + hookId: + nullable: true + type: string + jobStatus: + nullable: true + type: string + lastExecutedCommit: + nullable: true + type: string + lastSyncedTime: + nullable: true + type: string + observedGeneration: + type: integer + secretToken: + nullable: true + type: string + updateGeneration: + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} +{{- else -}} +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: gitjobs.gitjob.cattle.io +spec: + additionalPrinterColumns: + - JSONPath: .spec.git.repo + name: REPO + type: string + - JSONPath: .spec.git.branch + name: BRANCH + type: string + - JSONPath: .status.commit + name: COMMIT + type: string + - JSONPath: .status.jobStatus + name: JOBSTATUS + type: string + - JSONPath: .metadata.creationTimestamp + name: Age + type: date + group: gitjob.cattle.io + names: + kind: GitJob + plural: gitjobs + singular: gitjob + preserveUnknownFields: false + scope: Namespaced + subresources: + status: {} + validation: + openAPIV3Schema: + properties: + spec: + properties: + forceUpdateGeneration: + type: integer + git: + properties: + branch: + nullable: true + type: string + caBundle: + nullable: true + type: string + clientSecretName: + nullable: true + type: string + insecureSkipTLSVerify: + type: boolean + onTag: + nullable: true + type: string + provider: + nullable: true + type: string + repo: + nullable: true + type: string + revision: + nullable: true + type: string + type: object + jobSpec: + properties: + activeDeadlineSeconds: + nullable: true + type: integer + backoffLimit: + nullable: true + type: integer + completionMode: + nullable: true + type: string + completions: + nullable: true + type: integer + manualSelector: + nullable: true + type: boolean + parallelism: + nullable: true + type: integer + podFailurePolicy: + nullable: true + properties: + rules: + items: + properties: + action: + nullable: true + type: string + onExitCodes: + nullable: true + properties: + containerName: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + type: integer + nullable: true + type: array + type: object + onPodConditions: + items: + properties: + status: + nullable: true + type: string + type: + nullable: true + type: string + type: object + nullable: true + type: array + type: object + nullable: true + type: array + type: object + selector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + suspend: + nullable: true + type: boolean + template: + properties: + metadata: + properties: + annotations: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + creationTimestamp: + nullable: true + type: string + deletionGracePeriodSeconds: + nullable: true + type: integer + deletionTimestamp: + nullable: true + type: string + finalizers: + items: + nullable: true + type: string + nullable: true + type: array + generateName: + nullable: true + type: string + generation: + type: integer + labels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + managedFields: + items: + properties: + apiVersion: + nullable: true + type: string + fieldsType: + nullable: true + type: string + fieldsV1: + nullable: true + type: object + manager: + nullable: true + type: string + operation: + nullable: true + type: string + subresource: + nullable: true + type: string + time: + nullable: true + type: string + type: object + nullable: true + type: array + name: + nullable: true + type: string + namespace: + nullable: true + type: string + ownerReferences: + items: + properties: + apiVersion: + nullable: true + type: string + blockOwnerDeletion: + nullable: true + type: boolean + controller: + nullable: true + type: boolean + kind: + nullable: true + type: string + name: + nullable: true + type: string + uid: + nullable: true + type: string + type: object + nullable: true + type: array + resourceVersion: + nullable: true + type: string + selfLink: + nullable: true + type: string + uid: + nullable: true + type: string + type: object + spec: + properties: + activeDeadlineSeconds: + nullable: true + type: integer + affinity: + nullable: true + properties: + nodeAffinity: + nullable: true + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchFields: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + type: object + weight: + type: integer + type: object + nullable: true + type: array + requiredDuringSchedulingIgnoredDuringExecution: + nullable: true + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchFields: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + type: object + nullable: true + type: array + type: object + type: object + podAffinity: + nullable: true + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + namespaceSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + namespaces: + items: + nullable: true + type: string + nullable: true + type: array + topologyKey: + nullable: true + type: string + type: object + weight: + type: integer + type: object + nullable: true + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + namespaceSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + namespaces: + items: + nullable: true + type: string + nullable: true + type: array + topologyKey: + nullable: true + type: string + type: object + nullable: true + type: array + type: object + podAntiAffinity: + nullable: true + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + namespaceSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + namespaces: + items: + nullable: true + type: string + nullable: true + type: array + topologyKey: + nullable: true + type: string + type: object + weight: + type: integer + type: object + nullable: true + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + namespaceSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + namespaces: + items: + nullable: true + type: string + nullable: true + type: array + topologyKey: + nullable: true + type: string + type: object + nullable: true + type: array + type: object + type: object + automountServiceAccountToken: + nullable: true + type: boolean + containers: + items: + properties: + args: + items: + nullable: true + type: string + nullable: true + type: array + command: + items: + nullable: true + type: string + nullable: true + type: array + env: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + valueFrom: + nullable: true + properties: + configMapKeyRef: + nullable: true + properties: + key: + nullable: true + type: string + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + fieldRef: + nullable: true + properties: + apiVersion: + nullable: true + type: string + fieldPath: + nullable: true + type: string + type: object + resourceFieldRef: + nullable: true + properties: + containerName: + nullable: true + type: string + divisor: + nullable: true + type: string + resource: + nullable: true + type: string + type: object + secretKeyRef: + nullable: true + properties: + key: + nullable: true + type: string + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + type: object + type: object + nullable: true + type: array + envFrom: + items: + properties: + configMapRef: + nullable: true + properties: + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + prefix: + nullable: true + type: string + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + type: object + nullable: true + type: array + image: + nullable: true + type: string + imagePullPolicy: + nullable: true + type: string + lifecycle: + nullable: true + properties: + postStart: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + nullable: true + type: string + scheme: + nullable: true + type: string + type: object + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + nullable: true + type: string + type: object + type: object + preStop: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + nullable: true + type: string + scheme: + nullable: true + type: string + type: object + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + nullable: true + type: string + type: object + type: object + type: object + livenessProbe: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + failureThreshold: + type: integer + grpc: + nullable: true + properties: + port: + type: integer + service: + nullable: true + type: string + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + nullable: true + type: string + scheme: + nullable: true + type: string + type: object + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + nullable: true + type: string + type: object + terminationGracePeriodSeconds: + nullable: true + type: integer + timeoutSeconds: + type: integer + type: object + name: + nullable: true + type: string + ports: + items: + properties: + containerPort: + type: integer + hostIP: + nullable: true + type: string + hostPort: + type: integer + name: + nullable: true + type: string + protocol: + nullable: true + type: string + type: object + nullable: true + type: array + readinessProbe: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + failureThreshold: + type: integer + grpc: + nullable: true + properties: + port: + type: integer + service: + nullable: true + type: string + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + nullable: true + type: string + scheme: + nullable: true + type: string + type: object + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + nullable: true + type: string + type: object + terminationGracePeriodSeconds: + nullable: true + type: integer + timeoutSeconds: + type: integer + type: object + resizePolicy: + items: + properties: + resourceName: + nullable: true + type: string + restartPolicy: + nullable: true + type: string + type: object + nullable: true + type: array + resources: + properties: + claims: + items: + properties: + name: + nullable: true + type: string + type: object + nullable: true + type: array + limits: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + requests: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + securityContext: + nullable: true + properties: + allowPrivilegeEscalation: + nullable: true + type: boolean + capabilities: + nullable: true + properties: + add: + items: + nullable: true + type: string + nullable: true + type: array + drop: + items: + nullable: true + type: string + nullable: true + type: array + type: object + privileged: + nullable: true + type: boolean + procMount: + nullable: true + type: string + readOnlyRootFilesystem: + nullable: true + type: boolean + runAsGroup: + nullable: true + type: integer + runAsNonRoot: + nullable: true + type: boolean + runAsUser: + nullable: true + type: integer + seLinuxOptions: + nullable: true + properties: + level: + nullable: true + type: string + role: + nullable: true + type: string + type: + nullable: true + type: string + user: + nullable: true + type: string + type: object + seccompProfile: + nullable: true + properties: + localhostProfile: + nullable: true + type: string + type: + nullable: true + type: string + type: object + windowsOptions: + nullable: true + properties: + gmsaCredentialSpec: + nullable: true + type: string + gmsaCredentialSpecName: + nullable: true + type: string + hostProcess: + nullable: true + type: boolean + runAsUserName: + nullable: true + type: string + type: object + type: object + startupProbe: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + failureThreshold: + type: integer + grpc: + nullable: true + properties: + port: + type: integer + service: + nullable: true + type: string + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + nullable: true + type: string + scheme: + nullable: true + type: string + type: object + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + nullable: true + type: string + type: object + terminationGracePeriodSeconds: + nullable: true + type: integer + timeoutSeconds: + type: integer + type: object + stdin: + type: boolean + stdinOnce: + type: boolean + terminationMessagePath: + nullable: true + type: string + terminationMessagePolicy: + nullable: true + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + nullable: true + type: string + name: + nullable: true + type: string + type: object + nullable: true + type: array + volumeMounts: + items: + properties: + mountPath: + nullable: true + type: string + mountPropagation: + nullable: true + type: string + name: + nullable: true + type: string + readOnly: + type: boolean + subPath: + nullable: true + type: string + subPathExpr: + nullable: true + type: string + type: object + nullable: true + type: array + workingDir: + nullable: true + type: string + type: object + nullable: true + type: array + dnsConfig: + nullable: true + properties: + nameservers: + items: + nullable: true + type: string + nullable: true + type: array + options: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + searches: + items: + nullable: true + type: string + nullable: true + type: array + type: object + dnsPolicy: + nullable: true + type: string + enableServiceLinks: + nullable: true + type: boolean + ephemeralContainers: + items: + properties: + args: + items: + nullable: true + type: string + nullable: true + type: array + command: + items: + nullable: true + type: string + nullable: true + type: array + env: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + valueFrom: + nullable: true + properties: + configMapKeyRef: + nullable: true + properties: + key: + nullable: true + type: string + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + fieldRef: + nullable: true + properties: + apiVersion: + nullable: true + type: string + fieldPath: + nullable: true + type: string + type: object + resourceFieldRef: + nullable: true + properties: + containerName: + nullable: true + type: string + divisor: + nullable: true + type: string + resource: + nullable: true + type: string + type: object + secretKeyRef: + nullable: true + properties: + key: + nullable: true + type: string + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + type: object + type: object + nullable: true + type: array + envFrom: + items: + properties: + configMapRef: + nullable: true + properties: + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + prefix: + nullable: true + type: string + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + type: object + nullable: true + type: array + image: + nullable: true + type: string + imagePullPolicy: + nullable: true + type: string + lifecycle: + nullable: true + properties: + postStart: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + nullable: true + type: string + scheme: + nullable: true + type: string + type: object + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + nullable: true + type: string + type: object + type: object + preStop: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + nullable: true + type: string + scheme: + nullable: true + type: string + type: object + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + nullable: true + type: string + type: object + type: object + type: object + livenessProbe: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + failureThreshold: + type: integer + grpc: + nullable: true + properties: + port: + type: integer + service: + nullable: true + type: string + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + nullable: true + type: string + scheme: + nullable: true + type: string + type: object + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + nullable: true + type: string + type: object + terminationGracePeriodSeconds: + nullable: true + type: integer + timeoutSeconds: + type: integer + type: object + name: + nullable: true + type: string + ports: + items: + properties: + containerPort: + type: integer + hostIP: + nullable: true + type: string + hostPort: + type: integer + name: + nullable: true + type: string + protocol: + nullable: true + type: string + type: object + nullable: true + type: array + readinessProbe: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + failureThreshold: + type: integer + grpc: + nullable: true + properties: + port: + type: integer + service: + nullable: true + type: string + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + nullable: true + type: string + scheme: + nullable: true + type: string + type: object + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + nullable: true + type: string + type: object + terminationGracePeriodSeconds: + nullable: true + type: integer + timeoutSeconds: + type: integer + type: object + resizePolicy: + items: + properties: + resourceName: + nullable: true + type: string + restartPolicy: + nullable: true + type: string + type: object + nullable: true + type: array + resources: + properties: + claims: + items: + properties: + name: + nullable: true + type: string + type: object + nullable: true + type: array + limits: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + requests: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + securityContext: + nullable: true + properties: + allowPrivilegeEscalation: + nullable: true + type: boolean + capabilities: + nullable: true + properties: + add: + items: + nullable: true + type: string + nullable: true + type: array + drop: + items: + nullable: true + type: string + nullable: true + type: array + type: object + privileged: + nullable: true + type: boolean + procMount: + nullable: true + type: string + readOnlyRootFilesystem: + nullable: true + type: boolean + runAsGroup: + nullable: true + type: integer + runAsNonRoot: + nullable: true + type: boolean + runAsUser: + nullable: true + type: integer + seLinuxOptions: + nullable: true + properties: + level: + nullable: true + type: string + role: + nullable: true + type: string + type: + nullable: true + type: string + user: + nullable: true + type: string + type: object + seccompProfile: + nullable: true + properties: + localhostProfile: + nullable: true + type: string + type: + nullable: true + type: string + type: object + windowsOptions: + nullable: true + properties: + gmsaCredentialSpec: + nullable: true + type: string + gmsaCredentialSpecName: + nullable: true + type: string + hostProcess: + nullable: true + type: boolean + runAsUserName: + nullable: true + type: string + type: object + type: object + startupProbe: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + failureThreshold: + type: integer + grpc: + nullable: true + properties: + port: + type: integer + service: + nullable: true + type: string + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + nullable: true + type: string + scheme: + nullable: true + type: string + type: object + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + nullable: true + type: string + type: object + terminationGracePeriodSeconds: + nullable: true + type: integer + timeoutSeconds: + type: integer + type: object + stdin: + type: boolean + stdinOnce: + type: boolean + targetContainerName: + nullable: true + type: string + terminationMessagePath: + nullable: true + type: string + terminationMessagePolicy: + nullable: true + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + nullable: true + type: string + name: + nullable: true + type: string + type: object + nullable: true + type: array + volumeMounts: + items: + properties: + mountPath: + nullable: true + type: string + mountPropagation: + nullable: true + type: string + name: + nullable: true + type: string + readOnly: + type: boolean + subPath: + nullable: true + type: string + subPathExpr: + nullable: true + type: string + type: object + nullable: true + type: array + workingDir: + nullable: true + type: string + type: object + nullable: true + type: array + hostAliases: + items: + properties: + hostnames: + items: + nullable: true + type: string + nullable: true + type: array + ip: + nullable: true + type: string + type: object + nullable: true + type: array + hostIPC: + type: boolean + hostNetwork: + type: boolean + hostPID: + type: boolean + hostUsers: + nullable: true + type: boolean + hostname: + nullable: true + type: string + imagePullSecrets: + items: + properties: + name: + nullable: true + type: string + type: object + nullable: true + type: array + initContainers: + items: + properties: + args: + items: + nullable: true + type: string + nullable: true + type: array + command: + items: + nullable: true + type: string + nullable: true + type: array + env: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + valueFrom: + nullable: true + properties: + configMapKeyRef: + nullable: true + properties: + key: + nullable: true + type: string + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + fieldRef: + nullable: true + properties: + apiVersion: + nullable: true + type: string + fieldPath: + nullable: true + type: string + type: object + resourceFieldRef: + nullable: true + properties: + containerName: + nullable: true + type: string + divisor: + nullable: true + type: string + resource: + nullable: true + type: string + type: object + secretKeyRef: + nullable: true + properties: + key: + nullable: true + type: string + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + type: object + type: object + nullable: true + type: array + envFrom: + items: + properties: + configMapRef: + nullable: true + properties: + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + prefix: + nullable: true + type: string + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + type: object + nullable: true + type: array + image: + nullable: true + type: string + imagePullPolicy: + nullable: true + type: string + lifecycle: + nullable: true + properties: + postStart: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + nullable: true + type: string + scheme: + nullable: true + type: string + type: object + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + nullable: true + type: string + type: object + type: object + preStop: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + nullable: true + type: string + scheme: + nullable: true + type: string + type: object + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + nullable: true + type: string + type: object + type: object + type: object + livenessProbe: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + failureThreshold: + type: integer + grpc: + nullable: true + properties: + port: + type: integer + service: + nullable: true + type: string + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + nullable: true + type: string + scheme: + nullable: true + type: string + type: object + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + nullable: true + type: string + type: object + terminationGracePeriodSeconds: + nullable: true + type: integer + timeoutSeconds: + type: integer + type: object + name: + nullable: true + type: string + ports: + items: + properties: + containerPort: + type: integer + hostIP: + nullable: true + type: string + hostPort: + type: integer + name: + nullable: true + type: string + protocol: + nullable: true + type: string + type: object + nullable: true + type: array + readinessProbe: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + failureThreshold: + type: integer + grpc: + nullable: true + properties: + port: + type: integer + service: + nullable: true + type: string + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + nullable: true + type: string + scheme: + nullable: true + type: string + type: object + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + nullable: true + type: string + type: object + terminationGracePeriodSeconds: + nullable: true + type: integer + timeoutSeconds: + type: integer + type: object + resizePolicy: + items: + properties: + resourceName: + nullable: true + type: string + restartPolicy: + nullable: true + type: string + type: object + nullable: true + type: array + resources: + properties: + claims: + items: + properties: + name: + nullable: true + type: string + type: object + nullable: true + type: array + limits: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + requests: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + securityContext: + nullable: true + properties: + allowPrivilegeEscalation: + nullable: true + type: boolean + capabilities: + nullable: true + properties: + add: + items: + nullable: true + type: string + nullable: true + type: array + drop: + items: + nullable: true + type: string + nullable: true + type: array + type: object + privileged: + nullable: true + type: boolean + procMount: + nullable: true + type: string + readOnlyRootFilesystem: + nullable: true + type: boolean + runAsGroup: + nullable: true + type: integer + runAsNonRoot: + nullable: true + type: boolean + runAsUser: + nullable: true + type: integer + seLinuxOptions: + nullable: true + properties: + level: + nullable: true + type: string + role: + nullable: true + type: string + type: + nullable: true + type: string + user: + nullable: true + type: string + type: object + seccompProfile: + nullable: true + properties: + localhostProfile: + nullable: true + type: string + type: + nullable: true + type: string + type: object + windowsOptions: + nullable: true + properties: + gmsaCredentialSpec: + nullable: true + type: string + gmsaCredentialSpecName: + nullable: true + type: string + hostProcess: + nullable: true + type: boolean + runAsUserName: + nullable: true + type: string + type: object + type: object + startupProbe: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + failureThreshold: + type: integer + grpc: + nullable: true + properties: + port: + type: integer + service: + nullable: true + type: string + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + nullable: true + type: string + scheme: + nullable: true + type: string + type: object + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + nullable: true + type: string + type: object + terminationGracePeriodSeconds: + nullable: true + type: integer + timeoutSeconds: + type: integer + type: object + stdin: + type: boolean + stdinOnce: + type: boolean + terminationMessagePath: + nullable: true + type: string + terminationMessagePolicy: + nullable: true + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + nullable: true + type: string + name: + nullable: true + type: string + type: object + nullable: true + type: array + volumeMounts: + items: + properties: + mountPath: + nullable: true + type: string + mountPropagation: + nullable: true + type: string + name: + nullable: true + type: string + readOnly: + type: boolean + subPath: + nullable: true + type: string + subPathExpr: + nullable: true + type: string + type: object + nullable: true + type: array + workingDir: + nullable: true + type: string + type: object + nullable: true + type: array + nodeName: + nullable: true + type: string + nodeSelector: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + os: + nullable: true + properties: + name: + nullable: true + type: string + type: object + overhead: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + preemptionPolicy: + nullable: true + type: string + priority: + nullable: true + type: integer + priorityClassName: + nullable: true + type: string + readinessGates: + items: + properties: + conditionType: + nullable: true + type: string + type: object + nullable: true + type: array + resourceClaims: + items: + properties: + name: + nullable: true + type: string + source: + properties: + resourceClaimName: + nullable: true + type: string + resourceClaimTemplateName: + nullable: true + type: string + type: object + type: object + nullable: true + type: array + restartPolicy: + nullable: true + type: string + runtimeClassName: + nullable: true + type: string + schedulerName: + nullable: true + type: string + schedulingGates: + items: + properties: + name: + nullable: true + type: string + type: object + nullable: true + type: array + securityContext: + nullable: true + properties: + fsGroup: + nullable: true + type: integer + fsGroupChangePolicy: + nullable: true + type: string + runAsGroup: + nullable: true + type: integer + runAsNonRoot: + nullable: true + type: boolean + runAsUser: + nullable: true + type: integer + seLinuxOptions: + nullable: true + properties: + level: + nullable: true + type: string + role: + nullable: true + type: string + type: + nullable: true + type: string + user: + nullable: true + type: string + type: object + seccompProfile: + nullable: true + properties: + localhostProfile: + nullable: true + type: string + type: + nullable: true + type: string + type: object + supplementalGroups: + items: + type: integer + nullable: true + type: array + sysctls: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + windowsOptions: + nullable: true + properties: + gmsaCredentialSpec: + nullable: true + type: string + gmsaCredentialSpecName: + nullable: true + type: string + hostProcess: + nullable: true + type: boolean + runAsUserName: + nullable: true + type: string + type: object + type: object + serviceAccount: + nullable: true + type: string + serviceAccountName: + nullable: true + type: string + setHostnameAsFQDN: + nullable: true + type: boolean + shareProcessNamespace: + nullable: true + type: boolean + subdomain: + nullable: true + type: string + terminationGracePeriodSeconds: + nullable: true + type: integer + tolerations: + items: + properties: + effect: + nullable: true + type: string + key: + nullable: true + type: string + operator: + nullable: true + type: string + tolerationSeconds: + nullable: true + type: integer + value: + nullable: true + type: string + type: object + nullable: true + type: array + topologySpreadConstraints: + items: + properties: + labelSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + matchLabelKeys: + items: + nullable: true + type: string + nullable: true + type: array + maxSkew: + type: integer + minDomains: + nullable: true + type: integer + nodeAffinityPolicy: + nullable: true + type: string + nodeTaintsPolicy: + nullable: true + type: string + topologyKey: + nullable: true + type: string + whenUnsatisfiable: + nullable: true + type: string + type: object + nullable: true + type: array + volumes: + items: + properties: + awsElasticBlockStore: + nullable: true + properties: + fsType: + nullable: true + type: string + partition: + type: integer + readOnly: + type: boolean + volumeID: + nullable: true + type: string + type: object + azureDisk: + nullable: true + properties: + cachingMode: + nullable: true + type: string + diskName: + nullable: true + type: string + diskURI: + nullable: true + type: string + fsType: + nullable: true + type: string + kind: + nullable: true + type: string + readOnly: + nullable: true + type: boolean + type: object + azureFile: + nullable: true + properties: + readOnly: + type: boolean + secretName: + nullable: true + type: string + shareName: + nullable: true + type: string + type: object + cephfs: + nullable: true + properties: + monitors: + items: + nullable: true + type: string + nullable: true + type: array + path: + nullable: true + type: string + readOnly: + type: boolean + secretFile: + nullable: true + type: string + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + type: object + user: + nullable: true + type: string + type: object + cinder: + nullable: true + properties: + fsType: + nullable: true + type: string + readOnly: + type: boolean + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + type: object + volumeID: + nullable: true + type: string + type: object + configMap: + nullable: true + properties: + defaultMode: + nullable: true + type: integer + items: + items: + properties: + key: + nullable: true + type: string + mode: + nullable: true + type: integer + path: + nullable: true + type: string + type: object + nullable: true + type: array + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + csi: + nullable: true + properties: + driver: + nullable: true + type: string + fsType: + nullable: true + type: string + nodePublishSecretRef: + nullable: true + properties: + name: + nullable: true + type: string + type: object + readOnly: + nullable: true + type: boolean + volumeAttributes: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + downwardAPI: + nullable: true + properties: + defaultMode: + nullable: true + type: integer + items: + items: + properties: + fieldRef: + nullable: true + properties: + apiVersion: + nullable: true + type: string + fieldPath: + nullable: true + type: string + type: object + mode: + nullable: true + type: integer + path: + nullable: true + type: string + resourceFieldRef: + nullable: true + properties: + containerName: + nullable: true + type: string + divisor: + nullable: true + type: string + resource: + nullable: true + type: string + type: object + type: object + nullable: true + type: array + type: object + emptyDir: + nullable: true + properties: + medium: + nullable: true + type: string + sizeLimit: + nullable: true + type: string + type: object + ephemeral: + nullable: true + properties: + volumeClaimTemplate: + nullable: true + properties: + metadata: + properties: + annotations: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + creationTimestamp: + nullable: true + type: string + deletionGracePeriodSeconds: + nullable: true + type: integer + deletionTimestamp: + nullable: true + type: string + finalizers: + items: + nullable: true + type: string + nullable: true + type: array + generateName: + nullable: true + type: string + generation: + type: integer + labels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + managedFields: + items: + properties: + apiVersion: + nullable: true + type: string + fieldsType: + nullable: true + type: string + fieldsV1: + nullable: true + type: object + manager: + nullable: true + type: string + operation: + nullable: true + type: string + subresource: + nullable: true + type: string + time: + nullable: true + type: string + type: object + nullable: true + type: array + name: + nullable: true + type: string + namespace: + nullable: true + type: string + ownerReferences: + items: + properties: + apiVersion: + nullable: true + type: string + blockOwnerDeletion: + nullable: true + type: boolean + controller: + nullable: true + type: boolean + kind: + nullable: true + type: string + name: + nullable: true + type: string + uid: + nullable: true + type: string + type: object + nullable: true + type: array + resourceVersion: + nullable: true + type: string + selfLink: + nullable: true + type: string + uid: + nullable: true + type: string + type: object + spec: + properties: + accessModes: + items: + nullable: true + type: string + nullable: true + type: array + dataSource: + nullable: true + properties: + apiGroup: + nullable: true + type: string + kind: + nullable: true + type: string + name: + nullable: true + type: string + type: object + dataSourceRef: + nullable: true + properties: + apiGroup: + nullable: true + type: string + kind: + nullable: true + type: string + name: + nullable: true + type: string + namespace: + nullable: true + type: string + type: object + resources: + properties: + claims: + items: + properties: + name: + nullable: true + type: string + type: object + nullable: true + type: array + limits: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + requests: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + selector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + storageClassName: + nullable: true + type: string + volumeMode: + nullable: true + type: string + volumeName: + nullable: true + type: string + type: object + type: object + type: object + fc: + nullable: true + properties: + fsType: + nullable: true + type: string + lun: + nullable: true + type: integer + readOnly: + type: boolean + targetWWNs: + items: + nullable: true + type: string + nullable: true + type: array + wwids: + items: + nullable: true + type: string + nullable: true + type: array + type: object + flexVolume: + nullable: true + properties: + driver: + nullable: true + type: string + fsType: + nullable: true + type: string + options: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + readOnly: + type: boolean + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + type: object + type: object + flocker: + nullable: true + properties: + datasetName: + nullable: true + type: string + datasetUUID: + nullable: true + type: string + type: object + gcePersistentDisk: + nullable: true + properties: + fsType: + nullable: true + type: string + partition: + type: integer + pdName: + nullable: true + type: string + readOnly: + type: boolean + type: object + gitRepo: + nullable: true + properties: + directory: + nullable: true + type: string + repository: + nullable: true + type: string + revision: + nullable: true + type: string + type: object + glusterfs: + nullable: true + properties: + endpoints: + nullable: true + type: string + path: + nullable: true + type: string + readOnly: + type: boolean + type: object + hostPath: + nullable: true + properties: + path: + nullable: true + type: string + type: + nullable: true + type: string + type: object + iscsi: + nullable: true + properties: + chapAuthDiscovery: + type: boolean + chapAuthSession: + type: boolean + fsType: + nullable: true + type: string + initiatorName: + nullable: true + type: string + iqn: + nullable: true + type: string + iscsiInterface: + nullable: true + type: string + lun: + type: integer + portals: + items: + nullable: true + type: string + nullable: true + type: array + readOnly: + type: boolean + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + type: object + targetPortal: + nullable: true + type: string + type: object + name: + nullable: true + type: string + nfs: + nullable: true + properties: + path: + nullable: true + type: string + readOnly: + type: boolean + server: + nullable: true + type: string + type: object + persistentVolumeClaim: + nullable: true + properties: + claimName: + nullable: true + type: string + readOnly: + type: boolean + type: object + photonPersistentDisk: + nullable: true + properties: + fsType: + nullable: true + type: string + pdID: + nullable: true + type: string + type: object + portworxVolume: + nullable: true + properties: + fsType: + nullable: true + type: string + readOnly: + type: boolean + volumeID: + nullable: true + type: string + type: object + projected: + nullable: true + properties: + defaultMode: + nullable: true + type: integer + sources: + items: + properties: + configMap: + nullable: true + properties: + items: + items: + properties: + key: + nullable: true + type: string + mode: + nullable: true + type: integer + path: + nullable: true + type: string + type: object + nullable: true + type: array + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + downwardAPI: + nullable: true + properties: + items: + items: + properties: + fieldRef: + nullable: true + properties: + apiVersion: + nullable: true + type: string + fieldPath: + nullable: true + type: string + type: object + mode: + nullable: true + type: integer + path: + nullable: true + type: string + resourceFieldRef: + nullable: true + properties: + containerName: + nullable: true + type: string + divisor: + nullable: true + type: string + resource: + nullable: true + type: string + type: object + type: object + nullable: true + type: array + type: object + secret: + nullable: true + properties: + items: + items: + properties: + key: + nullable: true + type: string + mode: + nullable: true + type: integer + path: + nullable: true + type: string + type: object + nullable: true + type: array + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + serviceAccountToken: + nullable: true + properties: + audience: + nullable: true + type: string + expirationSeconds: + nullable: true + type: integer + path: + nullable: true + type: string + type: object + type: object + nullable: true + type: array + type: object + quobyte: + nullable: true + properties: + group: + nullable: true + type: string + readOnly: + type: boolean + registry: + nullable: true + type: string + tenant: + nullable: true + type: string + user: + nullable: true + type: string + volume: + nullable: true + type: string + type: object + rbd: + nullable: true + properties: + fsType: + nullable: true + type: string + image: + nullable: true + type: string + keyring: + nullable: true + type: string + monitors: + items: + nullable: true + type: string + nullable: true + type: array + pool: + nullable: true + type: string + readOnly: + type: boolean + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + type: object + user: + nullable: true + type: string + type: object + scaleIO: + nullable: true + properties: + fsType: + nullable: true + type: string + gateway: + nullable: true + type: string + protectionDomain: + nullable: true + type: string + readOnly: + type: boolean + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + type: object + sslEnabled: + type: boolean + storageMode: + nullable: true + type: string + storagePool: + nullable: true + type: string + system: + nullable: true + type: string + volumeName: + nullable: true + type: string + type: object + secret: + nullable: true + properties: + defaultMode: + nullable: true + type: integer + items: + items: + properties: + key: + nullable: true + type: string + mode: + nullable: true + type: integer + path: + nullable: true + type: string + type: object + nullable: true + type: array + optional: + nullable: true + type: boolean + secretName: + nullable: true + type: string + type: object + storageos: + nullable: true + properties: + fsType: + nullable: true + type: string + readOnly: + type: boolean + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + type: object + volumeName: + nullable: true + type: string + volumeNamespace: + nullable: true + type: string + type: object + vsphereVolume: + nullable: true + properties: + fsType: + nullable: true + type: string + storagePolicyID: + nullable: true + type: string + storagePolicyName: + nullable: true + type: string + volumePath: + nullable: true + type: string + type: object + type: object + nullable: true + type: array + type: object + type: object + ttlSecondsAfterFinished: + nullable: true + type: integer + type: object + syncInterval: + type: integer + type: object + status: + properties: + commit: + nullable: true + type: string + conditions: + items: + properties: + lastTransitionTime: + nullable: true + type: string + lastUpdateTime: + nullable: true + type: string + message: + nullable: true + type: string + reason: + nullable: true + type: string + status: + nullable: true + type: string + type: + nullable: true + type: string + type: object + nullable: true + type: array + event: + nullable: true + type: string + hookId: + nullable: true + type: string + jobStatus: + nullable: true + type: string + lastExecutedCommit: + nullable: true + type: string + lastSyncedTime: + nullable: true + type: string + observedGeneration: + type: integer + secretToken: + nullable: true + type: string + updateGeneration: + type: integer + type: object + type: object + version: v1 + versions: + - name: v1 + served: true + storage: true +{{- end -}} diff --git a/charts/fleet-crd/102.2.5+up0.8.5/values.yaml b/charts/fleet-crd/102.2.5+up0.8.5/values.yaml new file mode 100644 index 0000000000..d41d3a2444 --- /dev/null +++ b/charts/fleet-crd/102.2.5+up0.8.5/values.yaml @@ -0,0 +1 @@ +# This file is intentionally empty diff --git a/charts/fleet/102.2.5+up0.8.5/Chart.yaml b/charts/fleet/102.2.5+up0.8.5/Chart.yaml new file mode 100644 index 0000000000..6d06eb655e --- /dev/null +++ b/charts/fleet/102.2.5+up0.8.5/Chart.yaml @@ -0,0 +1,22 @@ +annotations: + catalog.cattle.io/auto-install: fleet-crd=match + catalog.cattle.io/certified: rancher + catalog.cattle.io/experimental: "true" + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.16.0-0 < 1.28.0-0' + catalog.cattle.io/namespace: cattle-fleet-system + catalog.cattle.io/os: linux + catalog.cattle.io/permits-os: linux,windows + catalog.cattle.io/provides-gvr: clusters.fleet.cattle.io/v1alpha1 + catalog.cattle.io/rancher-version: '>= 2.7.0-0 < 2.8.0-0' + catalog.cattle.io/release-name: fleet +apiVersion: v2 +appVersion: 0.8.5 +dependencies: +- condition: gitops.enabled + name: gitjob + repository: file://./charts/gitjob +description: Fleet Manager - GitOps at Scale +icon: https://charts.rancher.io/assets/logos/fleet.svg +name: fleet +version: 102.2.5+up0.8.5 diff --git a/charts/fleet/102.2.5+up0.8.5/README.md b/charts/fleet/102.2.5+up0.8.5/README.md new file mode 100644 index 0000000000..2f2a4c302a --- /dev/null +++ b/charts/fleet/102.2.5+up0.8.5/README.md @@ -0,0 +1,30 @@ +# Fleet Helm Chart + +Fleet is GitOps at scale. Fleet is designed to manage multiple clusters. + +## What is Fleet? + +* Cluster engine: Fleet is a container management and deployment engine designed to offer users more control on the local cluster and constant monitoring through GitOps. Fleet focuses not only on the ability to scale, but it also gives users a high degree of control and visibility to monitor exactly what is installed on the cluster. + +* Deployment management: Fleet can manage deployments from git of raw Kubernetes YAML, Helm charts, Kustomize, or any combination of the three. Regardless of the source, all resources are dynamically turned into Helm charts, and Helm is used as the engine to deploy all resources in the cluster. As a result, users can enjoy a high degree of control, consistency, and auditability of their clusters. + +## Introduction + +This chart deploys Fleet on a Kubernetes cluster. It also deploys some of its dependencies as subcharts. + +The documentation is centralized in the [doc website](https://fleet.rancher.io/). + +## Prerequisites + +Get helm if you don't have it. Helm 3 is just a CLI. + + +## Install Fleet + +Install the Fleet Helm charts (there are two because we separate out CRDs for ultimate flexibility.): + +``` +$ helm repo add fleet https://rancher.github.io/fleet-helm-charts/ +$ helm -n cattle-fleet-system install --create-namespace --wait fleet-crd fleet/fleet-crd +$ helm -n cattle-fleet-system install --create-namespace --wait fleet fleet/fleet +``` \ No newline at end of file diff --git a/charts/fleet/102.2.5+up0.8.5/charts/gitjob/.helmignore b/charts/fleet/102.2.5+up0.8.5/charts/gitjob/.helmignore new file mode 100644 index 0000000000..691fa13d6a --- /dev/null +++ b/charts/fleet/102.2.5+up0.8.5/charts/gitjob/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ \ No newline at end of file diff --git a/charts/fleet/102.2.5+up0.8.5/charts/gitjob/Chart.yaml b/charts/fleet/102.2.5+up0.8.5/charts/gitjob/Chart.yaml new file mode 100644 index 0000000000..ffbc70a1b6 --- /dev/null +++ b/charts/fleet/102.2.5+up0.8.5/charts/gitjob/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v2 +appVersion: 0.8.8 +description: Controller that run jobs based on git events +name: gitjob +version: 0.8.8 diff --git a/charts/fleet/102.2.5+up0.8.5/charts/gitjob/templates/_helpers.tpl b/charts/fleet/102.2.5+up0.8.5/charts/gitjob/templates/_helpers.tpl new file mode 100644 index 0000000000..f652b5643d --- /dev/null +++ b/charts/fleet/102.2.5+up0.8.5/charts/gitjob/templates/_helpers.tpl @@ -0,0 +1,7 @@ +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- else -}} +{{- "" -}} +{{- end -}} +{{- end -}} \ No newline at end of file diff --git a/charts/fleet/102.2.5+up0.8.5/charts/gitjob/templates/clusterrole.yaml b/charts/fleet/102.2.5+up0.8.5/charts/gitjob/templates/clusterrole.yaml new file mode 100644 index 0000000000..bcad90164f --- /dev/null +++ b/charts/fleet/102.2.5+up0.8.5/charts/gitjob/templates/clusterrole.yaml @@ -0,0 +1,38 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: gitjob +rules: + - apiGroups: + - "batch" + resources: + - 'jobs' + verbs: + - '*' + - apiGroups: + - "" + resources: + - 'pods' + verbs: + - 'list' + - 'get' + - 'watch' + - apiGroups: + - "" + resources: + - 'secrets' + verbs: + - '*' + - apiGroups: + - "" + resources: + - 'configmaps' + verbs: + - '*' + - apiGroups: + - "gitjob.cattle.io" + resources: + - "gitjobs" + - "gitjobs/status" + verbs: + - "*" \ No newline at end of file diff --git a/charts/fleet/102.2.5+up0.8.5/charts/gitjob/templates/clusterrolebinding.yaml b/charts/fleet/102.2.5+up0.8.5/charts/gitjob/templates/clusterrolebinding.yaml new file mode 100644 index 0000000000..0bf07c4ef8 --- /dev/null +++ b/charts/fleet/102.2.5+up0.8.5/charts/gitjob/templates/clusterrolebinding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: gitjob-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: gitjob +subjects: + - kind: ServiceAccount + name: gitjob + namespace: {{ .Release.Namespace }} \ No newline at end of file diff --git a/charts/fleet/102.2.5+up0.8.5/charts/gitjob/templates/deployment.yaml b/charts/fleet/102.2.5+up0.8.5/charts/gitjob/templates/deployment.yaml new file mode 100644 index 0000000000..e7bbe5f20a --- /dev/null +++ b/charts/fleet/102.2.5+up0.8.5/charts/gitjob/templates/deployment.yaml @@ -0,0 +1,51 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: gitjob +spec: + selector: + matchLabels: + app: "gitjob" + template: + metadata: + labels: + app: "gitjob" + spec: + serviceAccountName: gitjob + containers: + - image: "{{ template "system_default_registry" . }}{{ .Values.gitjob.repository }}:{{ .Values.gitjob.tag }}" + name: gitjob + args: + {{- if .Values.debug }} + - --debug + {{- end }} + - --tekton-image + - "{{ template "system_default_registry" . }}{{ .Values.tekton.repository }}:{{ .Values.tekton.tag }}" + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- if .Values.proxy }} + - name: HTTP_PROXY + value: {{ .Values.proxy }} + - name: HTTPS_PROXY + value: {{ .Values.proxy }} + - name: NO_PROXY + value: {{ .Values.noProxy }} + {{- end }} + {{- if .Values.debug }} + - name: CATTLE_DEV_MODE + value: "true" + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.priorityClassName }} + priorityClassName: "{{.Values.priorityClassName}}" + {{- end }} diff --git a/charts/fleet/102.2.5+up0.8.5/charts/gitjob/templates/leases.yaml b/charts/fleet/102.2.5+up0.8.5/charts/gitjob/templates/leases.yaml new file mode 100644 index 0000000000..51f9339509 --- /dev/null +++ b/charts/fleet/102.2.5+up0.8.5/charts/gitjob/templates/leases.yaml @@ -0,0 +1,23 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: gitjob +rules: + - apiGroups: + - "coordination.k8s.io" + resources: + - "leases" + verbs: + - "*" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: gitjob +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: gitjob +subjects: + - kind: ServiceAccount + name: gitjob diff --git a/charts/fleet/102.2.5+up0.8.5/charts/gitjob/templates/service.yaml b/charts/fleet/102.2.5+up0.8.5/charts/gitjob/templates/service.yaml new file mode 100644 index 0000000000..bf57c1b55c --- /dev/null +++ b/charts/fleet/102.2.5+up0.8.5/charts/gitjob/templates/service.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + name: gitjob +spec: + ports: + - name: http-80 + port: 80 + protocol: TCP + targetPort: 8080 + selector: + app: "gitjob" \ No newline at end of file diff --git a/charts/fleet/102.2.5+up0.8.5/charts/gitjob/templates/serviceaccount.yaml b/charts/fleet/102.2.5+up0.8.5/charts/gitjob/templates/serviceaccount.yaml new file mode 100644 index 0000000000..5f8aecb045 --- /dev/null +++ b/charts/fleet/102.2.5+up0.8.5/charts/gitjob/templates/serviceaccount.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: gitjob diff --git a/charts/fleet/102.2.5+up0.8.5/charts/gitjob/values.yaml b/charts/fleet/102.2.5+up0.8.5/charts/gitjob/values.yaml new file mode 100644 index 0000000000..85a398423e --- /dev/null +++ b/charts/fleet/102.2.5+up0.8.5/charts/gitjob/values.yaml @@ -0,0 +1,31 @@ +gitjob: + repository: rancher/gitjob + tag: v0.8.8 + +tekton: + repository: rancher/tekton-utils + tag: v0.1.44 + +global: + cattle: + systemDefaultRegistry: "" + +# http[s] proxy server +# proxy: http://@:: + +# comma separated list of domains or ip addresses that will not use the proxy +noProxy: 127.0.0.0/8,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,.svc,.cluster.local + +nodeSelector: + kubernetes.io/os: linux + +tolerations: + - key: cattle.io/os + operator: "Equal" + value: "linux" + effect: NoSchedule + +# PriorityClassName assigned to deployment. +priorityClassName: "" + +debug: false diff --git a/charts/fleet/102.2.5+up0.8.5/templates/_helpers.tpl b/charts/fleet/102.2.5+up0.8.5/templates/_helpers.tpl new file mode 100644 index 0000000000..6cd96c3ace --- /dev/null +++ b/charts/fleet/102.2.5+up0.8.5/templates/_helpers.tpl @@ -0,0 +1,22 @@ +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- else -}} +{{- "" -}} +{{- end -}} +{{- end -}} + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +kubernetes.io/os: linux +{{- end -}} \ No newline at end of file diff --git a/charts/fleet/102.2.5+up0.8.5/templates/configmap.yaml b/charts/fleet/102.2.5+up0.8.5/templates/configmap.yaml new file mode 100644 index 0000000000..07f1b5924d --- /dev/null +++ b/charts/fleet/102.2.5+up0.8.5/templates/configmap.yaml @@ -0,0 +1,25 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: fleet-controller +data: + config: | + { + "systemDefaultRegistry": "{{ template "system_default_registry" . }}", + "agentImage": "{{ template "system_default_registry" . }}{{.Values.agentImage.repository}}:{{.Values.agentImage.tag}}", + "agentImagePullPolicy": "{{ .Values.agentImage.imagePullPolicy }}", + "apiServerURL": "{{.Values.apiServerURL}}", + "apiServerCA": "{{b64enc .Values.apiServerCA}}", + "agentCheckinInterval": "{{.Values.agentCheckinInterval}}", + "ignoreClusterRegistrationLabels": {{.Values.ignoreClusterRegistrationLabels}}, + "bootstrap": { + "paths": "{{.Values.bootstrap.paths}}", + "repo": "{{.Values.bootstrap.repo}}", + "secret": "{{.Values.bootstrap.secret}}", + "branch": "{{.Values.bootstrap.branch}}", + "namespace": "{{.Values.bootstrap.namespace}}", + "agentNamespace": "{{.Values.bootstrap.agentNamespace}}", + }, + "webhookReceiverURL": "{{.Values.webhookReceiverURL}}", + "githubURLPrefix": "{{.Values.githubURLPrefix}}" + } diff --git a/charts/fleet/102.2.5+up0.8.5/templates/deployment.yaml b/charts/fleet/102.2.5+up0.8.5/templates/deployment.yaml new file mode 100644 index 0000000000..164340c444 --- /dev/null +++ b/charts/fleet/102.2.5+up0.8.5/templates/deployment.yaml @@ -0,0 +1,102 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: fleet-controller +spec: + selector: + matchLabels: + app: fleet-controller + template: + metadata: + labels: + app: fleet-controller + spec: + containers: + - env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: FLEET_PROPAGATE_DEBUG_SETTINGS_TO_AGENTS + value: {{ quote .Values.propagateDebugSettingsToAgents }} + {{- if .Values.clusterEnqueueDelay }} + - name: FLEET_CLUSTER_ENQUEUE_DELAY + value: {{ .Values.clusterEnqueueDelay }} + {{- end }} + {{- if .Values.proxy }} + - name: HTTP_PROXY + value: {{ .Values.proxy }} + - name: HTTPS_PROXY + value: {{ .Values.proxy }} + - name: NO_PROXY + value: {{ .Values.noProxy }} + {{- end }} + {{- if .Values.cpuPprof }} + - name: FLEET_CPU_PPROF_DIR + value: /tmp/pprof/ + {{- end }} + {{- if .Values.cpuPprof }} + - name: FLEET_CPU_PPROF_PERIOD + value: {{ quote .Values.cpuPprof.period }} + {{- end }} + {{- if .Values.debug }} + - name: CATTLE_DEV_MODE + value: "true" + {{- end }} + image: '{{ template "system_default_registry" . }}{{ .Values.image.repository }}:{{ .Values.image.tag }}' + name: fleet-controller + imagePullPolicy: "{{ .Values.image.imagePullPolicy }}" + command: + - fleetcontroller + {{- if not .Values.gitops.enabled }} + - --disable-gitops + {{- end }} + {{- if not .Values.bootstrap.enabled }} + - --disable-bootstrap + {{- end }} + {{- if .Values.debug }} + - --debug + - --debug-level + - {{ quote .Values.debugLevel }} + {{- else }} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + privileged: false + capabilities: + drop: + - ALL + {{- end }} + volumeMounts: + - mountPath: /tmp + name: tmp + {{- if .Values.cpuPprof }} + - mountPath: /tmp/pprof + name: pprof + {{- end }} + volumes: + - name: tmp + emptyDir: {} + {{- if .Values.cpuPprof }} + - name: pprof {{ toYaml .Values.cpuPprof.volumeConfiguration | nindent 10 }} + {{- end }} + + serviceAccountName: fleet-controller + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.nodeSelector }} +{{ toYaml .Values.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.tolerations }} +{{ toYaml .Values.tolerations | indent 8 }} +{{- end }} + {{- if .Values.priorityClassName }} + priorityClassName: "{{.Values.priorityClassName}}" + {{- end }} + +{{- if not .Values.debug }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 +{{- end }} diff --git a/charts/fleet/102.2.5+up0.8.5/templates/job_cleanup_clusterregistrations.yaml b/charts/fleet/102.2.5+up0.8.5/templates/job_cleanup_clusterregistrations.yaml new file mode 100644 index 0000000000..fa59cc575f --- /dev/null +++ b/charts/fleet/102.2.5+up0.8.5/templates/job_cleanup_clusterregistrations.yaml @@ -0,0 +1,29 @@ +{{- if .Values.migrations.clusterRegistrationCleanup }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: fleet-cleanup-clusterregistrations + annotations: + "helm.sh/hook": post-install, post-upgrade + "helm.sh/hook-delete-policy": hook-succeeded, before-hook-creation +spec: + template: + metadata: + labels: + app: fleet-job + spec: + serviceAccountName: fleet-controller + restartPolicy: Never + containers: + - name: cleanup + image: "{{ template "system_default_registry" . }}{{.Values.agentImage.repository}}:{{.Values.agentImage.tag}}" + imagePullPolicy: {{ .Values.global.imagePullPolicy }} + command: + - fleet + args: + - cleanup + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} + backoffLimit: 1 +{{- end }} diff --git a/charts/fleet/102.2.5+up0.8.5/templates/rbac.yaml b/charts/fleet/102.2.5+up0.8.5/templates/rbac.yaml new file mode 100644 index 0000000000..361d68c08b --- /dev/null +++ b/charts/fleet/102.2.5+up0.8.5/templates/rbac.yaml @@ -0,0 +1,114 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: fleet-controller +rules: +- apiGroups: + - gitjob.cattle.io + resources: + - '*' + verbs: + - '*' +- apiGroups: + - fleet.cattle.io + resources: + - '*' + verbs: + - '*' +- apiGroups: + - "" + resources: + - namespaces + - serviceaccounts + verbs: + - '*' +- apiGroups: + - "" + resources: + - secrets + - configmaps + verbs: + - '*' +- apiGroups: + - rbac.authorization.k8s.io + resources: + - clusterroles + - clusterrolebindings + - roles + - rolebindings + verbs: + - '*' + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: fleet-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: fleet-controller +subjects: +- kind: ServiceAccount + name: fleet-controller + namespace: {{.Release.Namespace}} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: fleet-controller +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - '*' +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - '*' + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: fleet-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: fleet-controller +subjects: +- kind: ServiceAccount + name: fleet-controller + +{{- if .Values.bootstrap.enabled }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: fleet-controller-bootstrap +rules: +- apiGroups: + - '*' + resources: + - '*' + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: fleet-controller-bootstrap +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: fleet-controller-bootstrap +subjects: +- kind: ServiceAccount + name: fleet-controller-bootstrap + namespace: {{.Release.Namespace}} +{{- end }} diff --git a/charts/fleet/102.2.5+up0.8.5/templates/serviceaccount.yaml b/charts/fleet/102.2.5+up0.8.5/templates/serviceaccount.yaml new file mode 100644 index 0000000000..ba27c748d7 --- /dev/null +++ b/charts/fleet/102.2.5+up0.8.5/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: fleet-controller + +{{- if .Values.bootstrap.enabled }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: fleet-controller-bootstrap +{{- end }} diff --git a/charts/fleet/102.2.5+up0.8.5/values.yaml b/charts/fleet/102.2.5+up0.8.5/values.yaml new file mode 100644 index 0000000000..807b9f8c84 --- /dev/null +++ b/charts/fleet/102.2.5+up0.8.5/values.yaml @@ -0,0 +1,83 @@ +image: + repository: rancher/fleet + tag: v0.8.5 + imagePullPolicy: IfNotPresent + +agentImage: + repository: rancher/fleet-agent + tag: v0.8.5 + imagePullPolicy: IfNotPresent + +# For cluster registration the public URL of the Kubernetes API server must be set here +# Example: https://example.com:6443 +apiServerURL: "" + +# For cluster registration the pem encoded value of the CA of the Kubernetes API server must be set here +# If left empty it is assumed this Kubernetes API TLS is signed by a well known CA. +apiServerCA: "" + +# A duration string for how often agents should report a heartbeat +agentCheckinInterval: "15m" + +# Whether you want to allow cluster upon registration to specify their labels. +ignoreClusterRegistrationLabels: false + +# Counts from gitrepo are out of sync with bundleDeployment state. +# Just retry in a number of seconds as there is no great way to trigger an event that doesn't cause a loop. +# If not set default is 15 seconds. +# clusterEnqueueDelay: 120s + +# http[s] proxy server +# proxy: http://@:: + +# comma separated list of domains or ip addresses that will not use the proxy +noProxy: 127.0.0.0/8,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,.svc,.cluster.local + +bootstrap: + enabled: true + # The namespace that will be autocreated and the local cluster will be registered in + namespace: fleet-local + # The namespace where the fleet agent for the local cluster will be ran, if empty + # this will default to cattle-fleet-system + agentNamespace: "" + # A repo to add at install time that will deploy to the local cluster. This allows + # one to fully bootstrap fleet, its configuration and all its downstream clusters + # in one shot. + repo: "" + secret: "" + branch: master + paths: "" + + +global: + cattle: + systemDefaultRegistry: "" + +## Node labels for pod assignment +## Ref: https://kubernetes.io/docs/user-guide/node-selection/ +## +nodeSelector: {} +## List of node taints to tolerate (requires Kubernetes >= 1.6) +tolerations: [] + +## PriorityClassName assigned to deployment. +priorityClassName: "" + +gitops: + enabled: true + +debug: false +debugLevel: 0 +propagateDebugSettingsToAgents: true + +## Optional CPU pprof configuration. Profiles are collected continuously and saved every period +## Any valid volume configuration can be provided, the example below uses hostPath +#cpuPprof: +# period: "60s" +# volumeConfiguration: +# hostPath: +# path: /tmp/pprof +# type: DirectoryOrCreate + +migrations: + clusterRegistrationCleanup: true diff --git a/charts/longhorn-crd/102.4.1+up1.6.2/Chart.yaml b/charts/longhorn-crd/102.4.1+up1.6.2/Chart.yaml new file mode 100644 index 0000000000..3cfec69120 --- /dev/null +++ b/charts/longhorn-crd/102.4.1+up1.6.2/Chart.yaml @@ -0,0 +1,11 @@ +annotations: + catalog.cattle.io/certified: rancher + catalog.cattle.io/hidden: "true" + catalog.cattle.io/namespace: longhorn-system + catalog.cattle.io/release-name: longhorn-crd +apiVersion: v1 +appVersion: v1.6.2 +description: Installs the CRDs for longhorn. +name: longhorn-crd +type: application +version: 102.4.1+up1.6.2 diff --git a/charts/longhorn-crd/102.4.1+up1.6.2/README.md b/charts/longhorn-crd/102.4.1+up1.6.2/README.md new file mode 100644 index 0000000000..d9f7f14b33 --- /dev/null +++ b/charts/longhorn-crd/102.4.1+up1.6.2/README.md @@ -0,0 +1,2 @@ +# longhorn-crd +A Rancher chart that installs the CRDs used by longhorn. diff --git a/charts/longhorn-crd/102.4.1+up1.6.2/templates/_helpers.tpl b/charts/longhorn-crd/102.4.1+up1.6.2/templates/_helpers.tpl new file mode 100644 index 0000000000..3fbc2ac02f --- /dev/null +++ b/charts/longhorn-crd/102.4.1+up1.6.2/templates/_helpers.tpl @@ -0,0 +1,66 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "longhorn.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "longhorn.fullname" -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} + + +{{- define "longhorn.managerIP" -}} +{{- $fullname := (include "longhorn.fullname" .) -}} +{{- printf "http://%s-backend:9500" $fullname | trunc 63 | trimSuffix "-" -}} +{{- end -}} + + +{{- define "secret" }} +{{- printf "{\"auths\": {\"%s\": {\"auth\": \"%s\"}}}" .Values.privateRegistry.registryUrl (printf "%s:%s" .Values.privateRegistry.registryUser .Values.privateRegistry.registryPasswd | b64enc) | b64enc }} +{{- end }} + +{{- /* +longhorn.labels generates the standard Helm labels. +*/ -}} +{{- define "longhorn.labels" -}} +app.kubernetes.io/name: {{ template "longhorn.name" . }} +helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/version: {{ .Chart.AppVersion }} +{{- end -}} + + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- else -}} +{{- "" -}} +{{- end -}} +{{- end -}} + +{{- define "registry_url" -}} +{{- if .Values.privateRegistry.registryUrl -}} +{{- printf "%s/" .Values.privateRegistry.registryUrl -}} +{{- else -}} +{{ include "system_default_registry" . }} +{{- end -}} +{{- end -}} + +{{- /* + define the longhorn release namespace +*/ -}} +{{- define "release_namespace" -}} +{{- if .Values.namespaceOverride -}} +{{- .Values.namespaceOverride -}} +{{- else -}} +{{- .Release.Namespace -}} +{{- end -}} +{{- end -}} diff --git a/charts/longhorn-crd/102.4.1+up1.6.2/templates/crds.yaml b/charts/longhorn-crd/102.4.1+up1.6.2/templates/crds.yaml new file mode 100644 index 0000000000..de5046a492 --- /dev/null +++ b/charts/longhorn-crd/102.4.1+up1.6.2/templates/crds.yaml @@ -0,0 +1,3942 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: {{- include "longhorn.labels" . | nindent 4 }} + longhorn-manager: "" + name: backingimagedatasources.longhorn.io +spec: + group: longhorn.io + names: + kind: BackingImageDataSource + listKind: BackingImageDataSourceList + plural: backingimagedatasources + shortNames: + - lhbids + singular: backingimagedatasource + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The current state of the pod used to provision the backing image file from source + jsonPath: .status.currentState + name: State + type: string + - description: The data source type + jsonPath: .spec.sourceType + name: SourceType + type: string + - description: The node the backing image file will be prepared on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The disk the backing image file will be prepared on + jsonPath: .spec.diskUUID + name: DiskUUID + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: BackingImageDataSource is where Longhorn stores backing image data source object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The system generated UUID of the provisioned backing image file + jsonPath: .spec.uuid + name: UUID + type: string + - description: The current state of the pod used to provision the backing image file from source + jsonPath: .status.currentState + name: State + type: string + - description: The data source type + jsonPath: .spec.sourceType + name: SourceType + type: string + - description: The backing image file size + jsonPath: .status.size + name: Size + type: string + - description: The node the backing image file will be prepared on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The disk the backing image file will be prepared on + jsonPath: .spec.diskUUID + name: DiskUUID + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: BackingImageDataSource is where Longhorn stores backing image data source object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BackingImageDataSourceSpec defines the desired state of the Longhorn backing image data source + properties: + checksum: + type: string + diskPath: + type: string + diskUUID: + type: string + fileTransferred: + type: boolean + nodeID: + type: string + parameters: + additionalProperties: + type: string + type: object + sourceType: + enum: + - download + - upload + - export-from-volume + - restore + type: string + uuid: + type: string + type: object + status: + description: BackingImageDataSourceStatus defines the observed state of the Longhorn backing image data source + properties: + checksum: + type: string + currentState: + type: string + ip: + type: string + message: + type: string + ownerID: + type: string + progress: + type: integer + runningParameters: + additionalProperties: + type: string + nullable: true + type: object + size: + format: int64 + type: integer + storageIP: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: {{- include "longhorn.labels" . | nindent 4 }} + longhorn-manager: "" + name: backingimagemanagers.longhorn.io +spec: + group: longhorn.io + names: + kind: BackingImageManager + listKind: BackingImageManagerList + plural: backingimagemanagers + shortNames: + - lhbim + singular: backingimagemanager + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The current state of the manager + jsonPath: .status.currentState + name: State + type: string + - description: The image the manager pod will use + jsonPath: .spec.image + name: Image + type: string + - description: The node the manager is on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The disk the manager is responsible for + jsonPath: .spec.diskUUID + name: DiskUUID + type: string + - description: The disk path the manager is using + jsonPath: .spec.diskPath + name: DiskPath + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: BackingImageManager is where Longhorn stores backing image manager object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The current state of the manager + jsonPath: .status.currentState + name: State + type: string + - description: The image the manager pod will use + jsonPath: .spec.image + name: Image + type: string + - description: The node the manager is on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The disk the manager is responsible for + jsonPath: .spec.diskUUID + name: DiskUUID + type: string + - description: The disk path the manager is using + jsonPath: .spec.diskPath + name: DiskPath + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: BackingImageManager is where Longhorn stores backing image manager object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BackingImageManagerSpec defines the desired state of the Longhorn backing image manager + properties: + backingImages: + additionalProperties: + type: string + type: object + diskPath: + type: string + diskUUID: + type: string + image: + type: string + nodeID: + type: string + type: object + status: + description: BackingImageManagerStatus defines the observed state of the Longhorn backing image manager + properties: + apiMinVersion: + type: integer + apiVersion: + type: integer + backingImageFileMap: + additionalProperties: + properties: + currentChecksum: + type: string + message: + type: string + name: + type: string + progress: + type: integer + senderManagerAddress: + type: string + sendingReference: + type: integer + size: + format: int64 + type: integer + state: + type: string + uuid: + type: string + virtualSize: + format: int64 + type: integer + type: object + nullable: true + type: object + currentState: + type: string + ip: + type: string + ownerID: + type: string + storageIP: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: {{- include "longhorn.labels" . | nindent 4 }} + longhorn-manager: "" + name: backingimages.longhorn.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + name: longhorn-conversion-webhook + namespace: {{ include "release_namespace" . }} + path: /v1/webhook/conversion + port: 9501 + conversionReviewVersions: + - v1beta2 + - v1beta1 + group: longhorn.io + names: + kind: BackingImage + listKind: BackingImageList + plural: backingimages + shortNames: + - lhbi + singular: backingimage + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The backing image name + jsonPath: .spec.image + name: Image + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: BackingImage is where Longhorn stores backing image object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The system generated UUID + jsonPath: .status.uuid + name: UUID + type: string + - description: The source of the backing image file data + jsonPath: .spec.sourceType + name: SourceType + type: string + - description: The backing image file size in each disk + jsonPath: .status.size + name: Size + type: string + - description: The virtual size of the image (may be larger than file size) + jsonPath: .status.virtualSize + name: VirtualSize + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: BackingImage is where Longhorn stores backing image object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BackingImageSpec defines the desired state of the Longhorn backing image + properties: + checksum: + type: string + disks: + additionalProperties: + type: string + type: object + sourceParameters: + additionalProperties: + type: string + type: object + sourceType: + enum: + - download + - upload + - export-from-volume + - restore + type: string + type: object + status: + description: BackingImageStatus defines the observed state of the Longhorn backing image status + properties: + checksum: + type: string + diskFileStatusMap: + additionalProperties: + properties: + lastStateTransitionTime: + type: string + message: + type: string + progress: + type: integer + state: + type: string + type: object + nullable: true + type: object + diskLastRefAtMap: + additionalProperties: + type: string + nullable: true + type: object + ownerID: + type: string + size: + format: int64 + type: integer + uuid: + type: string + virtualSize: + description: Virtual size of image, which may be larger than physical size. Will be zero until known (e.g. while a backing image is uploading) + format: int64 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + longhorn-manager: "" + name: backupbackingimages.longhorn.io +spec: + group: longhorn.io + names: + kind: BackupBackingImage + listKind: BackupBackingImageList + plural: backupbackingimages + shortNames: + - lhbbi + singular: backupbackingimage + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The backing image name + jsonPath: .status.backingImage + name: BackingImage + type: string + - description: The backing image size + jsonPath: .status.size + name: Size + type: string + - description: The backing image backup upload finished time + jsonPath: .status.backupCreatedAt + name: BackupCreatedAt + type: string + - description: The backing image backup state + jsonPath: .status.state + name: State + type: string + - description: The last synced time + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: BackupBackingImage is where Longhorn stores backing image backup object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BackupBackingImageSpec defines the desired state of the Longhorn backing image backup + properties: + labels: + additionalProperties: + type: string + description: The labels of backing image backup. + type: object + syncRequestedAt: + description: The time to request run sync the remote backing image backup. + format: date-time + nullable: true + type: string + userCreated: + description: Is this CR created by user through API or UI. Required + type: boolean + required: + - userCreated + type: object + status: + description: BackupBackingImageStatus defines the observed state of the Longhorn backing image backup + properties: + backingImage: + description: The backing image name. + type: string + backupCreatedAt: + description: The backing image backup upload finished time. + type: string + checksum: + description: The checksum of the backing image. + type: string + compressionMethod: + description: Compression method + type: string + error: + description: The error message when taking the backing image backup. + type: string + labels: + additionalProperties: + type: string + description: The labels of backing image backup. + nullable: true + type: object + lastSyncedAt: + description: The last time that the backing image backup was synced with the remote backup target. + format: date-time + nullable: true + type: string + managerAddress: + description: The address of the backing image manager that runs backing image backup. + type: string + messages: + additionalProperties: + type: string + description: The error messages when listing or inspecting backing image backup. + nullable: true + type: object + ownerID: + description: The node ID on which the controller is responsible to reconcile this CR. + type: string + progress: + description: The backing image backup progress. + type: integer + size: + description: The backing image size. + format: int64 + type: integer + state: + description: The backing image backup creation state. Can be "", "InProgress", "Completed", "Error", "Unknown". + type: string + url: + description: The backing image backup URL. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: {{- include "longhorn.labels" . | nindent 4 }} + longhorn-manager: "" + name: backups.longhorn.io +spec: + group: longhorn.io + names: + kind: Backup + listKind: BackupList + plural: backups + shortNames: + - lhb + singular: backup + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The snapshot name + jsonPath: .status.snapshotName + name: SnapshotName + type: string + - description: The snapshot size + jsonPath: .status.size + name: SnapshotSize + type: string + - description: The snapshot creation time + jsonPath: .status.snapshotCreatedAt + name: SnapshotCreatedAt + type: string + - description: The backup state + jsonPath: .status.state + name: State + type: string + - description: The backup last synced time + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: Backup is where Longhorn stores backup object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The snapshot name + jsonPath: .status.snapshotName + name: SnapshotName + type: string + - description: The snapshot size + jsonPath: .status.size + name: SnapshotSize + type: string + - description: The snapshot creation time + jsonPath: .status.snapshotCreatedAt + name: SnapshotCreatedAt + type: string + - description: The backup state + jsonPath: .status.state + name: State + type: string + - description: The backup last synced time + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: Backup is where Longhorn stores backup object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BackupSpec defines the desired state of the Longhorn backup + properties: + labels: + additionalProperties: + type: string + description: The labels of snapshot backup. + type: object + snapshotName: + description: The snapshot name. + type: string + syncRequestedAt: + description: The time to request run sync the remote backup. + format: date-time + nullable: true + type: string + type: object + status: + description: BackupStatus defines the observed state of the Longhorn backup + properties: + backupCreatedAt: + description: The snapshot backup upload finished time. + type: string + compressionMethod: + description: Compression method + type: string + error: + description: The error message when taking the snapshot backup. + type: string + labels: + additionalProperties: + type: string + description: The labels of snapshot backup. + nullable: true + type: object + lastSyncedAt: + description: The last time that the backup was synced with the remote backup target. + format: date-time + nullable: true + type: string + messages: + additionalProperties: + type: string + description: The error messages when calling longhorn engine on listing or inspecting backups. + nullable: true + type: object + ownerID: + description: The node ID on which the controller is responsible to reconcile this backup CR. + type: string + progress: + description: The snapshot backup progress. + type: integer + replicaAddress: + description: The address of the replica that runs snapshot backup. + type: string + size: + description: The snapshot size. + type: string + snapshotCreatedAt: + description: The snapshot creation time. + type: string + snapshotName: + description: The snapshot name. + type: string + state: + description: The backup creation state. Can be "", "InProgress", "Completed", "Error", "Unknown". + type: string + url: + description: The snapshot backup URL. + type: string + volumeBackingImageName: + description: The volume's backing image name. + type: string + volumeCreated: + description: The volume creation time. + type: string + volumeName: + description: The volume name. + type: string + volumeSize: + description: The volume size. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: {{- include "longhorn.labels" . | nindent 4 }} + longhorn-manager: "" + name: backuptargets.longhorn.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + name: longhorn-conversion-webhook + namespace: {{ include "release_namespace" . }} + path: /v1/webhook/conversion + port: 9501 + conversionReviewVersions: + - v1beta2 + - v1beta1 + group: longhorn.io + names: + kind: BackupTarget + listKind: BackupTargetList + plural: backuptargets + shortNames: + - lhbt + singular: backuptarget + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The backup target URL + jsonPath: .spec.backupTargetURL + name: URL + type: string + - description: The backup target credential secret + jsonPath: .spec.credentialSecret + name: Credential + type: string + - description: The backup target poll interval + jsonPath: .spec.pollInterval + name: LastBackupAt + type: string + - description: Indicate whether the backup target is available or not + jsonPath: .status.available + name: Available + type: boolean + - description: The backup target last synced time + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: BackupTarget is where Longhorn stores backup target object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The backup target URL + jsonPath: .spec.backupTargetURL + name: URL + type: string + - description: The backup target credential secret + jsonPath: .spec.credentialSecret + name: Credential + type: string + - description: The backup target poll interval + jsonPath: .spec.pollInterval + name: LastBackupAt + type: string + - description: Indicate whether the backup target is available or not + jsonPath: .status.available + name: Available + type: boolean + - description: The backup target last synced time + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: BackupTarget is where Longhorn stores backup target object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BackupTargetSpec defines the desired state of the Longhorn backup target + properties: + backupTargetURL: + description: The backup target URL. + type: string + credentialSecret: + description: The backup target credential secret. + type: string + pollInterval: + description: The interval that the cluster needs to run sync with the backup target. + type: string + syncRequestedAt: + description: The time to request run sync the remote backup target. + format: date-time + nullable: true + type: string + type: object + status: + description: BackupTargetStatus defines the observed state of the Longhorn backup target + properties: + available: + description: Available indicates if the remote backup target is available or not. + type: boolean + conditions: + description: Records the reason on why the backup target is unavailable. + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + lastSyncedAt: + description: The last time that the controller synced with the remote backup target. + format: date-time + nullable: true + type: string + ownerID: + description: The node ID on which the controller is responsible to reconcile this backup target CR. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: {{- include "longhorn.labels" . | nindent 4 }} + longhorn-manager: "" + name: backupvolumes.longhorn.io +spec: + group: longhorn.io + names: + kind: BackupVolume + listKind: BackupVolumeList + plural: backupvolumes + shortNames: + - lhbv + singular: backupvolume + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The backup volume creation time + jsonPath: .status.createdAt + name: CreatedAt + type: string + - description: The backup volume last backup name + jsonPath: .status.lastBackupName + name: LastBackupName + type: string + - description: The backup volume last backup time + jsonPath: .status.lastBackupAt + name: LastBackupAt + type: string + - description: The backup volume last synced time + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: BackupVolume is where Longhorn stores backup volume object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The backup volume creation time + jsonPath: .status.createdAt + name: CreatedAt + type: string + - description: The backup volume last backup name + jsonPath: .status.lastBackupName + name: LastBackupName + type: string + - description: The backup volume last backup time + jsonPath: .status.lastBackupAt + name: LastBackupAt + type: string + - description: The backup volume last synced time + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: BackupVolume is where Longhorn stores backup volume object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BackupVolumeSpec defines the desired state of the Longhorn backup volume + properties: + syncRequestedAt: + description: The time to request run sync the remote backup volume. + format: date-time + nullable: true + type: string + type: object + status: + description: BackupVolumeStatus defines the observed state of the Longhorn backup volume + properties: + backingImageChecksum: + description: the backing image checksum. + type: string + backingImageName: + description: The backing image name. + type: string + createdAt: + description: The backup volume creation time. + type: string + dataStored: + description: The backup volume block count. + type: string + labels: + additionalProperties: + type: string + description: The backup volume labels. + nullable: true + type: object + lastBackupAt: + description: The latest volume backup time. + type: string + lastBackupName: + description: The latest volume backup name. + type: string + lastModificationTime: + description: The backup volume config last modification time. + format: date-time + nullable: true + type: string + lastSyncedAt: + description: The last time that the backup volume was synced into the cluster. + format: date-time + nullable: true + type: string + messages: + additionalProperties: + type: string + description: The error messages when call longhorn engine on list or inspect backup volumes. + nullable: true + type: object + ownerID: + description: The node ID on which the controller is responsible to reconcile this backup volume CR. + type: string + size: + description: The backup volume size. + type: string + storageClassName: + description: the storage class name of pv/pvc binding with the volume. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: {{- include "longhorn.labels" . | nindent 4 }} + longhorn-manager: "" + name: engineimages.longhorn.io +spec: + preserveUnknownFields: false + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + name: longhorn-conversion-webhook + namespace: {{ include "release_namespace" . }} + path: /v1/webhook/conversion + port: 9501 + conversionReviewVersions: + - v1beta2 + - v1beta1 + group: longhorn.io + names: + kind: EngineImage + listKind: EngineImageList + plural: engineimages + shortNames: + - lhei + singular: engineimage + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: State of the engine image + jsonPath: .status.state + name: State + type: string + - description: The Longhorn engine image + jsonPath: .spec.image + name: Image + type: string + - description: Number of resources using the engine image + jsonPath: .status.refCount + name: RefCount + type: integer + - description: The build date of the engine image + jsonPath: .status.buildDate + name: BuildDate + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: EngineImage is where Longhorn stores engine image object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: Compatibility of the engine image + jsonPath: .status.incompatible + name: Incompatible + type: boolean + - description: State of the engine image + jsonPath: .status.state + name: State + type: string + - description: The Longhorn engine image + jsonPath: .spec.image + name: Image + type: string + - description: Number of resources using the engine image + jsonPath: .status.refCount + name: RefCount + type: integer + - description: The build date of the engine image + jsonPath: .status.buildDate + name: BuildDate + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: EngineImage is where Longhorn stores engine image object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: EngineImageSpec defines the desired state of the Longhorn engine image + properties: + image: + minLength: 1 + type: string + required: + - image + type: object + status: + description: EngineImageStatus defines the observed state of the Longhorn engine image + properties: + buildDate: + type: string + cliAPIMinVersion: + type: integer + cliAPIVersion: + type: integer + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + controllerAPIMinVersion: + type: integer + controllerAPIVersion: + type: integer + dataFormatMinVersion: + type: integer + dataFormatVersion: + type: integer + gitCommit: + type: string + incompatible: + type: boolean + noRefSince: + type: string + nodeDeploymentMap: + additionalProperties: + type: boolean + nullable: true + type: object + ownerID: + type: string + refCount: + type: integer + state: + type: string + version: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + labels: {{- include "longhorn.labels" . | nindent 4 }} + longhorn-manager: "" + name: engines.longhorn.io +spec: + group: longhorn.io + names: + kind: Engine + listKind: EngineList + plural: engines + shortNames: + - lhe + singular: engine + preserveUnknownFields: false + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The current state of the engine + jsonPath: .status.currentState + name: State + type: string + - description: The node that the engine is on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The instance manager of the engine + jsonPath: .status.instanceManagerName + name: InstanceManager + type: string + - description: The current image of the engine + jsonPath: .status.currentImage + name: Image + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: Engine is where Longhorn stores engine object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The data engine of the engine + jsonPath: .spec.dataEngine + name: Data Engine + type: string + - description: The current state of the engine + jsonPath: .status.currentState + name: State + type: string + - description: The node that the engine is on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The instance manager of the engine + jsonPath: .status.instanceManagerName + name: InstanceManager + type: string + - description: The current image of the engine + jsonPath: .status.currentImage + name: Image + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: Engine is where Longhorn stores engine object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: EngineSpec defines the desired state of the Longhorn engine + properties: + active: + type: boolean + backendStoreDriver: + description: 'Deprecated: Replaced by field `dataEngine`.' + type: string + backupVolume: + type: string + dataEngine: + enum: + - v1 + - v2 + type: string + desireState: + type: string + disableFrontend: + type: boolean + engineImage: + description: 'Deprecated: Replaced by field `image`.' + type: string + frontend: + enum: + - blockdev + - iscsi + - nvmf + - "" + type: string + image: + type: string + logRequested: + type: boolean + nodeID: + type: string + replicaAddressMap: + additionalProperties: + type: string + type: object + requestedBackupRestore: + type: string + requestedDataSource: + type: string + revisionCounterDisabled: + type: boolean + salvageRequested: + type: boolean + snapshotMaxCount: + type: integer + snapshotMaxSize: + format: int64 + type: string + unmapMarkSnapChainRemovedEnabled: + type: boolean + upgradedReplicaAddressMap: + additionalProperties: + type: string + type: object + volumeName: + type: string + volumeSize: + format: int64 + type: string + type: object + status: + description: EngineStatus defines the observed state of the Longhorn engine + properties: + backupStatus: + additionalProperties: + properties: + backupURL: + type: string + error: + type: string + progress: + type: integer + replicaAddress: + type: string + snapshotName: + type: string + state: + type: string + type: object + nullable: true + type: object + cloneStatus: + additionalProperties: + properties: + error: + type: string + fromReplicaAddress: + type: string + isCloning: + type: boolean + progress: + type: integer + snapshotName: + type: string + state: + type: string + type: object + nullable: true + type: object + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + currentImage: + type: string + currentReplicaAddressMap: + additionalProperties: + type: string + nullable: true + type: object + currentSize: + format: int64 + type: string + currentState: + type: string + endpoint: + type: string + instanceManagerName: + type: string + ip: + type: string + isExpanding: + type: boolean + lastExpansionError: + type: string + lastExpansionFailedAt: + type: string + lastRestoredBackup: + type: string + logFetched: + type: boolean + ownerID: + type: string + port: + type: integer + purgeStatus: + additionalProperties: + properties: + error: + type: string + isPurging: + type: boolean + progress: + type: integer + state: + type: string + type: object + nullable: true + type: object + rebuildStatus: + additionalProperties: + properties: + error: + type: string + fromReplicaAddress: + type: string + isRebuilding: + type: boolean + progress: + type: integer + state: + type: string + type: object + nullable: true + type: object + replicaModeMap: + additionalProperties: + type: string + nullable: true + type: object + replicaTransitionTimeMap: + additionalProperties: + type: string + description: ReplicaTransitionTimeMap records the time a replica in ReplicaModeMap transitions from one mode to another (or from not being in the ReplicaModeMap to being in it). This information is sometimes required by other controllers (e.g. the volume controller uses it to determine the correct value for replica.Spec.lastHealthyAt). + type: object + restoreStatus: + additionalProperties: + properties: + backupURL: + type: string + currentRestoringBackup: + type: string + error: + type: string + filename: + type: string + isRestoring: + type: boolean + lastRestored: + type: string + progress: + type: integer + state: + type: string + type: object + nullable: true + type: object + salvageExecuted: + type: boolean + snapshotMaxCount: + type: integer + snapshotMaxSize: + format: int64 + type: string + snapshots: + additionalProperties: + properties: + children: + additionalProperties: + type: boolean + nullable: true + type: object + created: + type: string + labels: + additionalProperties: + type: string + nullable: true + type: object + name: + type: string + parent: + type: string + removed: + type: boolean + size: + type: string + usercreated: + type: boolean + type: object + nullable: true + type: object + snapshotsError: + type: string + started: + type: boolean + storageIP: + type: string + unmapMarkSnapChainRemovedEnabled: + type: boolean + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + labels: {{- include "longhorn.labels" . | nindent 4 }} + longhorn-manager: "" + name: instancemanagers.longhorn.io +spec: + group: longhorn.io + names: + kind: InstanceManager + listKind: InstanceManagerList + plural: instancemanagers + shortNames: + - lhim + singular: instancemanager + preserveUnknownFields: false + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The state of the instance manager + jsonPath: .status.currentState + name: State + type: string + - description: The type of the instance manager (engine or replica) + jsonPath: .spec.type + name: Type + type: string + - description: The node that the instance manager is running on + jsonPath: .spec.nodeID + name: Node + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: InstanceManager is where Longhorn stores instance manager object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The data engine of the instance manager + jsonPath: .spec.dataEngine + name: Data Engine + type: string + - description: The state of the instance manager + jsonPath: .status.currentState + name: State + type: string + - description: The type of the instance manager (engine or replica) + jsonPath: .spec.type + name: Type + type: string + - description: The node that the instance manager is running on + jsonPath: .spec.nodeID + name: Node + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: InstanceManager is where Longhorn stores instance manager object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: InstanceManagerSpec defines the desired state of the Longhorn instance manager + properties: + dataEngine: + type: string + image: + type: string + nodeID: + type: string + type: + enum: + - aio + - engine + - replica + type: string + type: object + status: + description: InstanceManagerStatus defines the observed state of the Longhorn instance manager + properties: + apiMinVersion: + type: integer + apiVersion: + type: integer + proxyApiMinVersion: + type: integer + proxyApiVersion: + type: integer + currentState: + type: string + instanceEngines: + additionalProperties: + properties: + spec: + properties: + backendStoreDriver: + description: 'Deprecated: Replaced by field `dataEngine`.' + type: string + dataEngine: + type: string + name: + type: string + type: object + status: + properties: + conditions: + additionalProperties: + type: boolean + nullable: true + type: object + endpoint: + type: string + errorMsg: + type: string + listen: + type: string + portEnd: + format: int32 + type: integer + portStart: + format: int32 + type: integer + resourceVersion: + format: int64 + type: integer + state: + type: string + type: + type: string + type: object + type: object + nullable: true + type: object + instanceReplicas: + additionalProperties: + properties: + spec: + properties: + backendStoreDriver: + description: 'Deprecated: Replaced by field `dataEngine`.' + type: string + dataEngine: + type: string + name: + type: string + type: object + status: + properties: + conditions: + additionalProperties: + type: boolean + nullable: true + type: object + endpoint: + type: string + errorMsg: + type: string + listen: + type: string + portEnd: + format: int32 + type: integer + portStart: + format: int32 + type: integer + resourceVersion: + format: int64 + type: integer + state: + type: string + type: + type: string + type: object + type: object + nullable: true + type: object + instances: + additionalProperties: + properties: + spec: + properties: + backendStoreDriver: + description: 'Deprecated: Replaced by field `dataEngine`.' + type: string + dataEngine: + type: string + name: + type: string + type: object + status: + properties: + conditions: + additionalProperties: + type: boolean + nullable: true + type: object + endpoint: + type: string + errorMsg: + type: string + listen: + type: string + portEnd: + format: int32 + type: integer + portStart: + format: int32 + type: integer + resourceVersion: + format: int64 + type: integer + state: + type: string + type: + type: string + type: object + type: object + nullable: true + description: 'Deprecated: Replaced by InstanceEngines and InstanceReplicas' + type: object + ip: + type: string + ownerID: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: {{- include "longhorn.labels" . | nindent 4 }} + longhorn-manager: "" + name: nodes.longhorn.io +spec: + preserveUnknownFields: false + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + name: longhorn-conversion-webhook + namespace: {{ include "release_namespace" . }} + path: /v1/webhook/conversion + port: 9501 + conversionReviewVersions: + - v1beta2 + - v1beta1 + group: longhorn.io + names: + kind: Node + listKind: NodeList + plural: nodes + shortNames: + - lhn + singular: node + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Indicate whether the node is ready + jsonPath: .status.conditions['Ready']['status'] + name: Ready + type: string + - description: Indicate whether the user disabled/enabled replica scheduling for the node + jsonPath: .spec.allowScheduling + name: AllowScheduling + type: boolean + - description: Indicate whether Longhorn can schedule replicas on the node + jsonPath: .status.conditions['Schedulable']['status'] + name: Schedulable + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: Node is where Longhorn stores Longhorn node object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: Indicate whether the node is ready + jsonPath: .status.conditions[?(@.type=='Ready')].status + name: Ready + type: string + - description: Indicate whether the user disabled/enabled replica scheduling for the node + jsonPath: .spec.allowScheduling + name: AllowScheduling + type: boolean + - description: Indicate whether Longhorn can schedule replicas on the node + jsonPath: .status.conditions[?(@.type=='Schedulable')].status + name: Schedulable + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: Node is where Longhorn stores Longhorn node object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: NodeSpec defines the desired state of the Longhorn node + properties: + allowScheduling: + description: Allow scheduling replicas on the node. + type: boolean + disks: + additionalProperties: + properties: + allowScheduling: + type: boolean + diskType: + enum: + - filesystem + - block + type: string + evictionRequested: + type: boolean + path: + type: string + storageReserved: + format: int64 + type: integer + tags: + items: + type: string + type: array + type: object + type: object + evictionRequested: + type: boolean + instanceManagerCPURequest: + type: integer + name: + type: string + tags: + items: + type: string + type: array + type: object + status: + description: NodeStatus defines the observed state of the Longhorn node + properties: + autoEvicting: + type: boolean + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + diskStatus: + additionalProperties: + properties: + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + diskType: + type: string + diskUUID: + type: string + filesystemType: + type: string + scheduledReplica: + additionalProperties: + format: int64 + type: integer + nullable: true + type: object + storageAvailable: + format: int64 + type: integer + storageMaximum: + format: int64 + type: integer + storageScheduled: + format: int64 + type: integer + type: object + description: The status of the disks on the node. + nullable: true + type: object + region: + description: The Region of the node. + type: string + snapshotCheckStatus: + description: The status of the snapshot integrity check. + properties: + lastPeriodicCheckedAt: + format: date-time + type: string + type: object + zone: + description: The Zone of the node. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: {{- include "longhorn.labels" . | nindent 4 }} + longhorn-manager: "" + name: orphans.longhorn.io +spec: + group: longhorn.io + names: + kind: Orphan + listKind: OrphanList + plural: orphans + shortNames: + - lho + singular: orphan + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The type of the orphan + jsonPath: .spec.orphanType + name: Type + type: string + - description: The node that the orphan is on + jsonPath: .spec.nodeID + name: Node + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: Orphan is where Longhorn stores orphan object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: OrphanSpec defines the desired state of the Longhorn orphaned data + properties: + nodeID: + description: The node ID on which the controller is responsible to reconcile this orphan CR. + type: string + orphanType: + description: The type of the orphaned data. Can be "replica". + type: string + parameters: + additionalProperties: + type: string + description: The parameters of the orphaned data + type: object + type: object + status: + description: OrphanStatus defines the observed state of the Longhorn orphaned data + properties: + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + ownerID: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + longhorn-manager: "" + name: recurringjobs.longhorn.io +spec: + group: longhorn.io + names: + kind: RecurringJob + listKind: RecurringJobList + plural: recurringjobs + shortNames: + - lhrj + singular: recurringjob + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Sets groupings to the jobs. When set to "default" group will be added to the volume label when no other job label exist in volume + jsonPath: .spec.groups + name: Groups + type: string + - description: Should be one of "backup" or "snapshot" + jsonPath: .spec.task + name: Task + type: string + - description: The cron expression represents recurring job scheduling + jsonPath: .spec.cron + name: Cron + type: string + - description: The number of snapshots/backups to keep for the volume + jsonPath: .spec.retain + name: Retain + type: integer + - description: The concurrent job to run by each cron job + jsonPath: .spec.concurrency + name: Concurrency + type: integer + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - description: Specify the labels + jsonPath: .spec.labels + name: Labels + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: RecurringJob is where Longhorn stores recurring job object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: Sets groupings to the jobs. When set to "default" group will be added to the volume label when no other job label exist in volume + jsonPath: .spec.groups + name: Groups + type: string + - description: Should be one of "snapshot", "snapshot-force-create", "snapshot-cleanup", "snapshot-delete", "backup", "backup-force-create" or "filesystem-trim" + jsonPath: .spec.task + name: Task + type: string + - description: The cron expression represents recurring job scheduling + jsonPath: .spec.cron + name: Cron + type: string + - description: The number of snapshots/backups to keep for the volume + jsonPath: .spec.retain + name: Retain + type: integer + - description: The concurrent job to run by each cron job + jsonPath: .spec.concurrency + name: Concurrency + type: integer + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - description: Specify the labels + jsonPath: .spec.labels + name: Labels + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: RecurringJob is where Longhorn stores recurring job object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: RecurringJobSpec defines the desired state of the Longhorn recurring job + properties: + concurrency: + description: The concurrency of taking the snapshot/backup. + type: integer + cron: + description: The cron setting. + type: string + groups: + description: The recurring job group. + items: + type: string + type: array + labels: + additionalProperties: + type: string + description: The label of the snapshot/backup. + type: object + name: + description: The recurring job name. + type: string + retain: + description: The retain count of the snapshot/backup. + type: integer + task: + description: The recurring job task. Can be "snapshot", "snapshot-force-create", "snapshot-cleanup", "snapshot-delete", "backup", "backup-force-create" or "filesystem-trim" + enum: + - snapshot + - snapshot-force-create + - snapshot-cleanup + - snapshot-delete + - backup + - backup-force-create + - filesystem-trim + type: string + type: object + status: + description: RecurringJobStatus defines the observed state of the Longhorn recurring job + properties: + ownerID: + description: The owner ID which is responsible to reconcile this recurring job CR. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + labels: {{- include "longhorn.labels" . | nindent 4 }} + longhorn-manager: "" + name: replicas.longhorn.io +spec: + group: longhorn.io + names: + kind: Replica + listKind: ReplicaList + plural: replicas + shortNames: + - lhr + singular: replica + preserveUnknownFields: false + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The current state of the replica + jsonPath: .status.currentState + name: State + type: string + - description: The node that the replica is on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The disk that the replica is on + jsonPath: .spec.diskID + name: Disk + type: string + - description: The instance manager of the replica + jsonPath: .status.instanceManagerName + name: InstanceManager + type: string + - description: The current image of the replica + jsonPath: .status.currentImage + name: Image + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: Replica is where Longhorn stores replica object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The data engine of the replica + jsonPath: .spec.dataEngine + name: Data Engine + type: string + - description: The current state of the replica + jsonPath: .status.currentState + name: State + type: string + - description: The node that the replica is on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The disk that the replica is on + jsonPath: .spec.diskID + name: Disk + type: string + - description: The instance manager of the replica + jsonPath: .status.instanceManagerName + name: InstanceManager + type: string + - description: The current image of the replica + jsonPath: .status.currentImage + name: Image + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: Replica is where Longhorn stores replica object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: ReplicaSpec defines the desired state of the Longhorn replica + properties: + active: + type: boolean + backendStoreDriver: + description: 'Deprecated: Replaced by field `dataEngine`.' + type: string + backingImage: + type: string + dataDirectoryName: + type: string + dataEngine: + enum: + - v1 + - v2 + type: string + desireState: + type: string + diskID: + type: string + diskPath: + type: string + engineImage: + description: 'Deprecated: Replaced by field `image`.' + type: string + engineName: + type: string + evictionRequested: + type: boolean + failedAt: + description: FailedAt is set when a running replica fails or when a running engine is unable to use a replica for any reason. FailedAt indicates the time the failure occurred. When FailedAt is set, a replica is likely to have useful (though possibly stale) data. A replica with FailedAt set must be rebuilt from a non-failed replica (or it can be used in a salvage if all replicas are failed). FailedAt is cleared before a rebuild or salvage. FailedAt may be later than the corresponding entry in an engine's replicaTransitionTimeMap because it is set when the volume controller acknowledges the change. + type: string + hardNodeAffinity: + type: string + healthyAt: + description: HealthyAt is set the first time a replica becomes read/write in an engine after creation or rebuild. HealthyAt indicates the time the last successful rebuild occurred. When HealthyAt is set, a replica is likely to have useful (though possibly stale) data. HealthyAt is cleared before a rebuild. HealthyAt may be later than the corresponding entry in an engine's replicaTransitionTimeMap because it is set when the volume controller acknowledges the change. + type: string + image: + type: string + lastFailedAt: + description: LastFailedAt is always set at the same time as FailedAt. Unlike FailedAt, LastFailedAt is never cleared. LastFailedAt is not a reliable indicator of the state of a replica's data. For example, a replica with LastFailedAt may already be healthy and in use again. However, because it is never cleared, it can be compared to LastHealthyAt to help prevent dangerous replica deletion in some corner cases. LastFailedAt may be later than the corresponding entry in an engine's replicaTransitionTimeMap because it is set when the volume controller acknowledges the change. + type: string + lastHealthyAt: + description: LastHealthyAt is set every time a replica becomes read/write in an engine. Unlike HealthyAt, LastHealthyAt is never cleared. LastHealthyAt is not a reliable indicator of the state of a replica's data. For example, a replica with LastHealthyAt set may be in the middle of a rebuild. However, because it is never cleared, it can be compared to LastFailedAt to help prevent dangerous replica deletion in some corner cases. LastHealthyAt may be later than the corresponding entry in an engine's replicaTransitionTimeMap because it is set when the volume controller acknowledges the change. + type: string + logRequested: + type: boolean + nodeID: + type: string + rebuildRetryCount: + type: integer + revisionCounterDisabled: + type: boolean + salvageRequested: + type: boolean + snapshotMaxCount: + type: integer + snapshotMaxSize: + format: int64 + type: string + unmapMarkDiskChainRemovedEnabled: + type: boolean + volumeName: + type: string + volumeSize: + format: int64 + type: string + type: object + status: + description: ReplicaStatus defines the observed state of the Longhorn replica + properties: + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + currentImage: + type: string + currentState: + type: string + evictionRequested: + description: 'Deprecated: Replaced by field `spec.evictionRequested`.' + type: boolean + instanceManagerName: + type: string + ip: + type: string + logFetched: + type: boolean + ownerID: + type: string + port: + type: integer + salvageExecuted: + type: boolean + started: + type: boolean + storageIP: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + labels: {{- include "longhorn.labels" . | nindent 4 }} + longhorn-manager: "" + name: settings.longhorn.io +spec: + group: longhorn.io + names: + kind: Setting + listKind: SettingList + plural: settings + shortNames: + - lhs + singular: setting + preserveUnknownFields: false + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The value of the setting + jsonPath: .value + name: Value + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: Setting is where Longhorn stores setting object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + value: + type: string + required: + - value + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The value of the setting + jsonPath: .value + name: Value + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: Setting is where Longhorn stores setting object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + value: + description: The value of the setting. + type: string + required: + - value + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: {{- include "longhorn.labels" . | nindent 4 }} + longhorn-manager: "" + name: sharemanagers.longhorn.io +spec: + group: longhorn.io + names: + kind: ShareManager + listKind: ShareManagerList + plural: sharemanagers + shortNames: + - lhsm + singular: sharemanager + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The state of the share manager + jsonPath: .status.state + name: State + type: string + - description: The node that the share manager is owned by + jsonPath: .status.ownerID + name: Node + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: ShareManager is where Longhorn stores share manager object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The state of the share manager + jsonPath: .status.state + name: State + type: string + - description: The node that the share manager is owned by + jsonPath: .status.ownerID + name: Node + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: ShareManager is where Longhorn stores share manager object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: ShareManagerSpec defines the desired state of the Longhorn share manager + properties: + image: + description: Share manager image used for creating a share manager pod + type: string + type: object + status: + description: ShareManagerStatus defines the observed state of the Longhorn share manager + properties: + endpoint: + description: NFS endpoint that can access the mounted filesystem of the volume + type: string + ownerID: + description: The node ID on which the controller is responsible to reconcile this share manager resource + type: string + state: + description: The state of the share manager resource + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: {{- include "longhorn.labels" . | nindent 4 }} + longhorn-manager: "" + name: snapshots.longhorn.io +spec: + group: longhorn.io + names: + kind: Snapshot + listKind: SnapshotList + plural: snapshots + shortNames: + - lhsnap + singular: snapshot + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The volume that this snapshot belongs to + jsonPath: .spec.volume + name: Volume + type: string + - description: Timestamp when the point-in-time snapshot was taken + jsonPath: .status.creationTime + name: CreationTime + type: string + - description: Indicates if the snapshot is ready to be used to restore/backup a volume + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: Represents the minimum size of volume required to rehydrate from this snapshot + jsonPath: .status.restoreSize + name: RestoreSize + type: string + - description: The actual size of the snapshot + jsonPath: .status.size + name: Size + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: Snapshot is the Schema for the snapshots API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: SnapshotSpec defines the desired state of Longhorn Snapshot + properties: + createSnapshot: + description: require creating a new snapshot + type: boolean + labels: + additionalProperties: + type: string + description: The labels of snapshot + nullable: true + type: object + volume: + description: the volume that this snapshot belongs to. This field is immutable after creation. Required + type: string + required: + - volume + type: object + status: + description: SnapshotStatus defines the observed state of Longhorn Snapshot + properties: + checksum: + type: string + children: + additionalProperties: + type: boolean + nullable: true + type: object + creationTime: + type: string + error: + type: string + labels: + additionalProperties: + type: string + nullable: true + type: object + markRemoved: + type: boolean + ownerID: + type: string + parent: + type: string + readyToUse: + type: boolean + restoreSize: + format: int64 + type: integer + size: + format: int64 + type: integer + userCreated: + type: boolean + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: {{- include "longhorn.labels" . | nindent 4 }} + longhorn-manager: "" + name: supportbundles.longhorn.io +spec: + group: longhorn.io + names: + kind: SupportBundle + listKind: SupportBundleList + plural: supportbundles + shortNames: + - lhbundle + singular: supportbundle + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The state of the support bundle + jsonPath: .status.state + name: State + type: string + - description: The issue URL + jsonPath: .spec.issueURL + name: Issue + type: string + - description: A brief description of the issue + jsonPath: .spec.description + name: Description + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: SupportBundle is where Longhorn stores support bundle object + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: SupportBundleSpec defines the desired state of the Longhorn SupportBundle + properties: + description: + description: A brief description of the issue + type: string + issueURL: + description: The issue URL + nullable: true + type: string + nodeID: + description: The preferred responsible controller node ID. + type: string + required: + - description + type: object + status: + description: SupportBundleStatus defines the observed state of the Longhorn SupportBundle + properties: + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + type: array + filename: + type: string + filesize: + format: int64 + type: integer + image: + description: The support bundle manager image + type: string + managerIP: + description: The support bundle manager IP + type: string + ownerID: + description: The current responsible controller node ID + type: string + progress: + type: integer + state: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: {{- include "longhorn.labels" . | nindent 4 }} + longhorn-manager: "" + name: systembackups.longhorn.io +spec: + group: longhorn.io + names: + kind: SystemBackup + listKind: SystemBackupList + plural: systembackups + shortNames: + - lhsb + singular: systembackup + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The system backup Longhorn version + jsonPath: .status.version + name: Version + type: string + - description: The system backup state + jsonPath: .status.state + name: State + type: string + - description: The system backup creation time + jsonPath: .status.createdAt + name: Created + type: string + - description: The last time that the system backup was synced into the cluster + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: SystemBackup is where Longhorn stores system backup object + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: SystemBackupSpec defines the desired state of the Longhorn SystemBackup + properties: + volumeBackupPolicy: + description: The create volume backup policy Can be "if-not-present", "always" or "disabled" + nullable: true + type: string + type: object + status: + description: SystemBackupStatus defines the observed state of the Longhorn SystemBackup + properties: + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + createdAt: + description: The system backup creation time. + format: date-time + type: string + gitCommit: + description: The saved Longhorn manager git commit. + nullable: true + type: string + lastSyncedAt: + description: The last time that the system backup was synced into the cluster. + format: date-time + nullable: true + type: string + managerImage: + description: The saved manager image. + type: string + ownerID: + description: The node ID of the responsible controller to reconcile this SystemBackup. + type: string + state: + description: The system backup state. + type: string + version: + description: The saved Longhorn version. + nullable: true + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: {{- include "longhorn.labels" . | nindent 4 }} + longhorn-manager: "" + name: systemrestores.longhorn.io +spec: + group: longhorn.io + names: + kind: SystemRestore + listKind: SystemRestoreList + plural: systemrestores + shortNames: + - lhsr + singular: systemrestore + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The system restore state + jsonPath: .status.state + name: State + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: SystemRestore is where Longhorn stores system restore object + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: SystemRestoreSpec defines the desired state of the Longhorn SystemRestore + properties: + systemBackup: + description: The system backup name in the object store. + type: string + required: + - systemBackup + type: object + status: + description: SystemRestoreStatus defines the observed state of the Longhorn SystemRestore + properties: + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + ownerID: + description: The node ID of the responsible controller to reconcile this SystemRestore. + type: string + sourceURL: + description: The source system backup URL. + type: string + state: + description: The system restore state. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + labels: {{- include "longhorn.labels" . | nindent 4 }} + longhorn-manager: "" + name: volumes.longhorn.io +spec: + preserveUnknownFields: false + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + name: longhorn-conversion-webhook + namespace: {{ include "release_namespace" . }} + path: /v1/webhook/conversion + port: 9501 + conversionReviewVersions: + - v1beta2 + - v1beta1 + group: longhorn.io + names: + kind: Volume + listKind: VolumeList + plural: volumes + shortNames: + - lhv + singular: volume + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The state of the volume + jsonPath: .status.state + name: State + type: string + - description: The robustness of the volume + jsonPath: .status.robustness + name: Robustness + type: string + - description: The scheduled condition of the volume + jsonPath: .status.conditions['scheduled']['status'] + name: Scheduled + type: string + - description: The size of the volume + jsonPath: .spec.size + name: Size + type: string + - description: The node that the volume is currently attaching to + jsonPath: .status.currentNodeID + name: Node + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: Volume is where Longhorn stores volume object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The data engine of the volume + jsonPath: .spec.dataEngine + name: Data Engine + type: string + - description: The state of the volume + jsonPath: .status.state + name: State + type: string + - description: The robustness of the volume + jsonPath: .status.robustness + name: Robustness + type: string + - description: The scheduled condition of the volume + jsonPath: .status.conditions[?(@.type=='Schedulable')].status + name: Scheduled + type: string + - description: The size of the volume + jsonPath: .spec.size + name: Size + type: string + - description: The node that the volume is currently attaching to + jsonPath: .status.currentNodeID + name: Node + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: Volume is where Longhorn stores volume object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: VolumeSpec defines the desired state of the Longhorn volume + properties: + Standby: + type: boolean + accessMode: + enum: + - rwo + - rwx + type: string + backendStoreDriver: + description: 'Deprecated: Replaced by field `dataEngine`.' + type: string + backingImage: + type: string + backupCompressionMethod: + enum: + - none + - lz4 + - gzip + type: string + dataEngine: + enum: + - v1 + - v2 + type: string + dataLocality: + enum: + - disabled + - best-effort + - strict-local + type: string + dataSource: + type: string + disableFrontend: + type: boolean + diskSelector: + items: + type: string + type: array + encrypted: + type: boolean + engineImage: + description: 'Deprecated: Replaced by field `image`.' + type: string + fromBackup: + type: string + frontend: + enum: + - blockdev + - iscsi + - nvmf + - "" + type: string + image: + type: string + lastAttachedBy: + type: string + migratable: + type: boolean + migrationNodeID: + type: string + nodeID: + type: string + nodeSelector: + items: + type: string + type: array + numberOfReplicas: + type: integer + offlineReplicaRebuilding: + description: OfflineReplicaRebuilding is used to determine if the offline replica rebuilding feature is enabled or not + enum: + - ignored + - disabled + - enabled + type: string + replicaAutoBalance: + enum: + - ignored + - disabled + - least-effort + - best-effort + type: string + replicaDiskSoftAntiAffinity: + description: Replica disk soft anti affinity of the volume. Set enabled to allow replicas to be scheduled in the same disk. + enum: + - ignored + - enabled + - disabled + type: string + replicaSoftAntiAffinity: + description: Replica soft anti affinity of the volume. Set enabled to allow replicas to be scheduled on the same node. + enum: + - ignored + - enabled + - disabled + type: string + replicaZoneSoftAntiAffinity: + description: Replica zone soft anti affinity of the volume. Set enabled to allow replicas to be scheduled in the same zone. + enum: + - ignored + - enabled + - disabled + type: string + restoreVolumeRecurringJob: + enum: + - ignored + - enabled + - disabled + type: string + revisionCounterDisabled: + type: boolean + size: + format: int64 + type: string + snapshotDataIntegrity: + enum: + - ignored + - disabled + - enabled + - fast-check + type: string + snapshotMaxCount: + type: integer + snapshotMaxSize: + format: int64 + type: string + staleReplicaTimeout: + type: integer + unmapMarkSnapChainRemoved: + enum: + - ignored + - disabled + - enabled + type: string + type: object + status: + description: VolumeStatus defines the observed state of the Longhorn volume + properties: + actualSize: + format: int64 + type: integer + cloneStatus: + properties: + snapshot: + type: string + sourceVolume: + type: string + state: + type: string + type: object + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + currentImage: + type: string + currentMigrationNodeID: + description: the node that this volume is currently migrating to + type: string + currentNodeID: + type: string + expansionRequired: + type: boolean + frontendDisabled: + type: boolean + isStandby: + type: boolean + kubernetesStatus: + properties: + lastPVCRefAt: + type: string + lastPodRefAt: + type: string + namespace: + description: determine if PVC/Namespace is history or not + type: string + pvName: + type: string + pvStatus: + type: string + pvcName: + type: string + workloadsStatus: + description: determine if Pod/Workload is history or not + items: + properties: + podName: + type: string + podStatus: + type: string + workloadName: + type: string + workloadType: + type: string + type: object + nullable: true + type: array + type: object + lastBackup: + type: string + lastBackupAt: + type: string + lastDegradedAt: + type: string + offlineReplicaRebuildingRequired: + type: boolean + ownerID: + type: string + pendingNodeID: + description: Deprecated. + type: string + remountRequestedAt: + type: string + restoreInitiated: + type: boolean + restoreRequired: + type: boolean + robustness: + type: string + shareEndpoint: + type: string + shareState: + type: string + state: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: {{- include "longhorn.labels" . | nindent 4 }} + longhorn-manager: "" + name: volumeattachments.longhorn.io +spec: + group: longhorn.io + names: + kind: VolumeAttachment + listKind: VolumeAttachmentList + plural: volumeattachments + shortNames: + - lhva + singular: volumeattachment + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: VolumeAttachment stores attachment information of a Longhorn volume + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: VolumeAttachmentSpec defines the desired state of Longhorn VolumeAttachment + properties: + attachmentTickets: + additionalProperties: + properties: + generation: + description: A sequence number representing a specific generation of the desired state. Populated by the system. Read-only. + format: int64 + type: integer + id: + description: The unique ID of this attachment. Used to differentiate different attachments of the same volume. + type: string + nodeID: + description: The node that this attachment is requesting + type: string + parameters: + additionalProperties: + type: string + description: Optional additional parameter for this attachment + type: object + type: + type: string + type: object + type: object + volume: + description: The name of Longhorn volume of this VolumeAttachment + type: string + required: + - volume + type: object + status: + description: VolumeAttachmentStatus defines the observed state of Longhorn VolumeAttachment + properties: + attachmentTicketStatuses: + additionalProperties: + properties: + conditions: + description: Record any error when trying to fulfill this attachment + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + generation: + description: A sequence number representing a specific generation of the desired state. Populated by the system. Read-only. + format: int64 + type: integer + id: + description: The unique ID of this attachment. Used to differentiate different attachments of the same volume. + type: string + satisfied: + description: Indicate whether this attachment ticket has been satisfied + type: boolean + required: + - conditions + - satisfied + type: object + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/charts/longhorn/102.4.1+up1.6.2/.helmignore b/charts/longhorn/102.4.1+up1.6.2/.helmignore new file mode 100644 index 0000000000..f0c1319444 --- /dev/null +++ b/charts/longhorn/102.4.1+up1.6.2/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/charts/longhorn/102.4.1+up1.6.2/Chart.yaml b/charts/longhorn/102.4.1+up1.6.2/Chart.yaml new file mode 100644 index 0000000000..1bdc84ed8a --- /dev/null +++ b/charts/longhorn/102.4.1+up1.6.2/Chart.yaml @@ -0,0 +1,40 @@ +annotations: + catalog.cattle.io/auto-install: longhorn-crd=match + catalog.cattle.io/certified: rancher + catalog.cattle.io/display-name: Longhorn + catalog.cattle.io/kube-version: '>= 1.21.0-0' + catalog.cattle.io/namespace: longhorn-system + catalog.cattle.io/permits-os: linux,windows + catalog.cattle.io/provides-gvr: longhorn.io/v1beta1 + catalog.cattle.io/rancher-version: '>= 2.7.0-0 < 2.8.0-0' + catalog.cattle.io/release-name: longhorn + catalog.cattle.io/type: cluster-tool + catalog.cattle.io/upstream-version: 1.6.2 +apiVersion: v1 +appVersion: v1.6.2 +description: Longhorn is a distributed block storage system for Kubernetes. +home: https://github.com/longhorn/longhorn +icon: https://raw.githubusercontent.com/cncf/artwork/master/projects/longhorn/icon/color/longhorn-icon-color.png +keywords: +- longhorn +- storage +- distributed +- block +- device +- iscsi +- nfs +kubeVersion: '>=1.21.0-0' +maintainers: +- email: maintainers@longhorn.io + name: Longhorn maintainers +name: longhorn +sources: +- https://github.com/longhorn/longhorn +- https://github.com/longhorn/longhorn-engine +- https://github.com/longhorn/longhorn-instance-manager +- https://github.com/longhorn/longhorn-share-manager +- https://github.com/longhorn/longhorn-manager +- https://github.com/longhorn/longhorn-ui +- https://github.com/longhorn/longhorn-tests +- https://github.com/longhorn/backing-image-manager +version: 102.4.1+up1.6.2 diff --git a/charts/longhorn/102.4.1+up1.6.2/README.md b/charts/longhorn/102.4.1+up1.6.2/README.md new file mode 100644 index 0000000000..adb190be3b --- /dev/null +++ b/charts/longhorn/102.4.1+up1.6.2/README.md @@ -0,0 +1,50 @@ +# Longhorn Chart + +> **Important**: Please install the Longhorn chart in the `longhorn-system` namespace only. + +> **Warning**: Longhorn doesn't support downgrading from a higher version to a lower version. + +> **Note**: Use Helm 3 when installing and upgrading Longhorn. Helm 2 is [no longer supported](https://helm.sh/blog/helm-2-becomes-unsupported/). + +## Source Code + +Longhorn is 100% open source software. Project source code is spread across a number of repos: + +1. Longhorn Engine -- Core controller/replica logic https://github.com/longhorn/longhorn-engine +2. Longhorn Instance Manager -- Controller/replica instance lifecycle management https://github.com/longhorn/longhorn-instance-manager +3. Longhorn Share Manager -- NFS provisioner that exposes Longhorn volumes as ReadWriteMany volumes. https://github.com/longhorn/longhorn-share-manager +4. Backing Image Manager -- Backing image file lifecycle management. https://github.com/longhorn/backing-image-manager +5. Longhorn Manager -- Longhorn orchestration, includes CSI driver for Kubernetes https://github.com/longhorn/longhorn-manager +6. Longhorn UI -- Dashboard https://github.com/longhorn/longhorn-ui + +## Prerequisites + +1. A container runtime compatible with Kubernetes (Docker v1.13+, containerd v1.3.7+, etc.) +2. Kubernetes >= v1.21 +3. Make sure `bash`, `curl`, `findmnt`, `grep`, `awk` and `blkid` has been installed in all nodes of the Kubernetes cluster. +4. Make sure `open-iscsi` has been installed, and the `iscsid` daemon is running on all nodes of the Kubernetes cluster. For GKE, recommended Ubuntu as guest OS image since it contains `open-iscsi` already. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `enablePSP` set to `false` if it has been previously set to `true`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, you may have to clean up your Helm release secrets. +Upon setting `enablePSP` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Longhorn docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Uninstallation + +To prevent Longhorn from being accidentally uninstalled (which leads to data lost), we introduce a new setting, deleting-confirmation-flag. If this flag is **false**, the Longhorn uninstallation job will fail. Set this flag to **true** to allow Longhorn uninstallation. You can set this flag using setting page in Longhorn UI or `kubectl -n longhorn-system patch -p '{"value": "true"}' --type=merge lhs deleting-confirmation-flag` + +To prevent damage to the Kubernetes cluster, we recommend deleting all Kubernetes workloads using Longhorn volumes (PersistentVolume, PersistentVolumeClaim, StorageClass, Deployment, StatefulSet, DaemonSet, etc). + +From Rancher Cluster Explorer UI, navigate to Apps page, delete app `longhorn` then app `longhorn-crd` in Installed Apps tab. + +--- +Please see [link](https://github.com/longhorn/longhorn) for more information. diff --git a/charts/longhorn/102.4.1+up1.6.2/app-readme.md b/charts/longhorn/102.4.1+up1.6.2/app-readme.md new file mode 100644 index 0000000000..321e5193c4 --- /dev/null +++ b/charts/longhorn/102.4.1+up1.6.2/app-readme.md @@ -0,0 +1,27 @@ +# Longhorn + +Longhorn is a lightweight, reliable and easy to use distributed block storage system for Kubernetes. Once deployed, users can leverage persistent volumes provided by Longhorn. + +Longhorn creates a dedicated storage controller for each volume and synchronously replicates the volume across multiple replicas stored on multiple nodes. The storage controller and replicas are themselves orchestrated using Kubernetes. Longhorn supports snapshots, backups and even allows you to schedule recurring snapshots and backups! + +**Important**: Please install Longhorn chart in `longhorn-system` namespace only. + +**Warning**: Longhorn doesn't support downgrading from a higher version to a lower version. + +[Chart Documentation](https://github.com/longhorn/longhorn/blob/master/chart/README.md) + + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `enablePSP` set to `false` if it has been previously set to `true`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `enablePSP` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. \ No newline at end of file diff --git a/charts/longhorn/102.4.1+up1.6.2/questions.yaml b/charts/longhorn/102.4.1+up1.6.2/questions.yaml new file mode 100644 index 0000000000..a61312c19c --- /dev/null +++ b/charts/longhorn/102.4.1+up1.6.2/questions.yaml @@ -0,0 +1,920 @@ +categories: +- storage +namespace: longhorn-system +questions: +- variable: image.defaultImage + default: "true" + description: "Use default Longhorn images" + label: Use Default Images + type: boolean + show_subquestion_if: false + group: "Longhorn Images" + subquestions: + - variable: image.longhorn.manager.repository + default: rancher/mirrored-longhornio-longhorn-manager + description: "Repository for the Longhorn Manager image." + type: string + label: Longhorn Manager Image Repository + group: "Longhorn Images Settings" + - variable: image.longhorn.manager.tag + default: v1.6.2 + description: "Specify Longhorn Manager Image Tag" + type: string + label: Longhorn Manager Image Tag + group: "Longhorn Images Settings" + - variable: image.longhorn.engine.repository + default: rancher/mirrored-longhornio-longhorn-engine + description: "Repository for the Longhorn Engine image." + type: string + label: Longhorn Engine Image Repository + group: "Longhorn Images Settings" + - variable: image.longhorn.engine.tag + default: v1.6.2 + description: "Specify Longhorn Engine Image Tag" + type: string + label: Longhorn Engine Image Tag + group: "Longhorn Images Settings" + - variable: image.longhorn.ui.repository + default: rancher/mirrored-longhornio-longhorn-ui + description: "Repository for the Longhorn UI image." + type: string + label: Longhorn UI Image Repository + group: "Longhorn Images Settings" + - variable: image.longhorn.ui.tag + default: v1.6.2 + description: "Specify Longhorn UI Image Tag" + type: string + label: Longhorn UI Image Tag + group: "Longhorn Images Settings" + - variable: image.longhorn.instanceManager.repository + default: rancher/mirrored-longhornio-longhorn-instance-manager + description: "Repository for the Longhorn Instance Manager image." + type: string + label: Longhorn Instance Manager Image Repository + group: "Longhorn Images Settings" + - variable: image.longhorn.instanceManager.tag + default: v1.6.2 + description: "Specify Longhorn Instance Manager Image Tag" + type: string + label: Longhorn Instance Manager Image Tag + group: "Longhorn Images Settings" + - variable: image.longhorn.shareManager.repository + default: rancher/mirrored-longhornio-longhorn-share-manager + description: "Repository for the Longhorn Share Manager image." + type: string + label: Longhorn Share Manager Image Repository + group: "Longhorn Images Settings" + - variable: image.longhorn.shareManager.tag + default: v1.6.2 + description: "Specify Longhorn Share Manager Image Tag" + type: string + label: Longhorn Share Manager Image Tag + group: "Longhorn Images Settings" + - variable: image.longhorn.backingImageManager.repository + default: rancher/mirrored-longhornio-backing-image-manager + description: "Repository for the Backing Image Manager image. When unspecified, Longhorn uses the default value." + type: string + label: Longhorn Backing Image Manager Image Repository + group: "Longhorn Images Settings" + - variable: image.longhorn.backingImageManager.tag + default: v1.6.2 + description: "Specify Longhorn Backing Image Manager Image Tag" + type: string + label: Longhorn Backing Image Manager Image Tag + group: "Longhorn Images Settings" + - variable: image.longhorn.supportBundleKit.repository + default: rancher/mirrored-longhornio-support-bundle-kit + description: "Repository for the Longhorn Support Bundle Manager image." + type: string + label: Longhorn Support Bundle Kit Image Repository + group: "Longhorn Images Settings" + - variable: image.longhorn.supportBundleKit.tag + default: v0.0.37 + description: "Tag for the Longhorn Support Bundle Manager image." + type: string + label: Longhorn Support Bundle Kit Image Tag + group: "Longhorn Images Settings" + - variable: image.csi.attacher.repository + default: rancher/mirrored-longhornio-csi-attacher + description: "Repository for the CSI attacher image. When unspecified, Longhorn uses the default value." + type: string + label: Longhorn CSI Attacher Image Repository + group: "Longhorn CSI Driver Images" + - variable: image.csi.attacher.tag + default: v4.5.1 + description: "Tag for the CSI attacher image. When unspecified, Longhorn uses the default value." + type: string + label: Longhorn CSI Attacher Image Tag + group: "Longhorn CSI Driver Images" + - variable: image.csi.provisioner.repository + default: rancher/mirrored-longhornio-csi-provisioner + description: "Repository for the CSI Provisioner image. When unspecified, Longhorn uses the default value." + type: string + label: Longhorn CSI Provisioner Image Repository + group: "Longhorn CSI Driver Images" + - variable: image.csi.provisioner.tag + default: v3.6.4 + description: "Tag for the CSI Provisioner image. When unspecified, Longhorn uses the default value." + type: string + label: Longhorn CSI Provisioner Image Tag + group: "Longhorn CSI Driver Images" + - variable: image.csi.nodeDriverRegistrar.repository + default: rancher/mirrored-longhornio-csi-node-driver-registrar + description: "Repository for the CSI Node Driver Registrar image. When unspecified, Longhorn uses the default value." + type: string + label: Longhorn CSI Node Driver Registrar Image Repository + group: "Longhorn CSI Driver Images" + - variable: image.csi.nodeDriverRegistrar.tag + default: v2.9.2 + description: "Tag for the CSI Node Driver Registrar image. When unspecified, Longhorn uses the default value." + type: string + label: Longhorn CSI Node Driver Registrar Image Tag + group: "Longhorn CSI Driver Images" + - variable: image.csi.resizer.repository + default: rancher/mirrored-longhornio-csi-resizer + description: "Repository for the CSI Resizer image. When unspecified, Longhorn uses the default value." + type: string + label: Longhorn CSI Driver Resizer Image Repository + group: "Longhorn CSI Driver Images" + - variable: image.csi.resizer.tag + default: v1.10.1 + description: "Tag for the CSI Resizer image. When unspecified, Longhorn uses the default value." + type: string + label: Longhorn CSI Driver Resizer Image Tag + group: "Longhorn CSI Driver Images" + - variable: image.csi.snapshotter.repository + default: rancher/mirrored-longhornio-csi-snapshotter + description: "Repository for the CSI Snapshotter image. When unspecified, Longhorn uses the default value." + type: string + label: Longhorn CSI Driver Snapshotter Image Repository + group: "Longhorn CSI Driver Images" + - variable: image.csi.snapshotter.tag + default: v6.3.4 + description: "Tag for the CSI Snapshotter image. When unspecified, Longhorn uses the default value." + type: string + label: Longhorn CSI Driver Snapshotter Image Tag + group: "Longhorn CSI Driver Images" + - variable: image.csi.livenessProbe.repository + default: rancher/mirrored-longhornio-livenessprobe + description: "Repository for the CSI liveness probe image. When unspecified, Longhorn uses the default value." + type: string + label: Longhorn CSI Liveness Probe Image Repository + group: "Longhorn CSI Driver Images" + - variable: image.csi.livenessProbe.tag + default: v2.12.0 + description: "Tag for the CSI liveness probe image. When unspecified, Longhorn uses the default value." + type: string + label: Longhorn CSI Liveness Probe Image Tag + group: "Longhorn CSI Driver Images" + - variable: image.openshift.oauthProxy.repository + default: rancher/mirrored-longhornio-openshift-origin-oauth-proxy + description: "Repository for the OAuth Proxy image. This setting applies only to OpenShift users" + type: string + label: OpenShift OAuth Proxy Image Repository + group: "OpenShift Images" + - variable: image.openshift.oauthProxy.tag + default: 4.14 + description: "Tag for the OAuth Proxy image. This setting applies only to OpenShift users. Specify OCP/OKD version 4.1 or later." + type: string + label: OpenShift OAuth Proxy Image Tag + group: "OpenShift Images" +- variable: privateRegistry.registryUrl + label: Private registry URL + description: "URL of a private registry. When unspecified, Longhorn uses the default system registry." + group: "Private Registry Settings" + type: string + default: "" +- variable: privateRegistry.registrySecret + label: Private registry secret name + description: "Kubernetes secret that allows you to pull images from a private registry. This setting applies only when creation of private registry secrets is enabled. You must include the private registry name in the secret name." + group: "Private Registry Settings" + type: string + default: "" +- variable: privateRegistry.createSecret + default: "true" + description: "Setting that allows you to create a private registry secret." + type: boolean + group: "Private Registry Settings" + label: Create Secret for Private Registry Settings + show_subquestion_if: true + subquestions: + - variable: privateRegistry.registryUser + label: Private registry user + description: "User account used for authenticating with a private registry." + type: string + default: "" + - variable: privateRegistry.registryPasswd + label: Private registry password + description: "Password for authenticating with a private registry." + type: password + default: "" +- variable: longhorn.default_setting + default: "false" + description: "Customize the default settings before installing Longhorn for the first time. This option will only work if the cluster hasn't installed Longhorn." + label: "Customize Default Settings" + type: boolean + show_subquestion_if: true + group: "Longhorn Default Settings" + subquestions: + - variable: csi.kubeletRootDir + default: + description: "kubelet root directory. When unspecified, Longhorn uses the default value." + type: string + label: Kubelet Root Directory + group: "Longhorn CSI Driver Settings" + - variable: csi.attacherReplicaCount + type: int + default: 3 + min: 1 + max: 10 + description: "Replica count of the CSI Attacher. When unspecified, Longhorn uses the default value (\"3\")." + label: Longhorn CSI Attacher replica count + group: "Longhorn CSI Driver Settings" + - variable: csi.provisionerReplicaCount + type: int + default: 3 + min: 1 + max: 10 + description: "Replica count of the CSI Provisioner. When unspecified, Longhorn uses the default value (\"3\")." + label: Longhorn CSI Provisioner replica count + group: "Longhorn CSI Driver Settings" + - variable: csi.resizerReplicaCount + type: int + default: 3 + min: 1 + max: 10 + description: "Replica count of the CSI Resizer. When unspecified, Longhorn uses the default value (\"3\")." + label: Longhorn CSI Resizer replica count + group: "Longhorn CSI Driver Settings" + - variable: csi.snapshotterReplicaCount + type: int + default: 3 + min: 1 + max: 10 + description: "Replica count of the CSI Snapshotter. When unspecified, Longhorn uses the default value (\"3\")." + label: Longhorn CSI Snapshotter replica count + group: "Longhorn CSI Driver Settings" + - variable: defaultSettings.backupTarget + label: Backup Target + description: "Endpoint used to access the backupstore. (Options: \"NFS\", \"CIFS\", \"AWS\", \"GCP\", \"AZURE\")" + group: "Longhorn Default Settings" + type: string + default: + - variable: defaultSettings.backupTargetCredentialSecret + label: Backup Target Credential Secret + description: "Name of the Kubernetes secret associated with the backup target." + group: "Longhorn Default Settings" + type: string + default: + - variable: defaultSettings.allowRecurringJobWhileVolumeDetached + label: Allow Recurring Job While Volume Is Detached + description: 'Setting that allows Longhorn to automatically attach a volume and create snapshots or backups when recurring jobs are run.' + group: "Longhorn Default Settings" + type: boolean + default: "false" + - variable: defaultSettings.snapshotMaxCount + label: Snapshot Maximum Count + description: 'Maximum snapshot count for a volume. The value should be between 2 to 250.' + group: "Longhorn Default Settings" + type: int + min: 2 + max: 250 + default: 250 + - variable: defaultSettings.createDefaultDiskLabeledNodes + label: Create Default Disk on Labeled Nodes + description: 'Setting that allows Longhorn to automatically create a default disk only on nodes with the label "node.longhorn.io/create-default-disk=true" (if no other disks exist). When this setting is disabled, Longhorn creates a default disk on each node that is added to the cluster.' + group: "Longhorn Default Settings" + type: boolean + default: "false" + - variable: defaultSettings.defaultDataPath + label: Default Data Path + description: 'Default path for storing data on a host. The default value is "/var/lib/longhorn/".' + group: "Longhorn Default Settings" + type: string + default: "/var/lib/longhorn/" + - variable: defaultSettings.defaultDataLocality + label: Default Data Locality + description: 'Default data locality. A Longhorn volume has data locality if a local replica of the volume exists on the same node as the pod that is using the volume.' + group: "Longhorn Default Settings" + type: enum + options: + - "disabled" + - "best-effort" + default: "disabled" + - variable: defaultSettings.replicaSoftAntiAffinity + label: Replica Node Level Soft Anti-Affinity + description: 'Allow scheduling on nodes with existing healthy replicas of the same volume. By default, false.' + group: "Longhorn Default Settings" + type: boolean + default: "false" + - variable: defaultSettings.replicaAutoBalance + label: Replica Auto Balance + description: 'Enable this setting automatically re-balances replicas when discovered an available node.' + group: "Longhorn Default Settings" + type: enum + options: + - "disabled" + - "least-effort" + - "best-effort" + default: "disabled" + - variable: defaultSettings.storageOverProvisioningPercentage + label: Storage Over Provisioning Percentage + description: "Percentage of storage that can be allocated relative to hard drive capacity. The default value is 100." + group: "Longhorn Default Settings" + type: int + min: 0 + default: 100 + - variable: defaultSettings.storageMinimalAvailablePercentage + label: Storage Minimal Available Percentage + description: "If the minimum available disk capacity exceeds the actual percentage of available disk capacity, the disk becomes unschedulable until more space is freed up. By default, 25." + group: "Longhorn Default Settings" + type: int + min: 0 + max: 100 + default: 25 + - variable: defaultSettings.storageReservedPercentageForDefaultDisk + label: Storage Reserved Percentage For Default Disk + description: "The reserved percentage specifies the percentage of disk space that will not be allocated to the default disk on each new Longhorn node." + group: "Longhorn Default Settings" + type: int + min: 0 + max: 100 + default: 30 + - variable: defaultSettings.upgradeChecker + label: Enable Upgrade Checker + description: 'Upgrade Checker that periodically checks for new Longhorn versions. When a new version is available, a notification appears on the Longhorn UI. This setting is enabled by default.' + group: "Longhorn Default Settings" + type: boolean + default: "true" + - variable: defaultSettings.defaultReplicaCount + label: Default Replica Count + description: "Default number of replicas for volumes created using the Longhorn UI. For Kubernetes configuration, modify the `numberOfReplicas` field in the StorageClass. The default value is \"3\"." + group: "Longhorn Default Settings" + type: int + min: 1 + max: 20 + default: 3 + - variable: defaultSettings.defaultLonghornStaticStorageClass + label: Default Longhorn Static StorageClass Name + description: "Default Longhorn StorageClass. \"storageClassName\" is assigned to PVs and PVCs that are created for an existing Longhorn volume. \"storageClassName\" can also be used as a label, so it is possible to use a Longhorn StorageClass to bind a workload to an existing PV without creating a Kubernetes StorageClass object. The default value is \"longhorn-static\"." + group: "Longhorn Default Settings" + type: string + default: "longhorn-static" + - variable: defaultSettings.backupstorePollInterval + label: Backupstore Poll Interval + description: "Number of seconds that Longhorn waits before checking the backupstore for new backups. The default value is \"300\". When the value is \"0\", polling is disabled." + group: "Longhorn Default Settings" + type: int + min: 0 + default: 300 + - variable: defaultSettings.failedBackupTTL + label: Failed Backup Time to Live + description: "Number of minutes that Longhorn keeps a failed backup resource. When the value is \"0\", automatic deletion is disabled." + group: "Longhorn Default Settings" + type: int + min: 0 + default: 1440 + - variable: defaultSettings.restoreVolumeRecurringJobs + label: Restore Volume Recurring Jobs + description: "Restore recurring jobs from the backup volume on the backup target and create recurring jobs if not exist during a backup restoration." + group: "Longhorn Default Settings" + type: boolean + default: "false" + - variable: defaultSettings.recurringSuccessfulJobsHistoryLimit + label: Cronjob Successful Jobs History Limit + description: "This setting specifies how many successful backup or snapshot job histories should be retained. History will not be retained if the value is 0." + group: "Longhorn Default Settings" + type: int + min: 0 + default: 1 + - variable: defaultSettings.recurringFailedJobsHistoryLimit + label: Cronjob Failed Jobs History Limit + description: 'Maximum number of failed recurring backup and snapshot jobs to be retained. When the value is "0", a history of failed recurring jobs is not retained.' + group: "Longhorn Default Settings" + type: int + min: 0 + default: 1 + - variable: defaultSettings.recurringJobMaxRetention + label: Maximum Retention Number for Recurring Job + description: "Maximum number of snapshots or backups to be retained." + group: "Longhorn Default Settings" + type: int + default: 100 + - variable: defaultSettings.supportBundleFailedHistoryLimit + label: SupportBundle Failed History Limit + description: "This setting specifies how many failed support bundles can exist in the cluster. Set this value to **0** to have Longhorn automatically purge all failed support bundles." + group: "Longhorn Default Settings" + type: int + min: 0 + default: 1 + - variable: defaultSettings.autoSalvage + label: Automatic salvage + description: "Setting that allows Longhorn to automatically salvage volumes when all replicas become faulty (for example, when the network connection is interrupted). Longhorn determines which replicas are usable and then uses these replicas for the volume. This setting is enabled by default." + group: "Longhorn Default Settings" + type: boolean + default: "true" + - variable: defaultSettings.autoDeletePodWhenVolumeDetachedUnexpectedly + label: Automatically Delete Workload Pod when The Volume Is Detached Unexpectedly + description: 'Setting that allows Longhorn to automatically delete a workload pod that is managed by a controller (for example, daemonset) whenever a Longhorn volume is detached unexpectedly (for example, during Kubernetes upgrades). After deletion, the controller restarts the pod and then Kubernetes handles volume reattachment and remounting.' + group: "Longhorn Default Settings" + type: boolean + default: "true" + - variable: defaultSettings.disableSchedulingOnCordonedNode + label: Disable Scheduling On Cordoned Node + description: "Setting that prevents Longhorn Manager from scheduling replicas on a cordoned Kubernetes node. This setting is enabled by default." + group: "Longhorn Default Settings" + type: boolean + default: "true" + - variable: defaultSettings.replicaZoneSoftAntiAffinity + label: Replica Zone Level Soft Anti-Affinity + description: "Allow scheduling new Replicas of Volume to the Nodes in the same Zone as existing healthy Replicas. Nodes don't belong to any Zone will be treated as in the same Zone. Notice that Longhorn relies on label `topology.kubernetes.io/zone=` in the Kubernetes node object to identify the zone. By, default true." + group: "Longhorn Default Settings" + type: boolean + default: "true" + - variable: defaultSettings.replicaDiskSoftAntiAffinity + label: Replica Disk Level Soft Anti-Affinity + description: 'Allow scheduling on disks with existing healthy replicas of the same volume. By default, true.' + group: "Longhorn Default Settings" + type: boolean + default: "true" + - variable: defaultSettings.allowEmptyNodeSelectorVolume + label: Allow Empty Node Selector Volume + description: "Setting that allows scheduling of empty node selector volumes to any node." + group: "Longhorn Default Settings" + type: boolean + default: "true" + - variable: defaultSettings.allowEmptyDiskSelectorVolume + label: Allow Empty Disk Selector Volume + description: "Setting that allows scheduling of empty disk selector volumes to any disk." + group: "Longhorn Default Settings" + type: boolean + default: "true" + - variable: defaultSettings.nodeDownPodDeletionPolicy + label: Pod Deletion Policy When Node is Down + description: "Policy that defines the action Longhorn takes when a volume is stuck with a StatefulSet or Deployment pod on a node that failed." + group: "Longhorn Default Settings" + type: enum + options: + - "do-nothing" + - "delete-statefulset-pod" + - "delete-deployment-pod" + - "delete-both-statefulset-and-deployment-pod" + default: "do-nothing" + - variable: defaultSettings.nodeDrainPolicy + label: Node Drain Policy + description: "Policy that defines the action Longhorn takes when a node with the last healthy replica of a volume is drained." + group: "Longhorn Default Settings" + type: enum + options: + - "block-for-eviction" + - "block-for-eviction-if-contains-last-replica" + - "block-if-contains-last-replica" + - "allow-if-replica-is-stopped" + - "always-allow" + default: "block-if-contains-last-replica" + - variable: defaultSettings.detachManuallyAttachedVolumesWhenCordoned + label: Detach Manually Attached Volumes When Cordoned + description: "Setting that allows automatic detaching of manually-attached volumes when a node is cordoned." + group: "Longhorn Default Settings" + type: boolean + default: "false" + - variable: defaultSettings.priorityClass + label: Priority Class + description: "PriorityClass for system-managed Longhorn components. This setting can help prevent Longhorn components from being evicted under Node Pressure. Longhorn system contains user deployed components (E.g, Longhorn manager, Longhorn driver, Longhorn UI) and system managed components (E.g, instance manager, engine image, CSI driver, etc.) Note that this will be applied to Longhorn user-deployed components by default if there are no priority class values set yet, such as `longhornManager.priorityClass`. WARNING: DO NOT CHANGE THIS SETTING WITH ATTACHED VOLUMES." + group: "Longhorn Default Settings" + type: string + default: "longhorn-critical" + - variable: defaultSettings.replicaReplenishmentWaitInterval + label: Replica Replenishment Wait Interval + description: "The interval in seconds determines how long Longhorn will at least wait to reuse the existing data on a failed replica rather than directly creating a new replica for a degraded volume." + group: "Longhorn Default Settings" + type: int + min: 0 + default: 600 + - variable: defaultSettings.concurrentReplicaRebuildPerNodeLimit + label: Concurrent Replica Rebuild Per Node Limit + description: "Maximum number of replicas that can be concurrently rebuilt on each node. + WARNING: + - The old setting \"Disable Replica Rebuild\" is replaced by this setting. + - Different from relying on replica starting delay to limit the concurrent rebuilding, if the rebuilding is disabled, replica object replenishment will be directly skipped. + - When the value is 0, the eviction and data locality feature won't work. But this shouldn't have any impact to any current replica rebuild and backup restore." + group: "Longhorn Default Settings" + type: int + min: 0 + default: 5 + - variable: defaultSettings.concurrentVolumeBackupRestorePerNodeLimit + label: Concurrent Volume Backup Restore Per Node Limit + description: "Maximum number of volumes that can be concurrently restored on each node using a backup. When the value is \"0\", restoration of volumes using a backup is disabled." + group: "Longhorn Default Settings" + type: int + min: 0 + default: 5 + - variable: defaultSettings.disableRevisionCounter + label: Disable Revision Counter + description: "Setting that disables the revision counter and thereby prevents Longhorn from tracking all write operations to a volume. When salvaging a volume, Longhorn uses properties of the \"volume-head-xxx.img\" file (the last file size and the last time the file was modified) to select the replica to be used for volume recovery. This setting applies only to volumes created using the Longhorn UI." + group: "Longhorn Default Settings" + type: boolean + default: "false" + - variable: defaultSettings.systemManagedPodsImagePullPolicy + label: System Managed Pod Image Pull Policy + description: "Image pull policy for system-managed pods, such as Instance Manager, engine images, and CSI Driver. Changes to the image pull policy are applied only after the system-managed pods restart." + group: "Longhorn Default Settings" + type: enum + options: + - "if-not-present" + - "always" + - "never" + default: "if-not-present" + - variable: defaultSettings.allowVolumeCreationWithDegradedAvailability + label: Allow Volume Creation with Degraded Availability + description: "Setting that allows you to create and attach a volume without having all replicas scheduled at the time of creation." + group: "Longhorn Default Settings" + type: boolean + default: "true" + - variable: defaultSettings.autoCleanupSystemGeneratedSnapshot + label: Automatically Cleanup System Generated Snapshot + description: "Setting that allows Longhorn to automatically clean up the system-generated snapshot after replica rebuilding is completed." + group: "Longhorn Default Settings" + type: boolean + default: "true" + - variable: defaultSettings.autoCleanupRecurringJobBackupSnapshot + label: Automatically Cleanup Recurring Job Backup Snapshot + description: "Setting that allows Longhorn to automatically clean up the snapshot generated by a recurring backup job." + group: "Longhorn Default Settings" + type: boolean + default: "true" + - variable: defaultSettings.concurrentAutomaticEngineUpgradePerNodeLimit + label: Concurrent Automatic Engine Upgrade Per Node Limit + description: "Maximum number of engines that are allowed to concurrently upgrade on each node after Longhorn Manager is upgraded. When the value is \"0\", Longhorn does not automatically upgrade volume engines to the new default engine image version." + group: "Longhorn Default Settings" + type: int + min: 0 + default: 0 + - variable: defaultSettings.backingImageCleanupWaitInterval + label: Backing Image Cleanup Wait Interval + description: "Number of minutes that Longhorn waits before cleaning up the backing image file when no replicas in the disk are using it." + group: "Longhorn Default Settings" + type: int + min: 0 + default: 60 + - variable: defaultSettings.backingImageRecoveryWaitInterval + label: Backing Image Recovery Wait Interval + description: "Number of seconds that Longhorn waits before downloading a backing image file again when the status of all image disk files changes to \"failed\" or \"unknown\"." + group: "Longhorn Default Settings" + type: int + min: 0 + default: 300 + - variable: defaultSettings.guaranteedInstanceManagerCPU + label: Guaranteed Instance Manager CPU + description: "Percentage of the total allocatable CPU resources on each node to be reserved for each instance manager pod when the V1 Data Engine is enabled. The default value is \"12\". + WARNING: + - Value 0 means removing the CPU requests from spec of instance manager pods. + - Considering the possible number of new instance manager pods in a further system upgrade, this integer value ranges from 0 to 40. + - One more set of instance manager pods may need to be deployed when the Longhorn system is upgraded. If current available CPUs of the nodes are not enough for the new instance manager pods, you need to detach the volumes using the oldest instance manager pods so that Longhorn can clean up the old pods automatically and release the CPU resources. And the new pods with the latest instance manager image will be launched then. + - This global setting will be ignored for a node if the field \"InstanceManagerCPURequest\" on the node is set. + - After this setting is changed, all instance manager pods using this global setting on all the nodes will be automatically restarted. In other words, DO NOT CHANGE THIS SETTING WITH ATTACHED VOLUMES." + group: "Longhorn Default Settings" + type: int + min: 0 + max: 40 + default: 12 + - variable: defaultSettings.logLevel + label: Log Level + description: 'Log levels that indicate the type and severity of logs in Longhorn Manager. The default value is "Info". (Options: "Panic", "Fatal", "Error", "Warn", "Info", "Debug", "Trace")' + group: "Longhorn Default Settings" + type: string + default: "Info" + - variable: defaultSettings.disableSnapshotPurge + label: Disable Snapshot Purge + description: "Setting that temporarily prevents all attempts to purge volume snapshots." + group: "Longhorn Default Settings" + type: boolean + default: "false" +- variable: defaultSettings.kubernetesClusterAutoscalerEnabled + label: Kubernetes Cluster Autoscaler Enabled (Experimental) + description: "Setting that notifies Longhorn that the cluster is using the Kubernetes Cluster Autoscaler. + WARNING: + - Replica rebuilding could be expensive because nodes with reusable replicas could get removed by the Kubernetes Cluster Autoscaler." + group: "Longhorn Default Settings" + type: boolean + default: false +- variable: defaultSettings.orphanAutoDeletion + label: Orphaned Data Cleanup + description: "Setting that allows Longhorn to automatically delete an orphaned resource and the corresponding data (for example, stale replicas). Orphaned resources on failed or unknown nodes are not automatically cleaned up." + group: "Longhorn Default Settings" + type: boolean + default: false +- variable: defaultSettings.storageNetwork + label: Storage Network + description: "Longhorn uses the storage network for in-cluster data traffic. Leave this blank to use the Kubernetes cluster network. + WARNING: + - This setting should change after detaching all Longhorn volumes, as some of the Longhorn system component pods will get recreated to apply the setting. Longhorn will try to block this setting update when there are attached volumes." + group: "Longhorn Default Settings" + type: string + default: +- variable: defaultSettings.deletingConfirmationFlag + label: Deleting Confirmation Flag + description: "Flag that prevents accidental uninstallation of Longhorn." + group: "Longhorn Default Settings" + type: boolean + default: "false" +- variable: defaultSettings.engineReplicaTimeout + label: Timeout between Engine and Replica + description: "Timeout between the Longhorn Engine and replicas. Specify a value between \"8\" and \"30\" seconds. The default value is \"8\"." + group: "Longhorn Default Settings" + type: int + default: "8" +- variable: defaultSettings.snapshotDataIntegrity + label: Snapshot Data Integrity + description: "This setting allows users to enable or disable snapshot hashing and data integrity checking." + group: "Longhorn Default Settings" + type: string + default: "disabled" +- variable: defaultSettings.snapshotDataIntegrityImmediateCheckAfterSnapshotCreation + label: Immediate Snapshot Data Integrity Check After Creating a Snapshot + description: "Hashing snapshot disk files impacts the performance of the system. The immediate snapshot hashing and checking can be disabled to minimize the impact after creating a snapshot." + group: "Longhorn Default Settings" + type: boolean + default: "false" +- variable: defaultSettings.snapshotDataIntegrityCronjob + label: Snapshot Data Integrity Check CronJob + description: "Unix-cron string format. The setting specifies when Longhorn checks the data integrity of snapshot disk files." + group: "Longhorn Default Settings" + type: string + default: "0 0 */7 * *" +- variable: defaultSettings.removeSnapshotsDuringFilesystemTrim + label: Remove Snapshots During Filesystem Trim + description: "This setting allows Longhorn filesystem trim feature to automatically mark the latest snapshot and its ancestors as removed and stops at the snapshot containing multiple children." + group: "Longhorn Default Settings" + type: boolean + default: "false" +- variable: defaultSettings.fastReplicaRebuildEnabled + label: Fast Replica Rebuild Enabled + description: "Setting that allows fast rebuilding of replicas using the checksum of snapshot disk files. Before enabling this setting, you must set the snapshot-data-integrity value to \"enable\" or \"fast-check\"." + group: "Longhorn Default Settings" + type: boolean + default: false +- variable: defaultSettings.replicaFileSyncHttpClientTimeout + label: Timeout of HTTP Client to Replica File Sync Server + description: "In seconds. The setting specifies the HTTP client timeout to the file sync server." + group: "Longhorn Default Settings" + type: int + default: "30" +- variable: defaultSettings.backupCompressionMethod + label: Backup Compression Method + description: "Setting that allows you to specify a backup compression method." + group: "Longhorn Default Settings" + type: string + default: "lz4" +- variable: defaultSettings.backupConcurrentLimit + label: Backup Concurrent Limit Per Backup + description: "Maximum number of worker threads that can concurrently run for each backup." + group: "Longhorn Default Settings" + type: int + min: 1 + default: 2 +- variable: defaultSettings.restoreConcurrentLimit + label: Restore Concurrent Limit Per Backup + description: "This setting controls how many worker threads per restore concurrently." + group: "Longhorn Default Settings" + type: int + min: 1 + default: 2 +- variable: defaultSettings.allowCollectingLonghornUsageMetrics + label: Allow Collecting Longhorn Usage Metrics + description: "Setting that allows Longhorn to periodically collect anonymous usage data for product improvement purposes. Longhorn sends collected data to the [Upgrade Responder](https://github.com/longhorn/upgrade-responder) server, which is the data source of the Longhorn Public Metrics Dashboard (https://metrics.longhorn.io). The Upgrade Responder server does not store data that can be used to identify clients, including IP addresses." + group: "Longhorn Default Settings" + type: boolean + default: true +- variable: defaultSettings.v1DataEngine + label: V1 Data Engine + description: "Setting that allows you to enable the V1 Data Engine." + group: "Longhorn V1 Data Engine Settings" + type: boolean + default: true +- variable: defaultSettings.v2DataEngine + label: V2 Data Engine + description: "Setting that allows you to enable the V2 Data Engine, which is based on the Storage Performance Development Kit (SPDK). The V2 Data Engine is a preview feature and should not be used in production environments. + WARNING: + - DO NOT CHANGE THIS SETTING WITH ATTACHED VOLUMES. Longhorn will block this setting update when there are attached volumes. + - When the V2 Data Engine is enabled, each instance-manager pod utilizes 1 CPU core. This high CPU usage is attributed to the spdk_tgt process running within each instance-manager pod. The spdk_tgt process is responsible for handling input/output (IO) operations and requires intensive polling. As a result, it consumes 100% of a dedicated CPU core to efficiently manage and process the IO requests, ensuring optimal performance and responsiveness for storage operations." + group: "Longhorn V2 Data Engine (Preview Feature) Settings" + type: boolean + default: false +- variable: defaultSettings.v2DataEngineHugepageLimit + label: V2 Data Engine + description: "This allows users to configure maximum huge page size (in MiB) for the V2 Data Engine." + group: "Longhorn V2 Data Engine (Preview Feature) Settings" + type: int + default: "2048" +- variable: defaultSettings.offlineReplicaRebuilding + label: Offline Replica Rebuilding + description: "Setting that allows rebuilding of offline replicas for volumes using the V2 Data Engine." + group: "Longhorn V2 Data Engine (Preview Feature) Settings" + required: true + type: enum + options: + - "enabled" + - "disabled" + default: "enabled" +- variable: persistence.defaultClass + default: "true" + description: "Setting that allows you to specify the default Longhorn StorageClass." + label: Default Storage Class + group: "Longhorn Storage Class Settings" + required: true + type: boolean +- variable: persistence.reclaimPolicy + label: Storage Class Retain Policy + description: "Reclaim policy that provides instructions for handling of a volume after its claim is released. (Options: \"Retain\", \"Delete\")" + group: "Longhorn Storage Class Settings" + required: true + type: enum + options: + - "Delete" + - "Retain" + default: "Delete" +- variable: persistence.defaultClassReplicaCount + description: "Replica count of the default Longhorn StorageClass." + label: Default Storage Class Replica Count + group: "Longhorn Storage Class Settings" + type: int + min: 1 + max: 10 + default: 3 +- variable: persistence.defaultDataLocality + description: "Data locality of the default Longhorn StorageClass. (Options: \"disabled\", \"best-effort\")" + label: Default Storage Class Data Locality + group: "Longhorn Storage Class Settings" + type: enum + options: + - "disabled" + - "best-effort" + default: "disabled" +- variable: persistence.recurringJobSelector.enable + description: "Setting that allows you to enable the recurring job selector for a Longhorn StorageClass." + group: "Longhorn Storage Class Settings" + label: Enable Storage Class Recurring Job Selector + type: boolean + default: false + show_subquestion_if: true + subquestions: + - variable: persistence.recurringJobSelector.jobList + description: 'Recurring job selector for a Longhorn StorageClass. Ensure that quotes are used correctly when specifying job parameters. (Example: `[{"name":"backup", "isGroup":true}]`)' + label: Storage Class Recurring Job Selector List + group: "Longhorn Storage Class Settings" + type: string + default: +- variable: persistence.defaultNodeSelector.enable + description: "Setting that allows you to enable the node selector for the default Longhorn StorageClass." + group: "Longhorn Storage Class Settings" + label: Enable Storage Class Node Selector + type: boolean + default: false + show_subquestion_if: true + subquestions: + - variable: persistence.defaultNodeSelector.selector + label: Storage Class Node Selector + description: 'Node selector for the default Longhorn StorageClass. Longhorn uses only nodes with the specified tags for storing volume data. (Examples: "storage,fast")' + group: "Longhorn Storage Class Settings" + type: string + default: +- variable: persistence.backingImage.enable + description: "Setting that allows you to use a backing image in a Longhorn StorageClass." + group: "Longhorn Storage Class Settings" + label: Default Storage Class Backing Image + type: boolean + default: false + show_subquestion_if: true + subquestions: + - variable: persistence.backingImage.name + description: 'Backing image to be used for creating and restoring volumes in a Longhorn StorageClass. When no backing images are available, specify the data source type and parameters that Longhorn can use to create a backing image.' + label: Storage Class Backing Image Name + group: "Longhorn Storage Class Settings" + type: string + default: + - variable: persistence.backingImage.expectedChecksum + description: 'Expected SHA-512 checksum of a backing image used in a Longhorn StorageClass. + WARNING: + - If the backing image name is not specified, setting this field is meaningless. + - It is not recommended to set this field if the data source type is \"export-from-volume\".' + label: Storage Class Backing Image Expected SHA512 Checksum + group: "Longhorn Storage Class Settings" + type: string + default: + - variable: persistence.backingImage.dataSourceType + description: 'Data source type of a backing image used in a Longhorn StorageClass. If the backing image exists in the cluster, Longhorn uses this setting to verify the image. If the backing image does not exist, Longhorn creates one using the specified data source type. + WARNING: + - If the backing image name is not specified, setting this field is meaningless. + - As for backing image creation with data source type \"upload\", it is recommended to do it via UI rather than StorageClass here. Uploading requires file data sending to the Longhorn backend after the object creation, which is complicated if you want to handle it manually.' + label: Storage Class Backing Image Data Source Type + group: "Longhorn Storage Class Settings" + type: enum + options: + - "" + - "download" + - "upload" + - "export-from-volume" + default: "" + - variable: persistence.backingImage.dataSourceParameters + description: "Data source parameters of a backing image used in a Longhorn StorageClass. You can specify a JSON string of a map. (Example: `'{\"url\":\"https://backing-image-example.s3-region.amazonaws.com/test-backing-image\"}'`) + WARNING: + - If the backing image name is not specified, setting this field is meaningless. + - Be careful of the quotes here." + label: Storage Class Backing Image Data Source Parameters + group: "Longhorn Storage Class Settings" + type: string + default: +- variable: persistence.removeSnapshotsDuringFilesystemTrim + description: "Setting that allows you to enable automatic snapshot removal during filesystem trim for a Longhorn StorageClass. (Options: \"ignored\", \"enabled\", \"disabled\")" + label: Default Storage Class Remove Snapshots During Filesystem Trim + group: "Longhorn Storage Class Settings" + type: enum + options: + - "ignored" + - "enabled" + - "disabled" + default: "ignored" +- variable: ingress.enabled + default: "false" + description: "Expose app using Layer 7 Load Balancer - ingress" + type: boolean + group: "Services and Load Balancing" + label: Expose app using Layer 7 Load Balancer + show_subquestion_if: true + subquestions: + - variable: ingress.host + default: "xip.io" + description: "Hostname of the Layer 7 load balancer." + type: hostname + required: true + label: Layer 7 Load Balancer Hostname + - variable: ingress.path + default: "/" + description: "Default ingress path. You can access the Longhorn UI by following the full ingress path {{host}}+{{path}}." + type: string + required: true + label: Ingress Path +- variable: service.ui.type + default: "Rancher-Proxy" + description: "Service type for Longhorn UI. (Options: \"ClusterIP\", \"NodePort\", \"LoadBalancer\", \"Rancher-Proxy\")" + type: enum + options: + - "ClusterIP" + - "NodePort" + - "LoadBalancer" + - "Rancher-Proxy" + label: Longhorn UI Service + show_if: "ingress.enabled=false" + group: "Services and Load Balancing" + show_subquestion_if: "NodePort" + subquestions: + - variable: service.ui.nodePort + default: "" + description: "NodePort port number for Longhorn UI. When unspecified, Longhorn selects a free port between 30000 and 32767." + type: int + min: 30000 + max: 32767 + show_if: "service.ui.type=NodePort||service.ui.type=LoadBalancer" + label: UI Service NodePort number +- variable: enablePSP + default: "false" + description: "Setting that allows you to enable pod security policies (PSPs) that allow privileged Longhorn pods to start. This setting applies only to clusters running Kubernetes 1.25 and earlier, and with the built-in Pod Security admission controller enabled." + label: Pod Security Policy + type: boolean + group: "Other Settings" +- variable: global.cattle.windowsCluster.enabled + default: "false" + description: "Setting that allows Longhorn to run on a Rancher Windows cluster." + label: Rancher Windows Cluster + type: boolean + group: "Other Settings" +- variable: networkPolicies.enabled + description: "Setting that allows you to enable network policies that control access to Longhorn pods. + Warning: The Rancher Proxy will not work if this feature is enabled and a custom NetworkPolicy must be added." + group: "Other Settings" + label: Network Policies + default: "false" + type: boolean + subquestions: + - variable: networkPolicies.type + label: Network Policies for Ingress + description: "Distribution that determines the policy for allowing access for an ingress. (Options: \"k3s\", \"rke2\", \"rke1\")" + show_if: "networkPolicies.enabled=true&&ingress.enabled=true" + type: enum + default: "rke2" + options: + - "rke1" + - "rke2" + - "k3s" + - variable: defaultSettings.v2DataEngineGuaranteedInstanceManagerCPU + label: Guaranteed Instance Manager CPU for V2 Data Engine + description: 'Number of millicpus on each node to be reserved for each Instance Manager pod when the V2 Data Engine is enabled. The default value is "1250". + WARNING: + - Specifying a value of 0 disables CPU requests for instance manager pods. You must specify an integer between 1000 and 8000. + - This is a global setting. Modifying the value triggers an automatic restart of the instance manager pods. Do not modify the value while volumes are still attached." + group: "Longhorn Default Settings' + type: int + min: 1000 + max: 8000 + default: 1250 \ No newline at end of file diff --git a/charts/longhorn/102.4.1+up1.6.2/templates/NOTES.txt b/charts/longhorn/102.4.1+up1.6.2/templates/NOTES.txt new file mode 100644 index 0000000000..cca7cd77b9 --- /dev/null +++ b/charts/longhorn/102.4.1+up1.6.2/templates/NOTES.txt @@ -0,0 +1,5 @@ +Longhorn is now installed on the cluster! + +Please wait a few minutes for other Longhorn components such as CSI deployments, Engine Images, and Instance Managers to be initialized. + +Visit our documentation at https://longhorn.io/docs/ diff --git a/charts/longhorn/102.4.1+up1.6.2/templates/_helpers.tpl b/charts/longhorn/102.4.1+up1.6.2/templates/_helpers.tpl new file mode 100644 index 0000000000..3fbc2ac02f --- /dev/null +++ b/charts/longhorn/102.4.1+up1.6.2/templates/_helpers.tpl @@ -0,0 +1,66 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "longhorn.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "longhorn.fullname" -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} + + +{{- define "longhorn.managerIP" -}} +{{- $fullname := (include "longhorn.fullname" .) -}} +{{- printf "http://%s-backend:9500" $fullname | trunc 63 | trimSuffix "-" -}} +{{- end -}} + + +{{- define "secret" }} +{{- printf "{\"auths\": {\"%s\": {\"auth\": \"%s\"}}}" .Values.privateRegistry.registryUrl (printf "%s:%s" .Values.privateRegistry.registryUser .Values.privateRegistry.registryPasswd | b64enc) | b64enc }} +{{- end }} + +{{- /* +longhorn.labels generates the standard Helm labels. +*/ -}} +{{- define "longhorn.labels" -}} +app.kubernetes.io/name: {{ template "longhorn.name" . }} +helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/version: {{ .Chart.AppVersion }} +{{- end -}} + + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- else -}} +{{- "" -}} +{{- end -}} +{{- end -}} + +{{- define "registry_url" -}} +{{- if .Values.privateRegistry.registryUrl -}} +{{- printf "%s/" .Values.privateRegistry.registryUrl -}} +{{- else -}} +{{ include "system_default_registry" . }} +{{- end -}} +{{- end -}} + +{{- /* + define the longhorn release namespace +*/ -}} +{{- define "release_namespace" -}} +{{- if .Values.namespaceOverride -}} +{{- .Values.namespaceOverride -}} +{{- else -}} +{{- .Release.Namespace -}} +{{- end -}} +{{- end -}} diff --git a/charts/longhorn/102.4.1+up1.6.2/templates/clusterrole.yaml b/charts/longhorn/102.4.1+up1.6.2/templates/clusterrole.yaml new file mode 100644 index 0000000000..f6e069f004 --- /dev/null +++ b/charts/longhorn/102.4.1+up1.6.2/templates/clusterrole.yaml @@ -0,0 +1,77 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: longhorn-role + labels: {{- include "longhorn.labels" . | nindent 4 }} +rules: +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - "*" +- apiGroups: [""] + resources: ["pods", "events", "persistentvolumes", "persistentvolumeclaims","persistentvolumeclaims/status", "nodes", "proxy/nodes", "pods/log", "secrets", "services", "endpoints", "configmaps", "serviceaccounts"] + verbs: ["*"] +- apiGroups: [""] + resources: ["namespaces"] + verbs: ["get", "list"] +- apiGroups: ["apps"] + resources: ["daemonsets", "statefulsets", "deployments"] + verbs: ["*"] +- apiGroups: ["batch"] + resources: ["jobs", "cronjobs"] + verbs: ["*"] +- apiGroups: ["policy"] + resources: ["poddisruptionbudgets", "podsecuritypolicies"] + verbs: ["*"] +- apiGroups: ["scheduling.k8s.io"] + resources: ["priorityclasses"] + verbs: ["watch", "list"] +- apiGroups: ["storage.k8s.io"] + resources: ["storageclasses", "volumeattachments", "volumeattachments/status", "csinodes", "csidrivers"] + verbs: ["*"] +- apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses", "volumesnapshots", "volumesnapshotcontents", "volumesnapshotcontents/status"] + verbs: ["*"] +- apiGroups: ["longhorn.io"] + resources: ["volumes", "volumes/status", "engines", "engines/status", "replicas", "replicas/status", "settings", + "engineimages", "engineimages/status", "nodes", "nodes/status", "instancemanagers", "instancemanagers/status", + {{- if .Values.openshift.enabled }} + "engineimages/finalizers", "nodes/finalizers", "instancemanagers/finalizers", + {{- end }} + "sharemanagers", "sharemanagers/status", "backingimages", "backingimages/status", + "backingimagemanagers", "backingimagemanagers/status", "backingimagedatasources", "backingimagedatasources/status", + "backuptargets", "backuptargets/status", "backupvolumes", "backupvolumes/status", "backups", "backups/status", + "recurringjobs", "recurringjobs/status", "orphans", "orphans/status", "snapshots", "snapshots/status", + "supportbundles", "supportbundles/status", "systembackups", "systembackups/status", "systemrestores", "systemrestores/status", + "volumeattachments", "volumeattachments/status", "backupbackingimages", "backupbackingimages/status"] + verbs: ["*"] +- apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["*"] +- apiGroups: ["metrics.k8s.io"] + resources: ["pods", "nodes"] + verbs: ["get", "list"] +- apiGroups: ["apiregistration.k8s.io"] + resources: ["apiservices"] + verbs: ["list", "watch"] +- apiGroups: ["admissionregistration.k8s.io"] + resources: ["mutatingwebhookconfigurations", "validatingwebhookconfigurations"] + verbs: ["get", "list", "create", "patch", "delete"] +- apiGroups: ["rbac.authorization.k8s.io"] + resources: ["roles", "rolebindings", "clusterrolebindings", "clusterroles"] + verbs: ["*"] +{{- if .Values.openshift.enabled }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: longhorn-ocp-privileged-role + labels: {{- include "longhorn.labels" . | nindent 4 }} +rules: +- apiGroups: ["security.openshift.io"] + resources: ["securitycontextconstraints"] + resourceNames: ["anyuid", "privileged"] + verbs: ["use"] +{{- end }} diff --git a/charts/longhorn/102.4.1+up1.6.2/templates/clusterrolebinding.yaml b/charts/longhorn/102.4.1+up1.6.2/templates/clusterrolebinding.yaml new file mode 100644 index 0000000000..2e34f014ce --- /dev/null +++ b/charts/longhorn/102.4.1+up1.6.2/templates/clusterrolebinding.yaml @@ -0,0 +1,49 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: longhorn-bind + labels: {{- include "longhorn.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: longhorn-role +subjects: +- kind: ServiceAccount + name: longhorn-service-account + namespace: {{ include "release_namespace" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: longhorn-support-bundle + labels: {{- include "longhorn.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: +- kind: ServiceAccount + name: longhorn-support-bundle + namespace: {{ include "release_namespace" . }} +{{- if .Values.openshift.enabled }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: longhorn-ocp-privileged-bind + labels: {{- include "longhorn.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: longhorn-ocp-privileged-role +subjects: +- kind: ServiceAccount + name: longhorn-service-account + namespace: {{ include "release_namespace" . }} +- kind: ServiceAccount + name: longhorn-ui-service-account + namespace: {{ include "release_namespace" . }} +- kind: ServiceAccount + name: default # supportbundle-agent-support-bundle uses default sa + namespace: {{ include "release_namespace" . }} +{{- end }} diff --git a/charts/longhorn/102.4.1+up1.6.2/templates/daemonset-sa.yaml b/charts/longhorn/102.4.1+up1.6.2/templates/daemonset-sa.yaml new file mode 100644 index 0000000000..2fa1cbc243 --- /dev/null +++ b/charts/longhorn/102.4.1+up1.6.2/templates/daemonset-sa.yaml @@ -0,0 +1,167 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: {{- include "longhorn.labels" . | nindent 4 }} + app: longhorn-manager + name: longhorn-manager + namespace: {{ include "release_namespace" . }} +spec: + selector: + matchLabels: + app: longhorn-manager + template: + metadata: + labels: {{- include "longhorn.labels" . | nindent 8 }} + app: longhorn-manager + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + containers: + - name: longhorn-manager + image: {{ template "registry_url" . }}{{ .Values.image.longhorn.manager.repository }}:{{ .Values.image.longhorn.manager.tag }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + securityContext: + privileged: true + command: + - longhorn-manager + - -d + {{- if eq .Values.longhornManager.log.format "json" }} + - -j + {{- end }} + - daemon + - --engine-image + - "{{ template "registry_url" . }}{{ .Values.image.longhorn.engine.repository }}:{{ .Values.image.longhorn.engine.tag }}" + - --instance-manager-image + - "{{ template "registry_url" . }}{{ .Values.image.longhorn.instanceManager.repository }}:{{ .Values.image.longhorn.instanceManager.tag }}" + - --share-manager-image + - "{{ template "registry_url" . }}{{ .Values.image.longhorn.shareManager.repository }}:{{ .Values.image.longhorn.shareManager.tag }}" + - --backing-image-manager-image + - "{{ template "registry_url" . }}{{ .Values.image.longhorn.backingImageManager.repository }}:{{ .Values.image.longhorn.backingImageManager.tag }}" + - --support-bundle-manager-image + - "{{ template "registry_url" . }}{{ .Values.image.longhorn.supportBundleKit.repository }}:{{ .Values.image.longhorn.supportBundleKit.tag }}" + - --manager-image + - "{{ template "registry_url" . }}{{ .Values.image.longhorn.manager.repository }}:{{ .Values.image.longhorn.manager.tag }}" + - --service-account + - longhorn-service-account + {{- if .Values.preUpgradeChecker.upgradeVersionCheck}} + - --upgrade-version-check + {{- end }} + ports: + - containerPort: 9500 + name: manager + - containerPort: 9501 + name: conversion-wh + - containerPort: 9502 + name: admission-wh + - containerPort: 9503 + name: recov-backend + readinessProbe: + httpGet: + path: /v1/healthz + port: 9501 + scheme: HTTPS + volumeMounts: + - name: dev + mountPath: /host/dev/ + - name: proc + mountPath: /host/proc/ + - name: longhorn + mountPath: /var/lib/longhorn/ + mountPropagation: Bidirectional + - name: longhorn-grpc-tls + mountPath: /tls-files/ + {{- if .Values.enableGoCoverDir }} + - name: go-cover-dir + mountPath: /go-cover-dir/ + {{- end }} + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + {{- if .Values.enableGoCoverDir }} + - name: GOCOVERDIR + value: /go-cover-dir/ + {{- end }} + volumes: + - name: dev + hostPath: + path: /dev/ + - name: proc + hostPath: + path: /proc/ + - name: longhorn + hostPath: + path: /var/lib/longhorn/ + {{- if .Values.enableGoCoverDir }} + - name: go-cover-dir + hostPath: + path: /go-cover-dir/ + type: DirectoryOrCreate + {{- end }} + - name: longhorn-grpc-tls + secret: + secretName: longhorn-grpc-tls + optional: true + {{- if .Values.privateRegistry.registrySecret }} + imagePullSecrets: + - name: {{ .Values.privateRegistry.registrySecret }} + {{- end }} + {{- if .Values.longhornManager.priorityClass }} + priorityClassName: {{ .Values.longhornManager.priorityClass | quote }} + {{- end }} + {{- if or .Values.longhornManager.tolerations .Values.global.cattle.windowsCluster.enabled }} + tolerations: + {{- if and .Values.global.cattle.windowsCluster.enabled .Values.global.cattle.windowsCluster.tolerations }} +{{ toYaml .Values.global.cattle.windowsCluster.tolerations | indent 6 }} + {{- end }} + {{- if .Values.longhornManager.tolerations }} +{{ toYaml .Values.longhornManager.tolerations | indent 6 }} + {{- end }} + {{- end }} + {{- if or .Values.longhornManager.nodeSelector .Values.global.cattle.windowsCluster.enabled }} + nodeSelector: + {{- if and .Values.global.cattle.windowsCluster.enabled .Values.global.cattle.windowsCluster.nodeSelector }} +{{ toYaml .Values.global.cattle.windowsCluster.nodeSelector | indent 8 }} + {{- end }} + {{- if .Values.longhornManager.nodeSelector }} +{{ toYaml .Values.longhornManager.nodeSelector | indent 8 }} + {{- end }} + {{- end }} + serviceAccountName: longhorn-service-account + updateStrategy: + rollingUpdate: + maxUnavailable: "100%" +--- +apiVersion: v1 +kind: Service +metadata: + labels: {{- include "longhorn.labels" . | nindent 4 }} + app: longhorn-manager + name: longhorn-backend + namespace: {{ include "release_namespace" . }} + {{- if .Values.longhornManager.serviceAnnotations }} + annotations: +{{ toYaml .Values.longhornManager.serviceAnnotations | indent 4 }} + {{- end }} +spec: + type: {{ .Values.service.manager.type }} + selector: + app: longhorn-manager + ports: + - name: manager + port: 9500 + targetPort: manager + {{- if .Values.service.manager.nodePort }} + nodePort: {{ .Values.service.manager.nodePort }} + {{- end }} diff --git a/charts/longhorn/102.4.1+up1.6.2/templates/default-setting.yaml b/charts/longhorn/102.4.1+up1.6.2/templates/default-setting.yaml new file mode 100644 index 0000000000..5261f7fef8 --- /dev/null +++ b/charts/longhorn/102.4.1+up1.6.2/templates/default-setting.yaml @@ -0,0 +1,229 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: longhorn-default-setting + namespace: {{ include "release_namespace" . }} + labels: {{- include "longhorn.labels" . | nindent 4 }} +data: + default-setting.yaml: |- + {{- if not (kindIs "invalid" .Values.defaultSettings.backupTarget) }} + backup-target: {{ .Values.defaultSettings.backupTarget }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.backupTargetCredentialSecret) }} + backup-target-credential-secret: {{ .Values.defaultSettings.backupTargetCredentialSecret }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.allowRecurringJobWhileVolumeDetached) }} + allow-recurring-job-while-volume-detached: {{ .Values.defaultSettings.allowRecurringJobWhileVolumeDetached }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.createDefaultDiskLabeledNodes) }} + create-default-disk-labeled-nodes: {{ .Values.defaultSettings.createDefaultDiskLabeledNodes }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.defaultDataPath) }} + default-data-path: {{ .Values.defaultSettings.defaultDataPath }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.replicaSoftAntiAffinity) }} + replica-soft-anti-affinity: {{ .Values.defaultSettings.replicaSoftAntiAffinity }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.replicaAutoBalance) }} + replica-auto-balance: {{ .Values.defaultSettings.replicaAutoBalance }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.storageOverProvisioningPercentage) }} + storage-over-provisioning-percentage: {{ .Values.defaultSettings.storageOverProvisioningPercentage }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.storageMinimalAvailablePercentage) }} + storage-minimal-available-percentage: {{ .Values.defaultSettings.storageMinimalAvailablePercentage }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.storageReservedPercentageForDefaultDisk) }} + storage-reserved-percentage-for-default-disk: {{ .Values.defaultSettings.storageReservedPercentageForDefaultDisk }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.upgradeChecker) }} + upgrade-checker: {{ .Values.defaultSettings.upgradeChecker }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.defaultReplicaCount) }} + default-replica-count: {{ .Values.defaultSettings.defaultReplicaCount }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.defaultDataLocality) }} + default-data-locality: {{ .Values.defaultSettings.defaultDataLocality }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.defaultLonghornStaticStorageClass) }} + default-longhorn-static-storage-class: {{ .Values.defaultSettings.defaultLonghornStaticStorageClass }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.backupstorePollInterval) }} + backupstore-poll-interval: {{ .Values.defaultSettings.backupstorePollInterval }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.failedBackupTTL) }} + failed-backup-ttl: {{ .Values.defaultSettings.failedBackupTTL }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.restoreVolumeRecurringJobs) }} + restore-volume-recurring-jobs: {{ .Values.defaultSettings.restoreVolumeRecurringJobs }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.recurringSuccessfulJobsHistoryLimit) }} + recurring-successful-jobs-history-limit: {{ .Values.defaultSettings.recurringSuccessfulJobsHistoryLimit }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.recurringJobMaxRetention) }} + recurring-job-max-retention: {{ .Values.defaultSettings.recurringJobMaxRetention }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.recurringFailedJobsHistoryLimit) }} + recurring-failed-jobs-history-limit: {{ .Values.defaultSettings.recurringFailedJobsHistoryLimit }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.supportBundleFailedHistoryLimit) }} + support-bundle-failed-history-limit: {{ .Values.defaultSettings.supportBundleFailedHistoryLimit }} + {{- end }} + {{- if or (not (kindIs "invalid" .Values.defaultSettings.taintToleration)) (.Values.global.cattle.windowsCluster.enabled) }} + taint-toleration: {{ $windowsDefaultSettingTaintToleration := list }}{{ $defaultSettingTaintToleration := list -}} + {{- if and .Values.global.cattle.windowsCluster.enabled .Values.global.cattle.windowsCluster.defaultSetting.taintToleration -}} + {{- $windowsDefaultSettingTaintToleration = .Values.global.cattle.windowsCluster.defaultSetting.taintToleration -}} + {{- end -}} + {{- if not (kindIs "invalid" .Values.defaultSettings.taintToleration) -}} + {{- $defaultSettingTaintToleration = .Values.defaultSettings.taintToleration -}} + {{- end -}} + {{- $taintToleration := list $windowsDefaultSettingTaintToleration $defaultSettingTaintToleration }}{{ join ";" (compact $taintToleration) -}} + {{- end }} + {{- if or (not (kindIs "invalid" .Values.defaultSettings.systemManagedComponentsNodeSelector)) (.Values.global.cattle.windowsCluster.enabled) }} + system-managed-components-node-selector: {{ $windowsDefaultSettingNodeSelector := list }}{{ $defaultSettingNodeSelector := list -}} + {{- if and .Values.global.cattle.windowsCluster.enabled .Values.global.cattle.windowsCluster.defaultSetting.systemManagedComponentsNodeSelector -}} + {{ $windowsDefaultSettingNodeSelector = .Values.global.cattle.windowsCluster.defaultSetting.systemManagedComponentsNodeSelector -}} + {{- end -}} + {{- if not (kindIs "invalid" .Values.defaultSettings.systemManagedComponentsNodeSelector) -}} + {{- $defaultSettingNodeSelector = .Values.defaultSettings.systemManagedComponentsNodeSelector -}} + {{- end -}} + {{- $nodeSelector := list $windowsDefaultSettingNodeSelector $defaultSettingNodeSelector }}{{ join ";" (compact $nodeSelector) -}} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.priorityClass) }} + priority-class: {{ .Values.defaultSettings.priorityClass }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.autoSalvage) }} + auto-salvage: {{ .Values.defaultSettings.autoSalvage }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.autoDeletePodWhenVolumeDetachedUnexpectedly) }} + auto-delete-pod-when-volume-detached-unexpectedly: {{ .Values.defaultSettings.autoDeletePodWhenVolumeDetachedUnexpectedly }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.disableSchedulingOnCordonedNode) }} + disable-scheduling-on-cordoned-node: {{ .Values.defaultSettings.disableSchedulingOnCordonedNode }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.replicaZoneSoftAntiAffinity) }} + replica-zone-soft-anti-affinity: {{ .Values.defaultSettings.replicaZoneSoftAntiAffinity }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.replicaDiskSoftAntiAffinity) }} + replica-disk-soft-anti-affinity: {{ .Values.defaultSettings.replicaDiskSoftAntiAffinity }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.nodeDownPodDeletionPolicy) }} + node-down-pod-deletion-policy: {{ .Values.defaultSettings.nodeDownPodDeletionPolicy }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.nodeDrainPolicy) }} + node-drain-policy: {{ .Values.defaultSettings.nodeDrainPolicy }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.detachManuallyAttachedVolumesWhenCordoned) }} + detach-manually-attached-volumes-when-cordoned: {{ .Values.defaultSettings.detachManuallyAttachedVolumesWhenCordoned }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.replicaReplenishmentWaitInterval) }} + replica-replenishment-wait-interval: {{ .Values.defaultSettings.replicaReplenishmentWaitInterval }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.concurrentReplicaRebuildPerNodeLimit) }} + concurrent-replica-rebuild-per-node-limit: {{ .Values.defaultSettings.concurrentReplicaRebuildPerNodeLimit }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.concurrentVolumeBackupRestorePerNodeLimit) }} + concurrent-volume-backup-restore-per-node-limit: {{ .Values.defaultSettings.concurrentVolumeBackupRestorePerNodeLimit }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.disableRevisionCounter) }} + disable-revision-counter: {{ .Values.defaultSettings.disableRevisionCounter }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.systemManagedPodsImagePullPolicy) }} + system-managed-pods-image-pull-policy: {{ .Values.defaultSettings.systemManagedPodsImagePullPolicy }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.allowVolumeCreationWithDegradedAvailability) }} + allow-volume-creation-with-degraded-availability: {{ .Values.defaultSettings.allowVolumeCreationWithDegradedAvailability }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.autoCleanupSystemGeneratedSnapshot) }} + auto-cleanup-system-generated-snapshot: {{ .Values.defaultSettings.autoCleanupSystemGeneratedSnapshot }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.autoCleanupRecurringJobBackupSnapshot) }} + auto-cleanup-recurring-job-backup-snapshot: {{ .Values.defaultSettings.autoCleanupRecurringJobBackupSnapshot }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.concurrentAutomaticEngineUpgradePerNodeLimit) }} + concurrent-automatic-engine-upgrade-per-node-limit: {{ .Values.defaultSettings.concurrentAutomaticEngineUpgradePerNodeLimit }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.backingImageCleanupWaitInterval) }} + backing-image-cleanup-wait-interval: {{ .Values.defaultSettings.backingImageCleanupWaitInterval }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.backingImageRecoveryWaitInterval) }} + backing-image-recovery-wait-interval: {{ .Values.defaultSettings.backingImageRecoveryWaitInterval }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.guaranteedInstanceManagerCPU) }} + guaranteed-instance-manager-cpu: {{ .Values.defaultSettings.guaranteedInstanceManagerCPU }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.kubernetesClusterAutoscalerEnabled) }} + kubernetes-cluster-autoscaler-enabled: {{ .Values.defaultSettings.kubernetesClusterAutoscalerEnabled }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.orphanAutoDeletion) }} + orphan-auto-deletion: {{ .Values.defaultSettings.orphanAutoDeletion }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.storageNetwork) }} + storage-network: {{ .Values.defaultSettings.storageNetwork }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.deletingConfirmationFlag) }} + deleting-confirmation-flag: {{ .Values.defaultSettings.deletingConfirmationFlag }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.engineReplicaTimeout) }} + engine-replica-timeout: {{ .Values.defaultSettings.engineReplicaTimeout }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.snapshotDataIntegrity) }} + snapshot-data-integrity: {{ .Values.defaultSettings.snapshotDataIntegrity }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.snapshotDataIntegrityImmediateCheckAfterSnapshotCreation) }} + snapshot-data-integrity-immediate-check-after-snapshot-creation: {{ .Values.defaultSettings.snapshotDataIntegrityImmediateCheckAfterSnapshotCreation }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.snapshotDataIntegrityCronjob) }} + snapshot-data-integrity-cronjob: {{ .Values.defaultSettings.snapshotDataIntegrityCronjob }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.removeSnapshotsDuringFilesystemTrim) }} + remove-snapshots-during-filesystem-trim: {{ .Values.defaultSettings.removeSnapshotsDuringFilesystemTrim }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.fastReplicaRebuildEnabled) }} + fast-replica-rebuild-enabled: {{ .Values.defaultSettings.fastReplicaRebuildEnabled }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.replicaFileSyncHttpClientTimeout) }} + replica-file-sync-http-client-timeout: {{ .Values.defaultSettings.replicaFileSyncHttpClientTimeout }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.logLevel) }} + log-level: {{ .Values.defaultSettings.logLevel }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.backupCompressionMethod) }} + backup-compression-method: {{ .Values.defaultSettings.backupCompressionMethod }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.backupConcurrentLimit) }} + backup-concurrent-limit: {{ .Values.defaultSettings.backupConcurrentLimit }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.restoreConcurrentLimit) }} + restore-concurrent-limit: {{ .Values.defaultSettings.restoreConcurrentLimit }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.v1DataEngine) }} + v1-data-engine: {{ .Values.defaultSettings.v1DataEngine }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.v2DataEngine) }} + v2-data-engine: {{ .Values.defaultSettings.v2DataEngine }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.v2DataEngineHugepageLimit) }} + v2-data-engine-hugepage-limit: {{ .Values.defaultSettings.v2DataEngineHugepageLimit }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.offlineReplicaRebuilding) }} + offline-replica-rebuilding: {{ .Values.defaultSettings.offlineReplicaRebuilding }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.allowEmptyNodeSelectorVolume) }} + allow-empty-node-selector-volume: {{ .Values.defaultSettings.allowEmptyNodeSelectorVolume }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.allowEmptyDiskSelectorVolume) }} + allow-empty-disk-selector-volume: {{ .Values.defaultSettings.allowEmptyDiskSelectorVolume }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.allowCollectingLonghornUsageMetrics) }} + allow-collecting-longhorn-usage-metrics: {{ .Values.defaultSettings.allowCollectingLonghornUsageMetrics }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.disableSnapshotPurge) }} + disable-snapshot-purge: {{ .Values.defaultSettings.disableSnapshotPurge }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.v2DataEngineGuaranteedInstanceManagerCPU) }} + v2-data-engine-guaranteed-instance-manager-cpu: {{ .Values.defaultSettings.v2DataEngineGuaranteedInstanceManagerCPU }} + {{- end }} + {{- if not (kindIs "invalid" .Values.defaultSettings.snapshotMaxCount) }} + snapshot-max-count: {{ .Values.defaultSettings.snapshotMaxCount }} + {{- end }} \ No newline at end of file diff --git a/charts/longhorn/102.4.1+up1.6.2/templates/deployment-driver.yaml b/charts/longhorn/102.4.1+up1.6.2/templates/deployment-driver.yaml new file mode 100644 index 0000000000..cd2ab3a344 --- /dev/null +++ b/charts/longhorn/102.4.1+up1.6.2/templates/deployment-driver.yaml @@ -0,0 +1,132 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: longhorn-driver-deployer + namespace: {{ include "release_namespace" . }} + labels: {{- include "longhorn.labels" . | nindent 4 }} +spec: + replicas: 1 + selector: + matchLabels: + app: longhorn-driver-deployer + template: + metadata: + labels: {{- include "longhorn.labels" . | nindent 8 }} + app: longhorn-driver-deployer + spec: + initContainers: + - name: wait-longhorn-manager + image: {{ template "registry_url" . }}{{ .Values.image.longhorn.manager.repository }}:{{ .Values.image.longhorn.manager.tag }} + command: ['sh', '-c', 'while [ $(curl -m 1 -s -o /dev/null -w "%{http_code}" http://longhorn-backend:9500/v1) != "200" ]; do echo waiting; sleep 2; done'] + containers: + - name: longhorn-driver-deployer + image: {{ template "registry_url" . }}{{ .Values.image.longhorn.manager.repository }}:{{ .Values.image.longhorn.manager.tag }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - longhorn-manager + - -d + - deploy-driver + - --manager-image + - "{{ template "registry_url" . }}{{ .Values.image.longhorn.manager.repository }}:{{ .Values.image.longhorn.manager.tag }}" + - --manager-url + - http://longhorn-backend:9500/v1 + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: SERVICE_ACCOUNT + valueFrom: + fieldRef: + fieldPath: spec.serviceAccountName + {{- if .Values.csi.kubeletRootDir }} + - name: KUBELET_ROOT_DIR + value: {{ .Values.csi.kubeletRootDir }} + {{- end }} + {{- if and .Values.image.csi.attacher.repository .Values.image.csi.attacher.tag }} + - name: CSI_ATTACHER_IMAGE + value: "{{ template "registry_url" . }}{{ .Values.image.csi.attacher.repository }}:{{ .Values.image.csi.attacher.tag }}" + {{- end }} + {{- if and .Values.image.csi.provisioner.repository .Values.image.csi.provisioner.tag }} + - name: CSI_PROVISIONER_IMAGE + value: "{{ template "registry_url" . }}{{ .Values.image.csi.provisioner.repository }}:{{ .Values.image.csi.provisioner.tag }}" + {{- end }} + {{- if and .Values.image.csi.nodeDriverRegistrar.repository .Values.image.csi.nodeDriverRegistrar.tag }} + - name: CSI_NODE_DRIVER_REGISTRAR_IMAGE + value: "{{ template "registry_url" . }}{{ .Values.image.csi.nodeDriverRegistrar.repository }}:{{ .Values.image.csi.nodeDriverRegistrar.tag }}" + {{- end }} + {{- if and .Values.image.csi.resizer.repository .Values.image.csi.resizer.tag }} + - name: CSI_RESIZER_IMAGE + value: "{{ template "registry_url" . }}{{ .Values.image.csi.resizer.repository }}:{{ .Values.image.csi.resizer.tag }}" + {{- end }} + {{- if and .Values.image.csi.snapshotter.repository .Values.image.csi.snapshotter.tag }} + - name: CSI_SNAPSHOTTER_IMAGE + value: "{{ template "registry_url" . }}{{ .Values.image.csi.snapshotter.repository }}:{{ .Values.image.csi.snapshotter.tag }}" + {{- end }} + {{- if and .Values.image.csi.livenessProbe.repository .Values.image.csi.livenessProbe.tag }} + - name: CSI_LIVENESS_PROBE_IMAGE + value: "{{ template "registry_url" . }}{{ .Values.image.csi.livenessProbe.repository }}:{{ .Values.image.csi.livenessProbe.tag }}" + {{- end }} + {{- if .Values.csi.attacherReplicaCount }} + - name: CSI_ATTACHER_REPLICA_COUNT + value: {{ .Values.csi.attacherReplicaCount | quote }} + {{- end }} + {{- if .Values.csi.provisionerReplicaCount }} + - name: CSI_PROVISIONER_REPLICA_COUNT + value: {{ .Values.csi.provisionerReplicaCount | quote }} + {{- end }} + {{- if .Values.csi.resizerReplicaCount }} + - name: CSI_RESIZER_REPLICA_COUNT + value: {{ .Values.csi.resizerReplicaCount | quote }} + {{- end }} + {{- if .Values.csi.snapshotterReplicaCount }} + - name: CSI_SNAPSHOTTER_REPLICA_COUNT + value: {{ .Values.csi.snapshotterReplicaCount | quote }} + {{- end }} + {{- if .Values.enableGoCoverDir }} + - name: GOCOVERDIR + value: /go-cover-dir/ + volumeMounts: + - name: go-cover-dir + mountPath: /go-cover-dir/ + {{- end }} + + {{- if .Values.privateRegistry.registrySecret }} + imagePullSecrets: + - name: {{ .Values.privateRegistry.registrySecret }} + {{- end }} + {{- if .Values.longhornDriver.priorityClass }} + priorityClassName: {{ .Values.longhornDriver.priorityClass | quote }} + {{- end }} + {{- if or .Values.longhornDriver.tolerations .Values.global.cattle.windowsCluster.enabled }} + tolerations: + {{- if and .Values.global.cattle.windowsCluster.enabled .Values.global.cattle.windowsCluster.tolerations }} +{{ toYaml .Values.global.cattle.windowsCluster.tolerations | indent 6 }} + {{- end }} + {{- if .Values.longhornDriver.tolerations }} +{{ toYaml .Values.longhornDriver.tolerations | indent 6 }} + {{- end }} + {{- end }} + {{- if or .Values.longhornDriver.nodeSelector .Values.global.cattle.windowsCluster.enabled }} + nodeSelector: + {{- if and .Values.global.cattle.windowsCluster.enabled .Values.global.cattle.windowsCluster.nodeSelector }} +{{ toYaml .Values.global.cattle.windowsCluster.nodeSelector | indent 8 }} + {{- end }} + {{- if .Values.longhornDriver.nodeSelector }} +{{ toYaml .Values.longhornDriver.nodeSelector | indent 8 }} + {{- end }} + {{- end }} + serviceAccountName: longhorn-service-account + securityContext: + runAsUser: 0 + {{- if .Values.enableGoCoverDir }} + volumes: + - name: go-cover-dir + hostPath: + path: /go-cover-dir/ + type: DirectoryOrCreate + {{- end }} diff --git a/charts/longhorn/102.4.1+up1.6.2/templates/deployment-ui.yaml b/charts/longhorn/102.4.1+up1.6.2/templates/deployment-ui.yaml new file mode 100644 index 0000000000..0ee86c7904 --- /dev/null +++ b/charts/longhorn/102.4.1+up1.6.2/templates/deployment-ui.yaml @@ -0,0 +1,182 @@ +{{- if .Values.openshift.enabled }} +{{- if .Values.openshift.ui.route }} +# https://github.com/openshift/oauth-proxy/blob/master/contrib/sidecar.yaml +# Create a proxy service account and ensure it will use the route "proxy" +# Create a secure connection to the proxy via a route +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + labels: {{- include "longhorn.labels" . | nindent 4 }} + app: longhorn-ui + name: {{ .Values.openshift.ui.route }} + namespace: {{ include "release_namespace" . }} +spec: + to: + kind: Service + name: longhorn-ui + tls: + termination: reencrypt +--- +apiVersion: v1 +kind: Service +metadata: + labels: {{- include "longhorn.labels" . | nindent 4 }} + app: longhorn-ui + name: longhorn-ui + namespace: {{ include "release_namespace" . }} + annotations: + service.alpha.openshift.io/serving-cert-secret-name: longhorn-ui-tls +spec: + ports: + - name: longhorn-ui + port: {{ .Values.openshift.ui.port | default 443 }} + targetPort: {{ .Values.openshift.ui.proxy | default 8443 }} + selector: + app: longhorn-ui +--- +{{- end }} +{{- end }} +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: {{- include "longhorn.labels" . | nindent 4 }} + app: longhorn-ui + name: longhorn-ui + namespace: {{ include "release_namespace" . }} +spec: + replicas: {{ .Values.longhornUI.replicas }} + selector: + matchLabels: + app: longhorn-ui + template: + metadata: + labels: {{- include "longhorn.labels" . | nindent 8 }} + app: longhorn-ui + spec: + serviceAccountName: longhorn-ui-service-account + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - longhorn-ui + topologyKey: kubernetes.io/hostname + containers: + {{- if .Values.openshift.enabled }} + {{- if .Values.openshift.ui.route }} + - name: oauth-proxy + image: {{ template "registry_url" . }}{{ .Values.image.openshift.oauthProxy.repository }}:{{ .Values.image.openshift.oauthProxy.tag }} + imagePullPolicy: IfNotPresent + ports: + - containerPort: {{ .Values.openshift.ui.proxy | default 8443 }} + name: public + args: + - --https-address=:{{ .Values.openshift.ui.proxy | default 8443 }} + - --provider=openshift + - --openshift-service-account=longhorn-ui-service-account + - --upstream=http://localhost:8000 + - --tls-cert=/etc/tls/private/tls.crt + - --tls-key=/etc/tls/private/tls.key + - --cookie-secret=SECRET + - --openshift-sar={"namespace":"{{ include "release_namespace" . }}","group":"longhorn.io","resource":"setting","verb":"delete"} + volumeMounts: + - mountPath: /etc/tls/private + name: longhorn-ui-tls + {{- end }} + {{- end }} + - name: longhorn-ui + image: {{ template "registry_url" . }}{{ .Values.image.longhorn.ui.repository }}:{{ .Values.image.longhorn.ui.tag }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + volumeMounts: + - name : nginx-cache + mountPath: /var/cache/nginx/ + - name : nginx-config + mountPath: /var/config/nginx/ + - name: var-run + mountPath: /var/run/ + ports: + - containerPort: 8000 + name: http + env: + - name: LONGHORN_MANAGER_IP + value: "http://longhorn-backend:9500" + - name: LONGHORN_UI_PORT + value: "8000" + volumes: + {{- if .Values.openshift.enabled }} + {{- if .Values.openshift.ui.route }} + - name: longhorn-ui-tls + secret: + secretName: longhorn-ui-tls + {{- end }} + {{- end }} + - emptyDir: {} + name: nginx-cache + - emptyDir: {} + name: nginx-config + - emptyDir: {} + name: var-run + {{- if .Values.privateRegistry.registrySecret }} + imagePullSecrets: + - name: {{ .Values.privateRegistry.registrySecret }} + {{- end }} + {{- if .Values.longhornUI.priorityClass }} + priorityClassName: {{ .Values.longhornUI.priorityClass | quote }} + {{- end }} + {{- if or .Values.longhornUI.tolerations .Values.global.cattle.windowsCluster.enabled }} + tolerations: + {{- if and .Values.global.cattle.windowsCluster.enabled .Values.global.cattle.windowsCluster.tolerations }} +{{ toYaml .Values.global.cattle.windowsCluster.tolerations | indent 6 }} + {{- end }} + {{- if .Values.longhornUI.tolerations }} +{{ toYaml .Values.longhornUI.tolerations | indent 6 }} + {{- end }} + {{- end }} + {{- if or .Values.longhornUI.nodeSelector .Values.global.cattle.windowsCluster.enabled }} + nodeSelector: + {{- if and .Values.global.cattle.windowsCluster.enabled .Values.global.cattle.windowsCluster.nodeSelector }} +{{ toYaml .Values.global.cattle.windowsCluster.nodeSelector | indent 8 }} + {{- end }} + {{- if .Values.longhornUI.nodeSelector }} +{{ toYaml .Values.longhornUI.nodeSelector | indent 8 }} + {{- end }} + {{- end }} +--- +kind: Service +apiVersion: v1 +metadata: + labels: {{- include "longhorn.labels" . | nindent 4 }} + app: longhorn-ui + {{- if eq .Values.service.ui.type "Rancher-Proxy" }} + kubernetes.io/cluster-service: "true" + {{- end }} + name: longhorn-frontend + namespace: {{ include "release_namespace" . }} +spec: + {{- if eq .Values.service.ui.type "Rancher-Proxy" }} + type: ClusterIP + {{- else }} + type: {{ .Values.service.ui.type }} + {{- end }} + {{- if and .Values.service.ui.loadBalancerIP (eq .Values.service.ui.type "LoadBalancer") }} + loadBalancerIP: {{ .Values.service.ui.loadBalancerIP }} + {{- end }} + {{- if and (eq .Values.service.ui.type "LoadBalancer") .Values.service.ui.loadBalancerSourceRanges }} + loadBalancerSourceRanges: {{- toYaml .Values.service.ui.loadBalancerSourceRanges | nindent 4 }} + {{- end }} + selector: + app: longhorn-ui + ports: + - name: http + port: 80 + targetPort: http + {{- if .Values.service.ui.nodePort }} + nodePort: {{ .Values.service.ui.nodePort }} + {{- else }} + nodePort: null + {{- end }} diff --git a/charts/longhorn/102.4.1+up1.6.2/templates/ingress.yaml b/charts/longhorn/102.4.1+up1.6.2/templates/ingress.yaml new file mode 100644 index 0000000000..9038ff0cc1 --- /dev/null +++ b/charts/longhorn/102.4.1+up1.6.2/templates/ingress.yaml @@ -0,0 +1,37 @@ +{{- if .Values.ingress.enabled }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: longhorn-ingress + namespace: {{ include "release_namespace" . }} + labels: {{- include "longhorn.labels" . | nindent 4 }} + app: longhorn-ingress + annotations: + {{- if .Values.ingress.secureBackends }} + ingress.kubernetes.io/secure-backends: "true" + {{- end }} + {{- range $key, $value := .Values.ingress.annotations }} + {{ $key }}: {{ $value | quote }} + {{- end }} +spec: + {{- if .Values.ingress.ingressClassName }} + ingressClassName: {{ .Values.ingress.ingressClassName }} + {{- end }} + rules: + - host: {{ .Values.ingress.host }} + http: + paths: + - path: {{ default "" .Values.ingress.path }} + pathType: ImplementationSpecific + backend: + service: + name: longhorn-frontend + port: + number: 80 +{{- if .Values.ingress.tls }} + tls: + - hosts: + - {{ .Values.ingress.host }} + secretName: {{ .Values.ingress.tlsSecret }} +{{- end }} +{{- end }} diff --git a/charts/longhorn/102.4.1+up1.6.2/templates/network-policies/backing-image-data-source-network-policy.yaml b/charts/longhorn/102.4.1+up1.6.2/templates/network-policies/backing-image-data-source-network-policy.yaml new file mode 100644 index 0000000000..7204d63caa --- /dev/null +++ b/charts/longhorn/102.4.1+up1.6.2/templates/network-policies/backing-image-data-source-network-policy.yaml @@ -0,0 +1,27 @@ +{{- if .Values.networkPolicies.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: backing-image-data-source + namespace: {{ include "release_namespace" . }} +spec: + podSelector: + matchLabels: + longhorn.io/component: backing-image-data-source + policyTypes: + - Ingress + ingress: + - from: + - podSelector: + matchLabels: + app: longhorn-manager + - podSelector: + matchLabels: + longhorn.io/component: instance-manager + - podSelector: + matchLabels: + longhorn.io/component: backing-image-manager + - podSelector: + matchLabels: + longhorn.io/component: backing-image-data-source +{{- end }} diff --git a/charts/longhorn/102.4.1+up1.6.2/templates/network-policies/backing-image-manager-network-policy.yaml b/charts/longhorn/102.4.1+up1.6.2/templates/network-policies/backing-image-manager-network-policy.yaml new file mode 100644 index 0000000000..119ebf08a1 --- /dev/null +++ b/charts/longhorn/102.4.1+up1.6.2/templates/network-policies/backing-image-manager-network-policy.yaml @@ -0,0 +1,27 @@ +{{- if .Values.networkPolicies.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: backing-image-manager + namespace: {{ include "release_namespace" . }} +spec: + podSelector: + matchLabels: + longhorn.io/component: backing-image-manager + policyTypes: + - Ingress + ingress: + - from: + - podSelector: + matchLabels: + app: longhorn-manager + - podSelector: + matchLabels: + longhorn.io/component: instance-manager + - podSelector: + matchLabels: + longhorn.io/component: backing-image-manager + - podSelector: + matchLabels: + longhorn.io/component: backing-image-data-source +{{- end }} diff --git a/charts/longhorn/102.4.1+up1.6.2/templates/network-policies/instance-manager-networking.yaml b/charts/longhorn/102.4.1+up1.6.2/templates/network-policies/instance-manager-networking.yaml new file mode 100644 index 0000000000..332aa2c2fe --- /dev/null +++ b/charts/longhorn/102.4.1+up1.6.2/templates/network-policies/instance-manager-networking.yaml @@ -0,0 +1,27 @@ +{{- if .Values.networkPolicies.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: instance-manager + namespace: {{ include "release_namespace" . }} +spec: + podSelector: + matchLabels: + longhorn.io/component: instance-manager + policyTypes: + - Ingress + ingress: + - from: + - podSelector: + matchLabels: + app: longhorn-manager + - podSelector: + matchLabels: + longhorn.io/component: instance-manager + - podSelector: + matchLabels: + longhorn.io/component: backing-image-manager + - podSelector: + matchLabels: + longhorn.io/component: backing-image-data-source +{{- end }} diff --git a/charts/longhorn/102.4.1+up1.6.2/templates/network-policies/manager-network-policy.yaml b/charts/longhorn/102.4.1+up1.6.2/templates/network-policies/manager-network-policy.yaml new file mode 100644 index 0000000000..6f94029a53 --- /dev/null +++ b/charts/longhorn/102.4.1+up1.6.2/templates/network-policies/manager-network-policy.yaml @@ -0,0 +1,35 @@ +{{- if .Values.networkPolicies.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: longhorn-manager + namespace: {{ include "release_namespace" . }} +spec: + podSelector: + matchLabels: + app: longhorn-manager + policyTypes: + - Ingress + ingress: + - from: + - podSelector: + matchLabels: + app: longhorn-manager + - podSelector: + matchLabels: + app: longhorn-ui + - podSelector: + matchLabels: + app: longhorn-csi-plugin + - podSelector: + matchLabels: + longhorn.io/managed-by: longhorn-manager + matchExpressions: + - { key: recurring-job.longhorn.io, operator: Exists } + - podSelector: + matchExpressions: + - { key: longhorn.io/job-task, operator: Exists } + - podSelector: + matchLabels: + app: longhorn-driver-deployer +{{- end }} diff --git a/charts/longhorn/102.4.1+up1.6.2/templates/network-policies/recovery-backend-network-policy.yaml b/charts/longhorn/102.4.1+up1.6.2/templates/network-policies/recovery-backend-network-policy.yaml new file mode 100644 index 0000000000..6e34dadfc2 --- /dev/null +++ b/charts/longhorn/102.4.1+up1.6.2/templates/network-policies/recovery-backend-network-policy.yaml @@ -0,0 +1,17 @@ +{{- if .Values.networkPolicies.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: longhorn-recovery-backend + namespace: {{ include "release_namespace" . }} +spec: + podSelector: + matchLabels: + app: longhorn-manager + policyTypes: + - Ingress + ingress: + - ports: + - protocol: TCP + port: 9503 +{{- end }} diff --git a/charts/longhorn/102.4.1+up1.6.2/templates/network-policies/ui-frontend-network-policy.yaml b/charts/longhorn/102.4.1+up1.6.2/templates/network-policies/ui-frontend-network-policy.yaml new file mode 100644 index 0000000000..6f37065980 --- /dev/null +++ b/charts/longhorn/102.4.1+up1.6.2/templates/network-policies/ui-frontend-network-policy.yaml @@ -0,0 +1,46 @@ +{{- if and .Values.networkPolicies.enabled .Values.ingress.enabled (not (eq .Values.networkPolicies.type "")) }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: longhorn-ui-frontend + namespace: {{ include "release_namespace" . }} +spec: + podSelector: + matchLabels: + app: longhorn-ui + policyTypes: + - Ingress + ingress: + - from: + {{- if eq .Values.networkPolicies.type "rke1"}} + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: ingress-nginx + podSelector: + matchLabels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + {{- else if eq .Values.networkPolicies.type "rke2" }} + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: rke2-ingress-nginx + app.kubernetes.io/name: rke2-ingress-nginx + {{- else if eq .Values.networkPolicies.type "k3s" }} + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + app.kubernetes.io/name: traefik + ports: + - port: 8000 + protocol: TCP + - port: 80 + protocol: TCP + {{- end }} +{{- end }} diff --git a/charts/longhorn/102.4.1+up1.6.2/templates/network-policies/webhook-network-policy.yaml b/charts/longhorn/102.4.1+up1.6.2/templates/network-policies/webhook-network-policy.yaml new file mode 100644 index 0000000000..3575763d39 --- /dev/null +++ b/charts/longhorn/102.4.1+up1.6.2/templates/network-policies/webhook-network-policy.yaml @@ -0,0 +1,33 @@ +{{- if .Values.networkPolicies.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: longhorn-conversion-webhook + namespace: {{ include "release_namespace" . }} +spec: + podSelector: + matchLabels: + app: longhorn-manager + policyTypes: + - Ingress + ingress: + - ports: + - protocol: TCP + port: 9501 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: longhorn-admission-webhook + namespace: {{ include "release_namespace" . }} +spec: + podSelector: + matchLabels: + app: longhorn-manager + policyTypes: + - Ingress + ingress: + - ports: + - protocol: TCP + port: 9502 +{{- end }} diff --git a/charts/longhorn/102.4.1+up1.6.2/templates/postupgrade-job.yaml b/charts/longhorn/102.4.1+up1.6.2/templates/postupgrade-job.yaml new file mode 100644 index 0000000000..bb25a54d4e --- /dev/null +++ b/charts/longhorn/102.4.1+up1.6.2/templates/postupgrade-job.yaml @@ -0,0 +1,56 @@ +apiVersion: batch/v1 +kind: Job +metadata: + annotations: + "helm.sh/hook": post-upgrade + "helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation + name: longhorn-post-upgrade + namespace: {{ include "release_namespace" . }} + labels: {{- include "longhorn.labels" . | nindent 4 }} +spec: + activeDeadlineSeconds: 900 + backoffLimit: 1 + template: + metadata: + name: longhorn-post-upgrade + labels: {{- include "longhorn.labels" . | nindent 8 }} + spec: + containers: + - name: longhorn-post-upgrade + image: {{ template "registry_url" . }}{{ .Values.image.longhorn.manager.repository }}:{{ .Values.image.longhorn.manager.tag }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - longhorn-manager + - post-upgrade + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + restartPolicy: OnFailure + {{- if .Values.privateRegistry.registrySecret }} + imagePullSecrets: + - name: {{ .Values.privateRegistry.registrySecret }} + {{- end }} + {{- if .Values.longhornManager.priorityClass }} + priorityClassName: {{ .Values.longhornManager.priorityClass | quote }} + {{- end }} + serviceAccountName: longhorn-service-account + {{- if or .Values.longhornManager.tolerations .Values.global.cattle.windowsCluster.enabled }} + tolerations: + {{- if and .Values.global.cattle.windowsCluster.enabled .Values.global.cattle.windowsCluster.tolerations }} +{{ toYaml .Values.global.cattle.windowsCluster.tolerations | indent 6 }} + {{- end }} + {{- if .Values.longhornManager.tolerations }} +{{ toYaml .Values.longhornManager.tolerations | indent 6 }} + {{- end }} + {{- end }} + {{- if or .Values.longhornManager.nodeSelector .Values.global.cattle.windowsCluster.enabled }} + nodeSelector: + {{- if and .Values.global.cattle.windowsCluster.enabled .Values.global.cattle.windowsCluster.nodeSelector }} +{{ toYaml .Values.global.cattle.windowsCluster.nodeSelector | indent 8 }} + {{- end }} + {{- if .Values.longhornManager.nodeSelector }} +{{ toYaml .Values.longhornManager.nodeSelector | indent 8 }} + {{- end }} + {{- end }} diff --git a/charts/longhorn/102.4.1+up1.6.2/templates/preupgrade-job.yaml b/charts/longhorn/102.4.1+up1.6.2/templates/preupgrade-job.yaml new file mode 100644 index 0000000000..ef0fe02f43 --- /dev/null +++ b/charts/longhorn/102.4.1+up1.6.2/templates/preupgrade-job.yaml @@ -0,0 +1,55 @@ +{{- if and .Values.preUpgradeChecker.jobEnabled .Values.preUpgradeChecker.upgradeVersionCheck}} +apiVersion: batch/v1 +kind: Job +metadata: + annotations: + "helm.sh/hook": pre-upgrade + "helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation,hook-failed + name: longhorn-pre-upgrade + namespace: {{ include "release_namespace" . }} + labels: {{- include "longhorn.labels" . | nindent 4 }} +spec: + activeDeadlineSeconds: 900 + backoffLimit: 1 + template: + metadata: + name: longhorn-pre-upgrade + labels: {{- include "longhorn.labels" . | nindent 8 }} + spec: + containers: + - name: longhorn-pre-upgrade + image: {{ template "registry_url" . }}{{ .Values.image.longhorn.manager.repository }}:{{ .Values.image.longhorn.manager.tag }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - longhorn-manager + - pre-upgrade + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + restartPolicy: OnFailure + {{- if .Values.privateRegistry.registrySecret }} + imagePullSecrets: + - name: {{ .Values.privateRegistry.registrySecret }} + {{- end }} + serviceAccountName: longhorn-service-account + {{- if or .Values.longhornManager.tolerations .Values.global.cattle.windowsCluster.enabled }} + tolerations: + {{- if and .Values.global.cattle.windowsCluster.enabled .Values.global.cattle.windowsCluster.tolerations }} +{{ toYaml .Values.global.cattle.windowsCluster.tolerations | indent 6 }} + {{- end }} + {{- if .Values.longhornManager.tolerations }} +{{ toYaml .Values.longhornManager.tolerations | indent 6 }} + {{- end }} + {{- end }} + {{- if or .Values.longhornManager.nodeSelector .Values.global.cattle.windowsCluster.enabled }} + nodeSelector: + {{- if and .Values.global.cattle.windowsCluster.enabled .Values.global.cattle.windowsCluster.nodeSelector }} +{{ toYaml .Values.global.cattle.windowsCluster.nodeSelector | indent 8 }} + {{- end }} + {{- if .Values.longhornManager.nodeSelector }} +{{ toYaml .Values.longhornManager.nodeSelector | indent 8 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/longhorn/102.4.1+up1.6.2/templates/priorityclass.yaml b/charts/longhorn/102.4.1+up1.6.2/templates/priorityclass.yaml new file mode 100644 index 0000000000..208adc84a2 --- /dev/null +++ b/charts/longhorn/102.4.1+up1.6.2/templates/priorityclass.yaml @@ -0,0 +1,9 @@ +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +metadata: + name: "longhorn-critical" + labels: {{- include "longhorn.labels" . | nindent 4 }} +description: "Ensure Longhorn pods have the highest priority to prevent any unexpected eviction by the Kubernetes scheduler under node pressure" +globalDefault: false +preemptionPolicy: PreemptLowerPriority +value: 1000000000 diff --git a/charts/longhorn/102.4.1+up1.6.2/templates/psp.yaml b/charts/longhorn/102.4.1+up1.6.2/templates/psp.yaml new file mode 100644 index 0000000000..a2dfc05bef --- /dev/null +++ b/charts/longhorn/102.4.1+up1.6.2/templates/psp.yaml @@ -0,0 +1,66 @@ +{{- if .Values.enablePSP }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: longhorn-psp + labels: {{- include "longhorn.labels" . | nindent 4 }} +spec: + privileged: true + allowPrivilegeEscalation: true + requiredDropCapabilities: + - NET_RAW + allowedCapabilities: + - SYS_ADMIN + hostNetwork: false + hostIPC: false + hostPID: true + runAsUser: + rule: RunAsAny + seLinux: + rule: RunAsAny + fsGroup: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + volumes: + - configMap + - downwardAPI + - emptyDir + - secret + - projected + - hostPath +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: longhorn-psp-role + labels: {{- include "longhorn.labels" . | nindent 4 }} + namespace: {{ include "release_namespace" . }} +rules: +- apiGroups: + - policy + resources: + - podsecuritypolicies + verbs: + - use + resourceNames: + - longhorn-psp +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: longhorn-psp-binding + labels: {{- include "longhorn.labels" . | nindent 4 }} + namespace: {{ include "release_namespace" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: longhorn-psp-role +subjects: +- kind: ServiceAccount + name: longhorn-service-account + namespace: {{ include "release_namespace" . }} +- kind: ServiceAccount + name: default + namespace: {{ include "release_namespace" . }} +{{- end }} diff --git a/charts/longhorn/102.4.1+up1.6.2/templates/registry-secret.yaml b/charts/longhorn/102.4.1+up1.6.2/templates/registry-secret.yaml new file mode 100644 index 0000000000..3c6b1dc510 --- /dev/null +++ b/charts/longhorn/102.4.1+up1.6.2/templates/registry-secret.yaml @@ -0,0 +1,13 @@ +{{- if .Values.privateRegistry.createSecret }} +{{- if .Values.privateRegistry.registrySecret }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Values.privateRegistry.registrySecret }} + namespace: {{ include "release_namespace" . }} + labels: {{- include "longhorn.labels" . | nindent 4 }} +type: kubernetes.io/dockerconfigjson +data: + .dockerconfigjson: {{ template "secret" . }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/longhorn/102.4.1+up1.6.2/templates/serviceaccount.yaml b/charts/longhorn/102.4.1+up1.6.2/templates/serviceaccount.yaml new file mode 100644 index 0000000000..b0d6dd505b --- /dev/null +++ b/charts/longhorn/102.4.1+up1.6.2/templates/serviceaccount.yaml @@ -0,0 +1,40 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: longhorn-service-account + namespace: {{ include "release_namespace" . }} + labels: {{- include "longhorn.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: longhorn-ui-service-account + namespace: {{ include "release_namespace" . }} + labels: {{- include "longhorn.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if .Values.openshift.enabled }} + {{- if .Values.openshift.ui.route }} + {{- if not .Values.serviceAccount.annotations }} + annotations: + {{- end }} + serviceaccounts.openshift.io/oauth-redirectreference.primary: '{"kind":"OAuthRedirectReference","apiVersion":"v1","reference":{"kind":"Route","name":"longhorn-ui"}}' + {{- end }} + {{- end }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: longhorn-support-bundle + namespace: {{ include "release_namespace" . }} + labels: {{- include "longhorn.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} \ No newline at end of file diff --git a/charts/longhorn/102.4.1+up1.6.2/templates/servicemonitor.yaml b/charts/longhorn/102.4.1+up1.6.2/templates/servicemonitor.yaml new file mode 100644 index 0000000000..fd11fe9d47 --- /dev/null +++ b/charts/longhorn/102.4.1+up1.6.2/templates/servicemonitor.yaml @@ -0,0 +1,19 @@ +{{- if .Values.metrics.serviceMonitor.enabled -}} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: longhorn-prometheus-servicemonitor + namespace: {{ include "release_namespace" . }} + labels: + {{- include "longhorn.labels" . | nindent 4 }} + name: longhorn-prometheus-servicemonitor +spec: + selector: + matchLabels: + app: longhorn-manager + namespaceSelector: + matchNames: + - {{ include "release_namespace" . }} + endpoints: + - port: manager +{{- end }} \ No newline at end of file diff --git a/charts/longhorn/102.4.1+up1.6.2/templates/services.yaml b/charts/longhorn/102.4.1+up1.6.2/templates/services.yaml new file mode 100644 index 0000000000..8baef021f3 --- /dev/null +++ b/charts/longhorn/102.4.1+up1.6.2/templates/services.yaml @@ -0,0 +1,71 @@ +apiVersion: v1 +kind: Service +metadata: + labels: {{- include "longhorn.labels" . | nindent 4 }} + app: longhorn-conversion-webhook + name: longhorn-conversion-webhook + namespace: {{ include "release_namespace" . }} +spec: + type: ClusterIP + selector: + app: longhorn-manager + ports: + - name: conversion-webhook + port: 9501 + targetPort: conversion-wh +--- +apiVersion: v1 +kind: Service +metadata: + labels: {{- include "longhorn.labels" . | nindent 4 }} + app: longhorn-admission-webhook + name: longhorn-admission-webhook + namespace: {{ include "release_namespace" . }} +spec: + type: ClusterIP + selector: + app: longhorn-manager + ports: + - name: admission-webhook + port: 9502 + targetPort: admission-wh +--- +apiVersion: v1 +kind: Service +metadata: + labels: {{- include "longhorn.labels" . | nindent 4 }} + app: longhorn-recovery-backend + name: longhorn-recovery-backend + namespace: {{ include "release_namespace" . }} +spec: + type: ClusterIP + selector: + app: longhorn-manager + ports: + - name: recovery-backend + port: 9503 + targetPort: recov-backend +--- +apiVersion: v1 +kind: Service +metadata: + labels: {{- include "longhorn.labels" . | nindent 4 }} + name: longhorn-engine-manager + namespace: {{ include "release_namespace" . }} +spec: + clusterIP: None + selector: + longhorn.io/component: instance-manager + longhorn.io/instance-manager-type: engine +--- +apiVersion: v1 +kind: Service +metadata: + labels: {{- include "longhorn.labels" . | nindent 4 }} + name: longhorn-replica-manager + namespace: {{ include "release_namespace" . }} +spec: + clusterIP: None + selector: + longhorn.io/component: instance-manager + longhorn.io/instance-manager-type: replica diff --git a/charts/longhorn/102.4.1+up1.6.2/templates/storageclass.yaml b/charts/longhorn/102.4.1+up1.6.2/templates/storageclass.yaml new file mode 100644 index 0000000000..f79699f5e0 --- /dev/null +++ b/charts/longhorn/102.4.1+up1.6.2/templates/storageclass.yaml @@ -0,0 +1,50 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: longhorn-storageclass + namespace: {{ include "release_namespace" . }} + labels: {{- include "longhorn.labels" . | nindent 4 }} +data: + storageclass.yaml: | + kind: StorageClass + apiVersion: storage.k8s.io/v1 + metadata: + name: longhorn + annotations: + storageclass.kubernetes.io/is-default-class: {{ .Values.persistence.defaultClass | quote }} + provisioner: driver.longhorn.io + allowVolumeExpansion: true + reclaimPolicy: "{{ .Values.persistence.reclaimPolicy }}" + volumeBindingMode: Immediate + parameters: + numberOfReplicas: "{{ .Values.persistence.defaultClassReplicaCount }}" + staleReplicaTimeout: "30" + fromBackup: "" + {{- if .Values.persistence.defaultFsType }} + fsType: "{{ .Values.persistence.defaultFsType }}" + {{- end }} + {{- if .Values.persistence.defaultMkfsParams }} + mkfsParams: "{{ .Values.persistence.defaultMkfsParams }}" + {{- end }} + {{- if .Values.persistence.migratable }} + migratable: "{{ .Values.persistence.migratable }}" + {{- end }} + {{- if .Values.persistence.nfsOptions }} + nfsOptions: "{{ .Values.persistence.nfsOptions }}" + {{- end }} + {{- if .Values.persistence.backingImage.enable }} + backingImage: {{ .Values.persistence.backingImage.name }} + backingImageDataSourceType: {{ .Values.persistence.backingImage.dataSourceType }} + backingImageDataSourceParameters: {{ .Values.persistence.backingImage.dataSourceParameters }} + backingImageChecksum: {{ .Values.persistence.backingImage.expectedChecksum }} + {{- end }} + {{- if .Values.persistence.recurringJobSelector.enable }} + recurringJobSelector: '{{ .Values.persistence.recurringJobSelector.jobList }}' + {{- end }} + dataLocality: {{ .Values.persistence.defaultDataLocality | quote }} + {{- if .Values.persistence.defaultNodeSelector.enable }} + nodeSelector: "{{ .Values.persistence.defaultNodeSelector.selector }}" + {{- end }} + {{- if .Values.persistence.removeSnapshotsDuringFilesystemTrim }} + unmapMarkSnapChainRemoved: "{{ .Values.persistence.removeSnapshotsDuringFilesystemTrim }}" + {{- end }} diff --git a/charts/longhorn/102.4.1+up1.6.2/templates/tls-secrets.yaml b/charts/longhorn/102.4.1+up1.6.2/templates/tls-secrets.yaml new file mode 100644 index 0000000000..74c43426de --- /dev/null +++ b/charts/longhorn/102.4.1+up1.6.2/templates/tls-secrets.yaml @@ -0,0 +1,16 @@ +{{- if .Values.ingress.enabled }} +{{- range .Values.ingress.secrets }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ .name }} + namespace: {{ include "release_namespace" $ }} + labels: {{- include "longhorn.labels" $ | nindent 4 }} + app: longhorn +type: kubernetes.io/tls +data: + tls.crt: {{ .certificate | b64enc }} + tls.key: {{ .key | b64enc }} +--- +{{- end }} +{{- end }} diff --git a/charts/longhorn/102.4.1+up1.6.2/templates/uninstall-job.yaml b/charts/longhorn/102.4.1+up1.6.2/templates/uninstall-job.yaml new file mode 100644 index 0000000000..968f420616 --- /dev/null +++ b/charts/longhorn/102.4.1+up1.6.2/templates/uninstall-job.yaml @@ -0,0 +1,57 @@ +apiVersion: batch/v1 +kind: Job +metadata: + annotations: + "helm.sh/hook": pre-delete + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + name: longhorn-uninstall + namespace: {{ include "release_namespace" . }} + labels: {{- include "longhorn.labels" . | nindent 4 }} +spec: + activeDeadlineSeconds: 900 + backoffLimit: 1 + template: + metadata: + name: longhorn-uninstall + labels: {{- include "longhorn.labels" . | nindent 8 }} + spec: + containers: + - name: longhorn-uninstall + image: {{ template "registry_url" . }}{{ .Values.image.longhorn.manager.repository }}:{{ .Values.image.longhorn.manager.tag }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - longhorn-manager + - uninstall + - --force + env: + - name: LONGHORN_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + restartPolicy: Never + {{- if .Values.privateRegistry.registrySecret }} + imagePullSecrets: + - name: {{ .Values.privateRegistry.registrySecret }} + {{- end }} + {{- if .Values.longhornManager.priorityClass }} + priorityClassName: {{ .Values.longhornManager.priorityClass | quote }} + {{- end }} + serviceAccountName: longhorn-service-account + {{- if or .Values.longhornManager.tolerations .Values.global.cattle.windowsCluster.enabled }} + tolerations: + {{- if and .Values.global.cattle.windowsCluster.enabled .Values.global.cattle.windowsCluster.tolerations }} +{{ toYaml .Values.global.cattle.windowsCluster.tolerations | indent 6 }} + {{- end }} + {{- if .Values.longhornManager.tolerations }} +{{ toYaml .Values.longhornManager.tolerations | indent 6 }} + {{- end }} + {{- end }} + {{- if or .Values.longhornManager.nodeSelector .Values.global.cattle.windowsCluster.enabled }} + nodeSelector: + {{- if and .Values.global.cattle.windowsCluster.enabled .Values.global.cattle.windowsCluster.nodeSelector }} +{{ toYaml .Values.global.cattle.windowsCluster.nodeSelector | indent 8 }} + {{- end }} + {{- if or .Values.longhornManager.nodeSelector }} +{{ toYaml .Values.longhornManager.nodeSelector | indent 8 }} + {{- end }} + {{- end }} diff --git a/charts/longhorn/102.4.1+up1.6.2/templates/userroles.yaml b/charts/longhorn/102.4.1+up1.6.2/templates/userroles.yaml new file mode 100644 index 0000000000..1dbb6be90e --- /dev/null +++ b/charts/longhorn/102.4.1+up1.6.2/templates/userroles.yaml @@ -0,0 +1,53 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: "longhorn-admin" + labels: + rbac.authorization.k8s.io/aggregate-to-admin: "true" +rules: +- apiGroups: [ "longhorn.io" ] + resources: ["volumes", "volumes/status", "engines", "engines/status", "replicas", "replicas/status", "settings", + "engineimages", "engineimages/status", "nodes", "nodes/status", "instancemanagers", "instancemanagers/status", + "sharemanagers", "sharemanagers/status", "backingimages", "backingimages/status", + "backingimagemanagers", "backingimagemanagers/status", "backingimagedatasources", "backingimagedatasources/status", "backupbackingimages", "backupbackingimages/status", + "backuptargets", "backuptargets/status", "backupvolumes", "backupvolumes/status", "backups", "backups/status", + "recurringjobs", "recurringjobs/status", "orphans", "orphans/status", "snapshots", "snapshots/status", + "supportbundles", "supportbundles/status", "systembackups", "systembackups/status", "systemrestores", "systemrestores/status", + "volumeattachments", "volumeattachments/status"] + verbs: [ "*" ] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: "longhorn-edit" + labels: + rbac.authorization.k8s.io/aggregate-to-edit: "true" +rules: +- apiGroups: [ "longhorn.io" ] + resources: ["volumes", "volumes/status", "engines", "engines/status", "replicas", "replicas/status", "settings", + "engineimages", "engineimages/status", "nodes", "nodes/status", "instancemanagers", "instancemanagers/status", + "sharemanagers", "sharemanagers/status", "backingimages", "backingimages/status", + "backingimagemanagers", "backingimagemanagers/status", "backingimagedatasources", "backingimagedatasources/status", "backupbackingimages", "backupbackingimages/status", + "backuptargets", "backuptargets/status", "backupvolumes", "backupvolumes/status", "backups", "backups/status", + "recurringjobs", "recurringjobs/status", "orphans", "orphans/status", "snapshots", "snapshots/status", + "supportbundles", "supportbundles/status", "systembackups", "systembackups/status", "systemrestores", "systemrestores/status", + "volumeattachments", "volumeattachments/status"] + verbs: [ "*" ] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: "longhorn-view" + labels: + rbac.authorization.k8s.io/aggregate-to-view: "true" +rules: +- apiGroups: [ "longhorn.io" ] + resources: ["volumes", "volumes/status", "engines", "engines/status", "replicas", "replicas/status", "settings", + "engineimages", "engineimages/status", "nodes", "nodes/status", "instancemanagers", "instancemanagers/status", + "sharemanagers", "sharemanagers/status", "backingimages", "backingimages/status", + "backingimagemanagers", "backingimagemanagers/status", "backingimagedatasources", "backingimagedatasources/status", "backupbackingimages", "backupbackingimages/status", + "backuptargets", "backuptargets/status", "backupvolumes", "backupvolumes/status", "backups", "backups/status", + "recurringjobs", "recurringjobs/status", "orphans", "orphans/status", "snapshots", "snapshots/status", + "supportbundles", "supportbundles/status", "systembackups", "systembackups/status", "systemrestores", "systemrestores/status", + "volumeattachments", "volumeattachments/status"] + verbs: [ "get", "list", "watch" ] diff --git a/charts/longhorn/102.4.1+up1.6.2/templates/validate-install-crd.yaml b/charts/longhorn/102.4.1+up1.6.2/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..aac4dd9c53 --- /dev/null +++ b/charts/longhorn/102.4.1+up1.6.2/templates/validate-install-crd.yaml @@ -0,0 +1,35 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "longhorn.io/v1beta1/BackingImageDataSource" false -}} +# {{- set $found "longhorn.io/v1beta1/BackingImageManager" false -}} +# {{- set $found "longhorn.io/v1beta1/BackingImage" false -}} +# {{- set $found "longhorn.io/v1beta1/Backup" false -}} +# {{- set $found "longhorn.io/v1beta2/BackupBackingImage" false -}} +# {{- set $found "longhorn.io/v1beta1/BackupTarget" false -}} +# {{- set $found "longhorn.io/v1beta1/BackupVolume" false -}} +# {{- set $found "longhorn.io/v1beta1/EngineImage" false -}} +# {{- set $found "longhorn.io/v1beta1/Engine" false -}} +# {{- set $found "longhorn.io/v1beta1/InstanceManager" false -}} +# {{- set $found "longhorn.io/v1beta1/Node" false -}} +# {{- set $found "longhorn.io/v1beta2/Orphan" false -}} +# {{- set $found "longhorn.io/v1beta1/RecurringJob" false -}} +# {{- set $found "longhorn.io/v1beta1/Replica" false -}} +# {{- set $found "longhorn.io/v1beta1/Setting" false -}} +# {{- set $found "longhorn.io/v1beta1/ShareManager" false -}} +# {{- set $found "longhorn.io/v1beta2/Snapshot" false -}} +# {{- set $found "longhorn.io/v1beta2/SupportBundle" false -}} +# {{- set $found "longhorn.io/v1beta2/SystemBackup" false -}} +# {{- set $found "longhorn.io/v1beta2/SystemRestore" false -}} +# {{- set $found "longhorn.io/v1beta1/Volume" false -}} +# {{- set $found "longhorn.io/v1beta2/VolumeAttachment" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install the corresponding CRD chart before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/longhorn/102.4.1+up1.6.2/templates/validate-psp-install.yaml b/charts/longhorn/102.4.1+up1.6.2/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..0df98e3657 --- /dev/null +++ b/charts/longhorn/102.4.1+up1.6.2/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.enablePSP }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} \ No newline at end of file diff --git a/charts/longhorn/102.4.1+up1.6.2/values.yaml b/charts/longhorn/102.4.1+up1.6.2/values.yaml new file mode 100644 index 0000000000..1cc85b92d1 --- /dev/null +++ b/charts/longhorn/102.4.1+up1.6.2/values.yaml @@ -0,0 +1,484 @@ +# Default values for longhorn. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. +global: + cattle: + # -- Default system registry. + systemDefaultRegistry: "" + windowsCluster: + # -- Setting that allows Longhorn to run on a Rancher Windows cluster. + enabled: false + # -- Toleration for Linux nodes that can run user-deployed Longhorn components. + tolerations: + - key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" + # -- Node selector for Linux nodes that can run user-deployed Longhorn components. + nodeSelector: + kubernetes.io/os: "linux" + defaultSetting: + # -- Toleration for system-managed Longhorn components. + taintToleration: cattle.io/os=linux:NoSchedule + # -- Node selector for system-managed Longhorn components. + systemManagedComponentsNodeSelector: kubernetes.io/os:linux + +networkPolicies: + # -- Setting that allows you to enable network policies that control access to Longhorn pods. + enabled: false + # -- Distribution that determines the policy for allowing access for an ingress. (Options: "k3s", "rke2", "rke1") + type: "k3s" + +image: + longhorn: + engine: + # -- Repository for the Longhorn Engine image. + repository: rancher/mirrored-longhornio-longhorn-engine + # -- Specify Longhorn engine image tag + tag: v1.6.2 + manager: + # -- Repository for the Longhorn Manager image. + repository: rancher/mirrored-longhornio-longhorn-manager + # -- Specify Longhorn manager image tag + tag: v1.6.2 + ui: + # -- Repository for the Longhorn UI image. + repository: rancher/mirrored-longhornio-longhorn-ui + # -- Specify Longhorn ui image tag + tag: v1.6.2 + instanceManager: + # -- Repository for the Longhorn Instance Manager image. + repository: rancher/mirrored-longhornio-longhorn-instance-manager + # -- Specify Longhorn instance manager image tag + tag: v1.6.2 + shareManager: + # -- Repository for the Longhorn Share Manager image. + repository: rancher/mirrored-longhornio-longhorn-share-manager + # -- Specify Longhorn share manager image tag + tag: v1.6.2 + backingImageManager: + # -- Repository for the Backing Image Manager image. When unspecified, Longhorn uses the default value. + repository: rancher/mirrored-longhornio-backing-image-manager + # -- Specify Longhorn backing image manager image tag + tag: v1.6.2 + supportBundleKit: + # -- Repository for the Longhorn Support Bundle Manager image. + repository: rancher/mirrored-longhornio-support-bundle-kit + # -- Tag for the Longhorn Support Bundle Manager image. + tag: v0.0.37 + csi: + attacher: + # -- Repository for the CSI attacher image. When unspecified, Longhorn uses the default value. + repository: rancher/mirrored-longhornio-csi-attacher + # -- Tag for the CSI attacher image. When unspecified, Longhorn uses the default value. + tag: v4.5.1 + provisioner: + # -- Repository for the CSI Provisioner image. When unspecified, Longhorn uses the default value. + repository: rancher/mirrored-longhornio-csi-provisioner + # -- Tag for the CSI Provisioner image. When unspecified, Longhorn uses the default value. + tag: v3.6.4 + nodeDriverRegistrar: + # -- Repository for the CSI Node Driver Registrar image. When unspecified, Longhorn uses the default value. + repository: rancher/mirrored-longhornio-csi-node-driver-registrar + # -- Tag for the CSI Node Driver Registrar image. When unspecified, Longhorn uses the default value. + tag: v2.9.2 + resizer: + # -- Repository for the CSI Resizer image. When unspecified, Longhorn uses the default value. + repository: rancher/mirrored-longhornio-csi-resizer + # -- Tag for the CSI Resizer image. When unspecified, Longhorn uses the default value. + tag: v1.10.1 + snapshotter: + # -- Repository for the CSI Snapshotter image. When unspecified, Longhorn uses the default value. + repository: rancher/mirrored-longhornio-csi-snapshotter + # -- Tag for the CSI Snapshotter image. When unspecified, Longhorn uses the default value. + tag: v6.3.4 + livenessProbe: + # -- Repository for the CSI liveness probe image. When unspecified, Longhorn uses the default value. + repository: rancher/mirrored-longhornio-livenessprobe + # -- Tag for the CSI liveness probe image. When unspecified, Longhorn uses the default value. + tag: v2.12.0 + openshift: + oauthProxy: + # -- Repository for the OAuth Proxy image. This setting applies only to OpenShift users. + repository: rancher/mirrored-longhornio-openshift-origin-oauth-proxy + # -- Tag for the OAuth Proxy image. This setting applies only to OpenShift users. Specify OCP/OKD version 4.1 or later. The latest stable version is 4.14. + tag: 4.14 + # -- Image pull policy that applies to all user-deployed Longhorn components, such as Longhorn Manager, Longhorn driver, and Longhorn UI. + pullPolicy: IfNotPresent + +service: + ui: + # -- Service type for Longhorn UI. (Options: "ClusterIP", "NodePort", "LoadBalancer", "Rancher-Proxy") + type: ClusterIP + # -- NodePort port number for Longhorn UI. When unspecified, Longhorn selects a free port between 30000 and 32767. + nodePort: null + manager: + # -- Service type for Longhorn Manager. + type: ClusterIP + # -- NodePort port number for Longhorn Manager. When unspecified, Longhorn selects a free port between 30000 and 32767. + nodePort: "" + +persistence: + # -- Setting that allows you to specify the default Longhorn StorageClass. + defaultClass: true + # -- Filesystem type of the default Longhorn StorageClass. + defaultFsType: ext4 + # -- mkfs parameters of the default Longhorn StorageClass. + defaultMkfsParams: "" + # -- Replica count of the default Longhorn StorageClass. + defaultClassReplicaCount: 3 + # -- Data locality of the default Longhorn StorageClass. (Options: "disabled", "best-effort") + defaultDataLocality: disabled + # -- Reclaim policy that provides instructions for handling of a volume after its claim is released. (Options: "Retain", "Delete") + reclaimPolicy: Delete + # -- Setting that allows you to enable live migration of a Longhorn volume from one node to another. + migratable: false + # -- Set NFS mount options for Longhorn StorageClass for RWX volumes + nfsOptions: "" + recurringJobSelector: + # -- Setting that allows you to enable the recurring job selector for a Longhorn StorageClass. + enable: false + # -- Recurring job selector for a Longhorn StorageClass. Ensure that quotes are used correctly when specifying job parameters. (Example: `[{"name":"backup", "isGroup":true}]`) + jobList: [] + backingImage: + # -- Setting that allows you to use a backing image in a Longhorn StorageClass. + enable: false + # -- Backing image to be used for creating and restoring volumes in a Longhorn StorageClass. When no backing images are available, specify the data source type and parameters that Longhorn can use to create a backing image. + name: ~ + # -- Data source type of a backing image used in a Longhorn StorageClass. + # If the backing image exists in the cluster, Longhorn uses this setting to verify the image. + # If the backing image does not exist, Longhorn creates one using the specified data source type. + dataSourceType: ~ + # -- Data source parameters of a backing image used in a Longhorn StorageClass. + # You can specify a JSON string of a map. (Example: `'{\"url\":\"https://backing-image-example.s3-region.amazonaws.com/test-backing-image\"}'`) + dataSourceParameters: ~ + # -- Expected SHA-512 checksum of a backing image used in a Longhorn StorageClass. + expectedChecksum: ~ + defaultNodeSelector: + # -- Setting that allows you to enable the node selector for the default Longhorn StorageClass. + enable: false + # -- Node selector for the default Longhorn StorageClass. Longhorn uses only nodes with the specified tags for storing volume data. (Examples: "storage,fast") + selector: "" + # -- Setting that allows you to enable automatic snapshot removal during filesystem trim for a Longhorn StorageClass. (Options: "ignored", "enabled", "disabled") + removeSnapshotsDuringFilesystemTrim: ignored + +preUpgradeChecker: + # -- Setting that allows Longhorn to perform pre-upgrade checks. Disable this setting when installing Longhorn using Argo CD or other GitOps solutions. + jobEnabled: true + # -- Setting that allows Longhorn to perform upgrade version checks after starting the Longhorn Manager DaemonSet Pods. Disabling this setting also disables `preUpgradeChecker.jobEnabled`. Longhorn recommends keeping this setting enabled. + upgradeVersionCheck: true + +csi: + # -- kubelet root directory. When unspecified, Longhorn uses the default value. + kubeletRootDir: ~ + # -- Replica count of the CSI Attacher. When unspecified, Longhorn uses the default value ("3"). + attacherReplicaCount: ~ + # -- Replica count of the CSI Provisioner. When unspecified, Longhorn uses the default value ("3"). + provisionerReplicaCount: ~ + # -- Replica count of the CSI Resizer. When unspecified, Longhorn uses the default value ("3"). + resizerReplicaCount: ~ + # -- Replica count of the CSI Snapshotter. When unspecified, Longhorn uses the default value ("3"). + snapshotterReplicaCount: ~ + +defaultSettings: + # -- Endpoint used to access the backupstore. (Options: "NFS", "CIFS", "AWS", "GCP", "AZURE") + backupTarget: ~ + # -- Name of the Kubernetes secret associated with the backup target. + backupTargetCredentialSecret: ~ + # -- Setting that allows Longhorn to automatically attach a volume and create snapshots or backups when recurring jobs are run. + allowRecurringJobWhileVolumeDetached: ~ + # -- Setting that allows Longhorn to automatically create a default disk only on nodes with the label "node.longhorn.io/create-default-disk=true" (if no other disks exist). When this setting is disabled, Longhorn creates a default disk on each node that is added to the cluster. + createDefaultDiskLabeledNodes: ~ + # -- Default path for storing data on a host. The default value is "/var/lib/longhorn/". + defaultDataPath: ~ + # -- Default data locality. A Longhorn volume has data locality if a local replica of the volume exists on the same node as the pod that is using the volume. + defaultDataLocality: ~ + # -- Setting that allows scheduling on nodes with healthy replicas of the same volume. This setting is disabled by default. + replicaSoftAntiAffinity: ~ + # -- Setting that automatically rebalances replicas when an available node is discovered. + replicaAutoBalance: ~ + # -- Percentage of storage that can be allocated relative to hard drive capacity. The default value is "100". + storageOverProvisioningPercentage: ~ + # -- Percentage of minimum available disk capacity. When the minimum available capacity exceeds the total available capacity, the disk becomes unschedulable until more space is made available for use. The default value is "25". + storageMinimalAvailablePercentage: ~ + # -- Percentage of disk space that is not allocated to the default disk on each new Longhorn node. + storageReservedPercentageForDefaultDisk: ~ + # -- Upgrade Checker that periodically checks for new Longhorn versions. When a new version is available, a notification appears on the Longhorn UI. This setting is enabled by default + upgradeChecker: ~ + # -- Default number of replicas for volumes created using the Longhorn UI. For Kubernetes configuration, modify the `numberOfReplicas` field in the StorageClass. The default value is "3". + defaultReplicaCount: ~ + # -- Default Longhorn StorageClass. "storageClassName" is assigned to PVs and PVCs that are created for an existing Longhorn volume. "storageClassName" can also be used as a label, so it is possible to use a Longhorn StorageClass to bind a workload to an existing PV without creating a Kubernetes StorageClass object. The default value is "longhorn-static". + defaultLonghornStaticStorageClass: ~ + # -- Number of seconds that Longhorn waits before checking the backupstore for new backups. The default value is "300". When the value is "0", polling is disabled. + backupstorePollInterval: ~ + # -- Number of minutes that Longhorn keeps a failed backup resource. When the value is "0", automatic deletion is disabled. + failedBackupTTL: ~ + # -- Setting that restores recurring jobs from a backup volume on a backup target and creates recurring jobs if none exist during backup restoration. + restoreVolumeRecurringJobs: ~ + # -- Maximum number of successful recurring backup and snapshot jobs to be retained. When the value is "0", a history of successful recurring jobs is not retained. + recurringSuccessfulJobsHistoryLimit: ~ + # -- Maximum number of failed recurring backup and snapshot jobs to be retained. When the value is "0", a history of failed recurring jobs is not retained. + recurringFailedJobsHistoryLimit: ~ + # -- Maximum number of snapshots or backups to be retained. + recurringJobMaxRetention: ~ + # -- Maximum number of failed support bundles that can exist in the cluster. When the value is "0", Longhorn automatically purges all failed support bundles. + supportBundleFailedHistoryLimit: ~ + # -- Taint or toleration for system-managed Longhorn components. + # Specify values using a semicolon-separated list in `kubectl taint` syntax (Example: key1=value1:effect; key2=value2:effect). + taintToleration: ~ + # -- Node selector for system-managed Longhorn components. + systemManagedComponentsNodeSelector: ~ + # -- PriorityClass for system-managed Longhorn components. + # This setting can help prevent Longhorn components from being evicted under Node Pressure. + # Notice that this will be applied to Longhorn user-deployed components by default if there are no priority class values set yet, such as `longhornManager.priorityClass`. + priorityClass: &defaultPriorityClassNameRef "longhorn-critical" + # -- Setting that allows Longhorn to automatically salvage volumes when all replicas become faulty (for example, when the network connection is interrupted). Longhorn determines which replicas are usable and then uses these replicas for the volume. This setting is enabled by default. + autoSalvage: ~ + # -- Setting that allows Longhorn to automatically delete a workload pod that is managed by a controller (for example, daemonset) whenever a Longhorn volume is detached unexpectedly (for example, during Kubernetes upgrades). After deletion, the controller restarts the pod and then Kubernetes handles volume reattachment and remounting. + autoDeletePodWhenVolumeDetachedUnexpectedly: ~ + # -- Setting that prevents Longhorn Manager from scheduling replicas on a cordoned Kubernetes node. This setting is enabled by default. + disableSchedulingOnCordonedNode: ~ + # -- Setting that allows Longhorn to schedule new replicas of a volume to nodes in the same zone as existing healthy replicas. Nodes that do not belong to any zone are treated as existing in the zone that contains healthy replicas. When identifying zones, Longhorn relies on the label "topology.kubernetes.io/zone=" in the Kubernetes node object. + replicaZoneSoftAntiAffinity: ~ + # -- Setting that allows scheduling on disks with existing healthy replicas of the same volume. This setting is enabled by default. + replicaDiskSoftAntiAffinity: ~ + # -- Policy that defines the action Longhorn takes when a volume is stuck with a StatefulSet or Deployment pod on a node that failed. + nodeDownPodDeletionPolicy: ~ + # -- Policy that defines the action Longhorn takes when a node with the last healthy replica of a volume is drained. + nodeDrainPolicy: ~ + # -- Setting that allows automatic detaching of manually-attached volumes when a node is cordoned. + detachManuallyAttachedVolumesWhenCordoned: ~ + # -- Number of seconds that Longhorn waits before reusing existing data on a failed replica instead of creating a new replica of a degraded volume. + replicaReplenishmentWaitInterval: ~ + # -- Maximum number of replicas that can be concurrently rebuilt on each node. + concurrentReplicaRebuildPerNodeLimit: ~ + # -- Maximum number of volumes that can be concurrently restored on each node using a backup. When the value is "0", restoration of volumes using a backup is disabled. + concurrentVolumeBackupRestorePerNodeLimit: ~ + # -- Setting that disables the revision counter and thereby prevents Longhorn from tracking all write operations to a volume. When salvaging a volume, Longhorn uses properties of the "volume-head-xxx.img" file (the last file size and the last time the file was modified) to select the replica to be used for volume recovery. This setting applies only to volumes created using the Longhorn UI. + disableRevisionCounter: ~ + # -- Image pull policy for system-managed pods, such as Instance Manager, engine images, and CSI Driver. Changes to the image pull policy are applied only after the system-managed pods restart. + systemManagedPodsImagePullPolicy: ~ + # -- Setting that allows you to create and attach a volume without having all replicas scheduled at the time of creation. + allowVolumeCreationWithDegradedAvailability: ~ + # -- Setting that allows Longhorn to automatically clean up the system-generated snapshot after replica rebuilding is completed. + autoCleanupSystemGeneratedSnapshot: ~ + # -- Setting that allows Longhorn to automatically clean up the snapshot generated by a recurring backup job. + autoCleanupRecurringJobBackupSnapshot: ~ + # -- Maximum number of engines that are allowed to concurrently upgrade on each node after Longhorn Manager is upgraded. When the value is "0", Longhorn does not automatically upgrade volume engines to the new default engine image version. + concurrentAutomaticEngineUpgradePerNodeLimit: ~ + # -- Number of minutes that Longhorn waits before cleaning up the backing image file when no replicas in the disk are using it. + backingImageCleanupWaitInterval: ~ + # -- Number of seconds that Longhorn waits before downloading a backing image file again when the status of all image disk files changes to "failed" or "unknown". + backingImageRecoveryWaitInterval: ~ + # -- Percentage of the total allocatable CPU resources on each node to be reserved for each instance manager pod when the V1 Data Engine is enabled. The default value is "12". + guaranteedInstanceManagerCPU: ~ + # -- Setting that notifies Longhorn that the cluster is using the Kubernetes Cluster Autoscaler. + kubernetesClusterAutoscalerEnabled: ~ + # -- Setting that allows Longhorn to automatically delete an orphaned resource and the corresponding data (for example, stale replicas). Orphaned resources on failed or unknown nodes are not automatically cleaned up. + orphanAutoDeletion: ~ + # -- Storage network for in-cluster traffic. When unspecified, Longhorn uses the Kubernetes cluster network. + storageNetwork: ~ + # -- Flag that prevents accidental uninstallation of Longhorn. + deletingConfirmationFlag: ~ + # -- Timeout between the Longhorn Engine and replicas. Specify a value between "8" and "30" seconds. The default value is "8". + engineReplicaTimeout: ~ + # -- Setting that allows you to enable and disable snapshot hashing and data integrity checks. + snapshotDataIntegrity: ~ + # -- Setting that allows disabling of snapshot hashing after snapshot creation to minimize impact on system performance. + snapshotDataIntegrityImmediateCheckAfterSnapshotCreation: ~ + # -- Setting that defines when Longhorn checks the integrity of data in snapshot disk files. You must use the Unix cron expression format. + snapshotDataIntegrityCronjob: ~ + # -- Setting that allows Longhorn to automatically mark the latest snapshot and its parent files as removed during a filesystem trim. Longhorn does not remove snapshots containing multiple child files. + removeSnapshotsDuringFilesystemTrim: ~ + # -- Setting that allows fast rebuilding of replicas using the checksum of snapshot disk files. Before enabling this setting, you must set the snapshot-data-integrity value to "enable" or "fast-check". + fastReplicaRebuildEnabled: ~ + # -- Number of seconds that an HTTP client waits for a response from a File Sync server before considering the connection to have failed. + replicaFileSyncHttpClientTimeout: ~ + # -- Log levels that indicate the type and severity of logs in Longhorn Manager. The default value is "Info". (Options: "Panic", "Fatal", "Error", "Warn", "Info", "Debug", "Trace") + logLevel: ~ + # -- Setting that allows you to specify a backup compression method. + backupCompressionMethod: ~ + # -- Maximum number of worker threads that can concurrently run for each backup. + backupConcurrentLimit: ~ + # -- Maximum number of worker threads that can concurrently run for each restore operation. + restoreConcurrentLimit: ~ + # -- Setting that allows you to enable the V1 Data Engine. + v1DataEngine: ~ + # -- Setting that allows you to enable the V2 Data Engine, which is based on the Storage Performance Development Kit (SPDK). The V2 Data Engine is a preview feature and should not be used in production environments. + v2DataEngine: ~ + # -- Setting that allows you to configure maximum huge page size (in MiB) for the V2 Data Engine. + v2DataEngineHugepageLimit: ~ + # -- Setting that allows rebuilding of offline replicas for volumes using the V2 Data Engine. + offlineReplicaRebuilding: ~ + # -- Number of millicpus on each node to be reserved for each Instance Manager pod when the V2 Data Engine is enabled. The default value is "1250". + v2DataEngineGuaranteedInstanceManagerCPU: ~ + # -- Setting that allows scheduling of empty node selector volumes to any node. + allowEmptyNodeSelectorVolume: ~ + # -- Setting that allows scheduling of empty disk selector volumes to any disk. + allowEmptyDiskSelectorVolume: ~ + # -- Setting that allows Longhorn to periodically collect anonymous usage data for product improvement purposes. Longhorn sends collected data to the [Upgrade Responder](https://github.com/longhorn/upgrade-responder) server, which is the data source of the Longhorn Public Metrics Dashboard (https://metrics.longhorn.io). The Upgrade Responder server does not store data that can be used to identify clients, including IP addresses. + allowCollectingLonghornUsageMetrics: ~ + # -- Setting that temporarily prevents all attempts to purge volume snapshots. + disableSnapshotPurge: ~ + # -- Maximum snapshot count for a volume. The value should be between 2 to 250 + snapshotMaxCount: ~ + +privateRegistry: + # -- Setting that allows you to create a private registry secret. + createSecret: ~ + # -- URL of a private registry. When unspecified, Longhorn uses the default system registry. + registryUrl: ~ + # -- User account used for authenticating with a private registry. + registryUser: ~ + # -- Password for authenticating with a private registry. + registryPasswd: ~ + # -- Kubernetes secret that allows you to pull images from a private registry. This setting applies only when creation of private registry secrets is enabled. You must include the private registry name in the secret name. + registrySecret: ~ + +longhornManager: + log: + # -- Format of Longhorn Manager logs. (Options: "plain", "json") + format: plain + # -- PriorityClass for Longhorn Manager. + priorityClass: *defaultPriorityClassNameRef + # -- Toleration for Longhorn Manager on nodes allowed to run Longhorn Manager. + tolerations: [] + ## If you want to set tolerations for Longhorn Manager DaemonSet, delete the `[]` in the line above + ## and uncomment this example block + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + # -- Node selector for Longhorn Manager. Specify the nodes allowed to run Longhorn Manager. + nodeSelector: {} + ## If you want to set node selector for Longhorn Manager DaemonSet, delete the `{}` in the line above + ## and uncomment this example block + # label-key1: "label-value1" + # label-key2: "label-value2" + # -- Annotation for the Longhorn Manager service. + serviceAnnotations: {} + ## If you want to set annotations for the Longhorn Manager service, delete the `{}` in the line above + ## and uncomment this example block + # annotation-key1: "annotation-value1" + # annotation-key2: "annotation-value2" + +longhornDriver: + # -- PriorityClass for Longhorn Driver. + priorityClass: *defaultPriorityClassNameRef + # -- Toleration for Longhorn Driver on nodes allowed to run Longhorn components. + tolerations: [] + ## If you want to set tolerations for Longhorn Driver Deployer Deployment, delete the `[]` in the line above + ## and uncomment this example block + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + # -- Node selector for Longhorn Driver. Specify the nodes allowed to run Longhorn Driver. + nodeSelector: {} + ## If you want to set node selector for Longhorn Driver Deployer Deployment, delete the `{}` in the line above + ## and uncomment this example block + # label-key1: "label-value1" + # label-key2: "label-value2" + +longhornUI: + # -- Replica count for Longhorn UI. + replicas: 2 + # -- PriorityClass for Longhorn UI. + priorityClass: *defaultPriorityClassNameRef + # -- Toleration for Longhorn UI on nodes allowed to run Longhorn components. + tolerations: [] + ## If you want to set tolerations for Longhorn UI Deployment, delete the `[]` in the line above + ## and uncomment this example block + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + # -- Node selector for Longhorn UI. Specify the nodes allowed to run Longhorn UI. + nodeSelector: {} + ## If you want to set node selector for Longhorn UI Deployment, delete the `{}` in the line above + ## and uncomment this example block + # label-key1: "label-value1" + # label-key2: "label-value2" + +ingress: + # -- Setting that allows Longhorn to generate ingress records for the Longhorn UI service. + enabled: false + + # -- IngressClass resource that contains ingress configuration, including the name of the Ingress controller. + # ingressClassName can replace the kubernetes.io/ingress.class annotation used in earlier Kubernetes releases. + ingressClassName: ~ + + # -- Hostname of the Layer 7 load balancer. + host: sslip.io + + # -- Setting that allows you to enable TLS on ingress records. + tls: false + + # -- Setting that allows you to enable secure connections to the Longhorn UI service via port 443. + secureBackends: false + + # -- TLS secret that contains the private key and certificate to be used for TLS. This setting applies only when TLS is enabled on ingress records. + tlsSecret: longhorn.local-tls + + # -- Default ingress path. You can access the Longhorn UI by following the full ingress path {{host}}+{{path}}. + path: / + + ## If you're using kube-lego, you will want to add: + ## kubernetes.io/tls-acme: true + ## + ## For a full list of possible ingress annotations, please see + ## ref: https://github.com/kubernetes/ingress-nginx/blob/master/docs/annotations.md + ## + ## If tls is set to true, annotation ingress.kubernetes.io/secure-backends: "true" will automatically be set + # -- Ingress annotations in the form of key-value pairs. + annotations: + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: true + + # -- Secret that contains a TLS private key and certificate. Use secrets if you want to use your own certificates to secure ingresses. + secrets: + ## If you're providing your own certificates, please use this to add the certificates as secrets + ## key and certificate should start with -----BEGIN CERTIFICATE----- or + ## -----BEGIN RSA PRIVATE KEY----- + ## + ## name should line up with a tlsSecret set further up + ## If you're using kube-lego, this is unneeded, as it will create the secret for you if it is not set + ## + ## It is also possible to create and manage the certificates outside of this helm chart + ## Please see README.md for more information + # - name: longhorn.local-tls + # key: + # certificate: + +# -- Setting that allows you to enable pod security policies (PSPs) that allow privileged Longhorn pods to start. This setting applies only to clusters running Kubernetes 1.25 and earlier, and with the built-in Pod Security admission controller enabled. +enablePSP: false + +# -- Specify override namespace, specifically this is useful for using longhorn as sub-chart and its release namespace is not the `longhorn-system`. +namespaceOverride: "" + +# -- Annotation for the Longhorn Manager DaemonSet pods. This setting is optional. +annotations: {} + +serviceAccount: + # -- Annotations to add to the service account + annotations: {} + +metrics: + serviceMonitor: + # -- Setting that allows the creation of a Prometheus ServiceMonitor resource for Longhorn Manager components. + enabled: false + +## openshift settings +openshift: + # -- Setting that allows Longhorn to integrate with OpenShift. + enabled: false + ui: + # -- Route for connections between Longhorn and the OpenShift web console. + route: "longhorn-ui" + # -- Port for accessing the OpenShift web console. + port: 443 + # -- Port for proxy that provides access to the OpenShift web console. + proxy: 8443 + +# -- Setting that allows Longhorn to generate code coverage profiles. +enableGoCoverDir: false diff --git a/charts/prometheus-federator/3.0.2+up0.3.3/Chart.yaml b/charts/prometheus-federator/3.0.2+up0.3.3/Chart.yaml new file mode 100644 index 0000000000..14b0270e19 --- /dev/null +++ b/charts/prometheus-federator/3.0.2+up0.3.3/Chart.yaml @@ -0,0 +1,21 @@ +annotations: + catalog.cattle.io/certified: rancher + catalog.cattle.io/display-name: Prometheus Federator + catalog.cattle.io/kube-version: '>= 1.16.0-0 < 1.28.0-0' + catalog.cattle.io/namespace: cattle-monitoring-system + catalog.cattle.io/os: linux,windows + catalog.cattle.io/permits-os: linux,windows + catalog.cattle.io/provides-gvr: helm.cattle.io.projecthelmchart/v1alpha1 + catalog.cattle.io/rancher-version: '>= 2.7.0-0 < 2.8.0-0' + catalog.cattle.io/release-name: prometheus-federator +apiVersion: v2 +appVersion: 0.3.2 +dependencies: +- condition: helmProjectOperator.enabled + name: helmProjectOperator + repository: file://./charts/helmProjectOperator + version: 0.2.1 +description: Prometheus Federator +icon: https://raw.githubusercontent.com/rancher/prometheus-federator/main/assets/logos/prometheus-federator.svg +name: prometheus-federator +version: 3.0.2+up0.3.3 diff --git a/charts/prometheus-federator/3.0.2+up0.3.3/README.md b/charts/prometheus-federator/3.0.2+up0.3.3/README.md new file mode 100644 index 0000000000..7da4edfc22 --- /dev/null +++ b/charts/prometheus-federator/3.0.2+up0.3.3/README.md @@ -0,0 +1,120 @@ +# Prometheus Federator + +This chart is deploys a Helm Project Operator (based on the [rancher/helm-project-operator](https://github.com/rancher/helm-project-operator)), an operator that manages deploying Helm charts each containing a Project Monitoring Stack, where each stack contains: +- [Prometheus](https://prometheus.io/) (managed externally by [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator)) +- [Alertmanager](https://prometheus.io/docs/alerting/latest/alertmanager/) (managed externally by [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator)) +- [Grafana](https://github.com/helm/charts/tree/master/stable/grafana) (deployed via an embedded Helm chart) +- Default PrometheusRules and Grafana dashboards based on the collection of community-curated resources from [kube-prometheus](https://github.com/prometheus-operator/kube-prometheus/) +- Default ServiceMonitors that watch the deployed resources + +> **Important Note: Prometheus Federator is designed to be deployed alongside an existing Prometheus Operator deployment in a cluster that has already installed the Prometheus Operator CRDs.** + +By default, the chart is configured and intended to be deployed alongside [rancher-monitoring](https://rancher.com/docs/rancher/v2.6/en/monitoring-alerting/), which deploys Prometheus Operator alongside a Cluster Prometheus that each Project Monitoring Stack is configured to federate namespace-scoped metrics from by default. + +## Pre-Installation: Using Prometheus Federator with Rancher and rancher-monitoring + +If you are running your cluster on [Rancher](https://rancher.com/) and already have [rancher-monitoring](https://rancher.com/docs/rancher/v2.6/en/monitoring-alerting/) deployed onto your cluster, Prometheus Federator's default configuration should already be configured to work with your existing Cluster Monitoring Stack; however, here are some notes on how we recommend you configure rancher-monitoring to optimize the security and usability of Prometheus Federator in your cluster: + +### Ensure the cattle-monitoring-system namespace is placed into the System Project (or a similarly locked down Project that has access to other Projects in the cluster) + +Prometheus Operator's security model expects that the namespace it is deployed into (`cattle-monitoring-system`) has limited access for anyone except Cluster Admins to avoid privilege escalation via execing into Pods (such as the Jobs executing Helm operations). In addition, deploying Prometheus Federator and all Project Prometheus stacks into the System Project ensures that the each Project Prometheus is able to reach out to scrape workloads across all Projects (even if Network Policies are defined via Project Network Isolation) but has limited access for Project Owners, Project Members, and other users to be able to access data they shouldn't have access to (i.e. being allowed to exec into pods, set up the ability to scrape namespaces outside of a given Project, etc.). + +### Configure rancher-monitoring to only watch for resources created by the Helm chart itself + +Since each Project Monitoring Stack will watch the other namespaces and collect additional custom workload metrics or dashboards already, it's recommended to configure the following settings on all selectors to ensure that the Cluster Prometheus Stack only monitors resources created by the Helm Chart itself: + +``` +matchLabels: + release: "rancher-monitoring" +``` + +The following selector fields are recommended to have this value: +- `.Values.alertmanager.alertmanagerSpec.alertmanagerConfigSelector` +- `.Values.prometheus.prometheusSpec.serviceMonitorSelector` +- `.Values.prometheus.prometheusSpec.podMonitorSelector` +- `.Values.prometheus.prometheusSpec.ruleSelector` +- `.Values.prometheus.prometheusSpec.probeSelector` + +Once this setting is turned on, you can always create ServiceMonitors or PodMonitors that are picked up by the Cluster Prometheus by adding the label `release: "rancher-monitoring"` to them (in which case they will be ignored by Project Monitoring Stacks automatically by default, even if the namespace in which those ServiceMonitors or PodMonitors reside in are not system namespaces). + +> Note: If you don't want to allow users to be able to create ServiceMonitors and PodMonitors that aggregate into the Cluster Prometheus in Project namespaces, you can additionally set the namespaceSelectors on the chart to only target system namespaces (which must contain `cattle-monitoring-system` and `cattle-dashboards`, where resources are deployed into by default by rancher-monitoring; you will also need to monitor the `default` namespace to get apiserver metrics or create a custom ServiceMonitor to scrape apiserver metrics from the Service residing in the default namespace) to limit your Cluster Prometheus from picking up other Prometheus Operator CRs; in that case, it would be recommended to turn `.Values.prometheus.prometheusSpec.ignoreNamespaceSelectors=true` to allow you to define ServiceMonitors that can monitor non-system namespaces from within a system namespace. + +In addition, if you modified the default `.Values.grafana.sidecar.*.searchNamespace` values on the Grafana Helm subchart for Monitoring V2, it is also recommended to remove the overrides or ensure that your defaults are scoped to only system namespaces for the following values: +- `.Values.grafana.sidecar.dashboards.searchNamespace` (default `cattle-dashboards`) +- `.Values.grafana.sidecar.datasources.searchNamespace` (default `null`, which means it uses the release namespace `cattle-monitoring-system`) +- `.Values.grafana.sidecar.plugins.searchNamespace` (default `null`, which means it uses the release namespace `cattle-monitoring-system`) +- `.Values.grafana.sidecar.notifiers.searchNamespace` (default `null`, which means it uses the release namespace `cattle-monitoring-system`) + +### Increase the CPU / memory limits of the Cluster Prometheus + +Depending on a cluster's setup, it's generally recommended to give a large amount of dedicated memory to the Cluster Prometheus to avoid restarts due to out-of-memory errors (OOMKilled), usually caused by churn created in the cluster that causes a large number of high cardinality metrics to be generated and ingested by Prometheus within one block of time; this is one of the reasons why the default Rancher Monitoring stack expects around 4GB of RAM to be able to operate in a normal-sized cluster. However, when introducing Project Monitoring Stacks that are all sending `/federate` requests to the same Cluster Prometheus and are reliant on the Cluster Prometheus being "up" to federate that system data on their namespaces, it's even more important that the Cluster Prometheus has an ample amount of CPU / memory assigned to it to prevent an outage that can cause data gaps across all Project Prometheis in the cluster. + +> Note: There are no specific recommendations on how much memory the Cluster Prometheus should be configured with since it depends entirely on the user's setup (namely the likelihood of encountering a high churn rate and the scale of metrics that could be generated at that time); it generally varies per setup. + +## How does the operator work? + +1. On deploying this chart, users can create ProjectHelmCharts CRs with `spec.helmApiVersion` set to `monitoring.cattle.io/v1alpha1` (also known as "Project Monitors" in the Rancher UI) in a **Project Registration Namespace (`cattle-project-`)**. +2. On seeing each ProjectHelmChartCR, the operator will automatically deploy a Project Prometheus stack on the Project Owner's behalf in the **Project Release Namespace (`cattle-project--monitoring`)** based on a HelmChart CR and a HelmRelease CR automatically created by the ProjectHelmChart controller in the **Operator / System Namespace**. +3. RBAC will automatically be assigned in the Project Release Namespace to allow users to view the Prometheus, Alertmanager, and Grafana UIs of the Project Monitoring Stack deployed; this will be based on RBAC defined on the Project Registration Namespace against the [default Kubernetes user-facing roles](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles) (see below for more information about configuring RBAC). + +### What is a Project? + +In Prometheus Federator, a Project is a group of namespaces that can be identified by a `metav1.LabelSelector`; by default, the label used to identify projects is `field.cattle.io/projectId`, the label used to identify namespaces that are contained within a given [Rancher](https://rancher.com/) Project. + +### Configuring the Helm release created by a ProjectHelmChart + +The `spec.values` of this ProjectHelmChart resources will correspond to the `values.yaml` override to be supplied to the underlying Helm chart deployed by the operator on the user's behalf; to see the underlying chart's `values.yaml` spec, either: +- View to the chart's definition located at [`rancher/prometheus-federator` under `charts/rancher-project-monitoring`](https://github.com/rancher/prometheus-federator/blob/main/charts/rancher-project-monitoring) (where the chart version will be tied to the version of this operator) +- Look for the ConfigMap named `monitoring.cattle.io.v1alpha1` that is automatically created in each Project Registration Namespace, which will contain both the `values.yaml` and `questions.yaml` that was used to configure the chart (which was embedded directly into the `prometheus-federator` binary). + +### Namespaces + +As a Project Operator based on [rancher/helm-project-operator](https://github.com/rancher/helm-project-operator), Prometheus Federator has three different classifications of namespaces that the operator looks out for: +1. **Operator / System Namespace**: this is the namespace that the operator is deployed into (e.g. `cattle-monitoring-system`). This namespace will contain all HelmCharts and HelmReleases for all ProjectHelmCharts watched by this operator. **Only Cluster Admins should have access to this namespace.** +2. **Project Registration Namespace (`cattle-project-`)**: this is the set of namespaces that the operator watches for ProjectHelmCharts within. The RoleBindings and ClusterRoleBindings that apply to this namespace will also be the source of truth for the auto-assigned RBAC created in the Project Release Namespace (see more details below). **Project Owners (admin), Project Members (edit), and Read-Only Members (view) should have access to this namespace**. +> Note: Project Registration Namespaces will be auto-generated by the operator and imported into the Project it is tied to if `.Values.global.cattle.projectLabel` is provided (which is set to `field.cattle.io/projectId` by default); this indicates that a Project Registration Namespace should be created by the operator if at least one namespace is observed with that label. The operator will not let these namespaces be deleted unless either all namespaces with that label are gone (e.g. this is the last namespace in that project, in which case the namespace will be marked with the label `"helm.cattle.io/helm-project-operator-orphaned": "true"`, which signals that it can be deleted) or it is no longer watching that project (because the project ID was provided under `.Values.helmProjectOperator.otherSystemProjectLabelValues`, which serves as a denylist for Projects). These namespaces will also never be auto-deleted to avoid destroying user data; it is recommended that users clean up these namespaces manually if desired on creating or deleting a project +> Note: if `.Values.global.cattle.projectLabel` is not provided, the Operator / System Namespace will also be the Project Registration Namespace +3. **Project Release Namespace (`cattle-project--monitoring`)**: this is the set of namespaces that the operator deploys Project Monitoring Stacks within on behalf of a ProjectHelmChart; the operator will also automatically assign RBAC to Roles created in this namespace by the Project Monitoring Stack based on bindings found in the Project Registration Namespace. **Only Cluster Admins should have access to this namespace; Project Owners (admin), Project Members (edit), and Read-Only Members (view) will be assigned limited access to this namespace by the deployed Helm Chart and Prometheus Federator.** +> Note: Project Release Namespaces are automatically deployed and imported into the project whose ID is specified under `.Values.helmProjectOperator.projectReleaseNamespaces.labelValue` (which defaults to the value of `.Values.global.cattle.systemProjectId` if not specified) whenever a ProjectHelmChart is specified in a Project Registration Namespace +> Note: Project Release Namespaces follow the same orphaning conventions as Project Registration Namespaces (see note above) +> Note: if `.Values.projectReleaseNamespaces.enabled` is false, the Project Release Namespace will be the same as the Project Registration Namespace + +### Helm Resources (HelmChart, HelmRelease) + +On deploying a ProjectHelmChart, the Prometheus Federator will automatically create and manage two child custom resources that manage the underlying Helm resources in turn: +- A HelmChart CR (managed via an embedded [k3s-io/helm-contoller](https://github.com/k3s-io/helm-controller) in the operator): this custom resource automatically creates a Job in the same namespace that triggers a `helm install`, `helm upgrade`, or `helm uninstall` depending on the change applied to the HelmChart CR; this CR is automatically updated on changes to the ProjectHelmChart (e.g. modifying the values.yaml) or changes to the underlying Project definition (e.g. adding or removing namespaces from a project). +> **Important Note: If a ProjectHelmChart is not deploying or updating the underlying Project Monitoring Stack for some reason, the Job created by this resource in the Operator / System namespace should be the first place you check to see if there's something wrong with the Helm operation; however, this is generally only accessible by a Cluster Admin.** +- A HelmRelease CR (managed via an embedded [rancher/helm-locker](https://github.com/rancher/helm-locker) in the operator): this custom resource automatically locks a deployed Helm release in place and automatically overwrites updates to underlying resources unless the change happens via a Helm operation (`helm install`, `helm upgrade`, or `helm uninstall` performed by the HelmChart CR). +> Note: HelmRelease CRs emit Kubernetes Events that detect when an underlying Helm release is being modified and locks it back to place; to view these events, you can use `kubectl describe helmrelease -n `; you can also view the logs on this operator to see when changes are detected and which resources were attempted to be modified + +Both of these resources are created for all Helm charts in the Operator / System namespaces to avoid escalation of privileges to underprivileged users. + +### RBAC + +As described in the section on namespaces above, Prometheus Federator expects that Project Owners, Project Members, and other users in the cluster with Project-level permissions (e.g. permissions in a certain set of namespaces identified by a single label selector) have minimal permissions in any namespaces except the Project Registration Namespace (which is imported into the project by default) and those that already comprise their projects. Therefore, in order to allow Project Owners to assign specific chart permissions to other users in their Project namespaces, the Helm Project Operator will automatically watch the following bindings: +- ClusterRoleBindings +- RoleBindings in the Project Release Namespace + +On observing a change to one of those types of bindings, the Helm Project Operator will check whether the `roleRef` that the the binding points to matches a ClusterRole with the name provided under `helmProjectOperator.releaseRoleBindings.clusterRoleRefs.admin`, `helmProjectOperator.releaseRoleBindings.clusterRoleRefs.edit`, or `helmProjectOperator.releaseRoleBindings.clusterRoleRefs.view`; by default, these roleRefs correspond will correspond to `admin`, `edit`, and `view` respectively, which are the [default Kubernetes user-facing roles](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles). + +> Note: for Rancher RBAC users, these [default Kubernetes user-facing roles](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles) directly correlate to the `Project Owner`, `Project Member`, and `Read-Only` default Project Role Templates. + +If the `roleRef` matches, the Helm Project Operator will filter the `subjects` of the binding for all Users and Groups and use that to automatically construct a RoleBinding for each Role in the Project Release Namespace with the same name as the role and the following labels: +- `helm.cattle.io/project-helm-chart-role: {{ .Release.Name }}` +- `helm.cattle.io/project-helm-chart-role-aggregate-from: ` + +By default, the `rancher-project-monitoring` (the underlying chart deployed by Prometheus Federator) creates three default Roles per Project Release Namespace that provide `admin`, `edit`, and `view` users to permissions to view the Prometheus, Alertmanager, and Grafana UIs of the Project Monitoring Stack to provide least privilege; however, if a Cluster Admin would like to assign additional permissions to certain users, they can either directly assign RoleBindings in the Project Release Namespace to certain users or created Roles with the above two labels on them to allow Project Owners to control assigning those RBAC roles to users in their Project Registration namespaces. + +### Advanced Helm Project Operator Configuration + +|Value|Configuration| +|---|---------------------------| +|`helmProjectOperator.valuesOverride`| Allows an Operator to override values that are set on each ProjectHelmChart deployment on an operator-level; user-provided options (specified on the `spec.values` of the ProjectHelmChart) are automatically overridden if operator-level values are provided. For an exmaple, see how the default value overrides `federate.targets` (note: when overriding list values like `federate.targets`, user-provided list values will **not** be concatenated) | +|`helmProjectOperator.projectReleaseNamespaces.labelValues`| The value of the Project that all Project Release Namespaces should be auto-imported into (via label and annotation). Not recommended to be overridden on a Rancher setup. | +|`helmProjectOperator.otherSystemProjectLabelValues`| Other namespaces that the operator should treat as a system namespace that should not be monitored. By default, all namespaces that match `global.cattle.systemProjectId` will not be matched. `cattle-monitoring-system`, `cattle-dashboards`, and `kube-system` are explicitly marked as system namespaces as well, regardless of label or annotation. | +|`helmProjectOperator.releaseRoleBindings.aggregate`| Whether to automatically create RBAC resources in Project Release namespaces +|`helmProjectOperator.releaseRoleBindings.clusterRoleRefs.`| ClusterRoles to reference to discover subjects to create RoleBindings for in the Project Release Namespace for all corresponding Project Release Roles. See RBAC above for more information | +|`helmProjectOperator.hardenedNamespaces.enabled`| Whether to automatically patch the default ServiceAccount with `automountServiceAccountToken: false` and create a default NetworkPolicy in all managed namespaces in the cluster; the default values ensure that the creation of the namespace does not break a CIS 1.16 hardened scan | +|`helmProjectOperator.hardenedNamespaces.configuration`| The configuration to be supplied to the default ServiceAccount or auto-generated NetworkPolicy on managing a namespace | +|`helmProjectOperator.helmController.enabled`| Whether to enable an embedded k3s-io/helm-controller instance within the Helm Project Operator. Should be disabled for RKE2/K3s clusters before v1.23.14 / v1.24.8 / v1.25.4 since RKE2/K3s clusters already run Helm Controller at a cluster-wide level to manage internal Kubernetes components | +|`helmProjectOperator.helmLocker.enabled`| Whether to enable an embedded rancher/helm-locker instance within the Helm Project Operator. | diff --git a/charts/prometheus-federator/3.0.2+up0.3.3/app-README.md b/charts/prometheus-federator/3.0.2+up0.3.3/app-README.md new file mode 100644 index 0000000000..99fa7ca1c8 --- /dev/null +++ b/charts/prometheus-federator/3.0.2+up0.3.3/app-README.md @@ -0,0 +1,27 @@ +# Prometheus Federator + +This chart deploys an operator that manages Project Monitoring Stacks composed of the following set of resources that are scoped to project namespaces: +- [Prometheus](https://prometheus.io/) (managed externally by [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator)) +- [Alertmanager](https://prometheus.io/docs/alerting/latest/alertmanager/) (managed externally by [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator)) +- [Grafana](https://github.com/helm/charts/tree/master/stable/grafana) (deployed via an embedded Helm chart) +- Default PrometheusRules and Grafana dashboards based on the collection of community-curated resources from [kube-prometheus](https://github.com/prometheus-operator/kube-prometheus/) +- Default ServiceMonitors that watch the deployed Prometheus, Grafana, and Alertmanager + +Since this Project Monitoring Stack deploys Prometheus Operator CRs, an existing Prometheus Operator instance must already be deployed in the cluster for Prometheus Federator to successfully be able to deploy Project Monitoring Stacks. It is recommended to use [`rancher-monitoring`](https://rancher.com/docs/rancher/v2.6/en/monitoring-alerting/) for this. For more information on how the chart works or advanced configurations, please read the `README.md`. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + ​ +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. +​ +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. \ No newline at end of file diff --git a/charts/prometheus-federator/3.0.2+up0.3.3/charts/helmProjectOperator/Chart.yaml b/charts/prometheus-federator/3.0.2+up0.3.3/charts/helmProjectOperator/Chart.yaml new file mode 100644 index 0000000000..c4d14e1d93 --- /dev/null +++ b/charts/prometheus-federator/3.0.2+up0.3.3/charts/helmProjectOperator/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/certified: rancher + catalog.cattle.io/display-name: Helm Project Operator + catalog.cattle.io/kube-version: '>=1.16.0-0' + catalog.cattle.io/namespace: cattle-helm-system + catalog.cattle.io/os: linux,windows + catalog.cattle.io/permits-os: linux,windows + catalog.cattle.io/provides-gvr: helm.cattle.io.projecthelmchart/v1alpha1 + catalog.cattle.io/rancher-version: '>= 2.6.0-0' + catalog.cattle.io/release-name: helm-project-operator +apiVersion: v2 +appVersion: 0.2.1 +description: Helm Project Operator +name: helmProjectOperator +version: 0.2.1 diff --git a/charts/prometheus-federator/3.0.2+up0.3.3/charts/helmProjectOperator/README.md b/charts/prometheus-federator/3.0.2+up0.3.3/charts/helmProjectOperator/README.md new file mode 100644 index 0000000000..fc1d39e81b --- /dev/null +++ b/charts/prometheus-federator/3.0.2+up0.3.3/charts/helmProjectOperator/README.md @@ -0,0 +1,77 @@ +# Helm Project Operator + +## How does the operator work? + +1. On deploying a Helm Project Operator, users can create ProjectHelmCharts CRs with `spec.helmApiVersion` set to `dummy.cattle.io/v1alpha1` in a **Project Registration Namespace (`cattle-project-`)**. +2. On seeing each ProjectHelmChartCR, the operator will automatically deploy the embedded Helm chart on the Project Owner's behalf in the **Project Release Namespace (`cattle-project--dummy`)** based on a HelmChart CR and a HelmRelease CR automatically created by the ProjectHelmChart controller in the **Operator / System Namespace**. +3. RBAC will automatically be assigned in the Project Release Namespace to allow users to based on Role created in the Project Release Namespace with a given set of labels; this will be based on RBAC defined on the Project Registration Namespace against the [default Kubernetes user-facing roles](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles) (see below for more information about configuring RBAC). + +### What is a Project? + +In Helm Project Operator, a Project is a group of namespaces that can be identified by a `metav1.LabelSelector`; by default, the label used to identify projects is `field.cattle.io/projectId`, the label used to identify namespaces that are contained within a given [Rancher](https://rancher.com/) Project. + +### What is a ProjectHelmChart? + +A ProjectHelmChart is an instance of a (project-scoped) Helm chart deployed on behalf of a user who has permissions to create ProjectHelmChart resources in a Project Registration namespace. + +Generally, the best way to think about the ProjectHelmChart model is by comparing it to two other models: +1. Managed Kubernetes providers (EKS, GKE, AKS, etc.): in this model, a user has the ability to say "I want a Kubernetes cluster" but the underlying cloud provider is responsible for provisioning the infrastructure and offering **limited view and access** of the underlying resources created on their behalf; similarly, Helm Project Operator allows a Project Owner to say "I want this Helm chart deployed", but the underlying Operator is responsible for "provisioning" (deploying) the Helm chart and offering **limited view and access** of the underlying Kubernetes resources created on their behalf (based on configuring "least-privilege" Kubernetes RBAC for the Project Owners / Members in the newly created Project Release Namespace). +2. Dynamically-provisioned Persistent Volumes: in this model, a single resource (PersistentVolume) exists that allows you to specify a Storage Class that actually implements provisioning the underlying storage via a Storage Class Provisioner (e.g. Longhorn). Similarly, the ProjectHelmChart exists that allows you to specify a `spec.helmApiVersion` ("storage class") that actually implements deploying the underlying Helm chart via a Helm Project Operator (e.g. [`rancher/prometheus-federator`](https://github.com/rancher/prometheus-federator)). + +### Configuring the Helm release created by a ProjectHelmChart + +The `spec.values` of this ProjectHelmChart resources will correspond to the `values.yaml` override to be supplied to the underlying Helm chart deployed by the operator on the user's behalf; to see the underlying chart's `values.yaml` spec, either: +- View to the chart's definition located at [`rancher/helm-project-operator` under `charts/example-chart`](https://github.com/rancher/helm-project-operator/blob/main/charts/example-chart) (where the chart version will be tied to the version of this operator) +- Look for the ConfigMap named `dummy.cattle.io.v1alpha1` that is automatically created in each Project Registration Namespace, which will contain both the `values.yaml` and `questions.yaml` that was used to configure the chart (which was embedded directly into the `helm-project-operator` binary). + +### Namespaces + +All Helm Project Operators have three different classifications of namespaces that the operator looks out for: +1. **Operator / System Namespace**: this is the namespace that the operator is deployed into (e.g. `cattle-helm-system`). This namespace will contain all HelmCharts and HelmReleases for all ProjectHelmCharts watched by this operator. **Only Cluster Admins should have access to this namespace.** +2. **Project Registration Namespace (`cattle-project-`)**: this is the set of namespaces that the operator watches for ProjectHelmCharts within. The RoleBindings and ClusterRoleBindings that apply to this namespace will also be the source of truth for the auto-assigned RBAC created in the Project Release Namespace (see more details below). **Project Owners (admin), Project Members (edit), and Read-Only Members (view) should have access to this namespace**. +> Note: Project Registration Namespaces will be auto-generated by the operator and imported into the Project it is tied to if `.Values.global.cattle.projectLabel` is provided (which is set to `field.cattle.io/projectId` by default); this indicates that a Project Registration Namespace should be created by the operator if at least one namespace is observed with that label. The operator will not let these namespaces be deleted unless either all namespaces with that label are gone (e.g. this is the last namespace in that project, in which case the namespace will be marked with the label `"helm.cattle.io/helm-project-operator-orphaned": "true"`, which signals that it can be deleted) or it is no longer watching that project (because the project ID was provided under `.Values.helmProjectOperator.otherSystemProjectLabelValues`, which serves as a denylist for Projects). These namespaces will also never be auto-deleted to avoid destroying user data; it is recommended that users clean up these namespaces manually if desired on creating or deleting a project +> Note: if `.Values.global.cattle.projectLabel` is not provided, the Operator / System Namespace will also be the Project Registration Namespace +3. **Project Release Namespace (`cattle-project--dummy`)**: this is the set of namespaces that the operator deploys Helm charts within on behalf of a ProjectHelmChart; the operator will also automatically assign RBAC to Roles created in this namespace by the Helm charts based on bindings found in the Project Registration Namespace. **Only Cluster Admins should have access to this namespace; Project Owners (admin), Project Members (edit), and Read-Only Members (view) will be assigned limited access to this namespace by the deployed Helm Chart and Helm Project Operator.** +> Note: Project Release Namespaces are automatically deployed and imported into the project whose ID is specified under `.Values.helmProjectOperator.projectReleaseNamespaces.labelValue` (which defaults to the value of `.Values.global.cattle.systemProjectId` if not specified) whenever a ProjectHelmChart is specified in a Project Registration Namespace +> Note: Project Release Namespaces follow the same orphaning conventions as Project Registration Namespaces (see note above) +> Note: if `.Values.projectReleaseNamespaces.enabled` is false, the Project Release Namespace will be the same as the Project Registration Namespace + +### Helm Resources (HelmChart, HelmRelease) + +On deploying a ProjectHelmChart, the Helm Project Operator will automatically create and manage two child custom resources that manage the underlying Helm resources in turn: +- A HelmChart CR (managed via an embedded [k3s-io/helm-contoller](https://github.com/k3s-io/helm-controller) in the operator): this custom resource automatically creates a Job in the same namespace that triggers a `helm install`, `helm upgrade`, or `helm uninstall` depending on the change applied to the HelmChart CR; this CR is automatically updated on changes to the ProjectHelmChart (e.g. modifying the values.yaml) or changes to the underlying Project definition (e.g. adding or removing namespaces from a project). +> **Important Note: If a ProjectHelmChart is not deploying or updating the underlying Project Monitoring Stack for some reason, the Job created by this resource in the Operator / System namespace should be the first place you check to see if there's something wrong with the Helm operation; however, this is generally only accessible by a Cluster Admin.** +- A HelmRelease CR (managed via an embedded [rancher/helm-locker](https://github.com/rancher/helm-locker) in the operator): this custom resource automatically locks a deployed Helm release in place and automatically overwrites updates to underlying resources unless the change happens via a Helm operation (`helm install`, `helm upgrade`, or `helm uninstall` performed by the HelmChart CR). +> Note: HelmRelease CRs emit Kubernetes Events that detect when an underlying Helm release is being modified and locks it back to place; to view these events, you can use `kubectl describe helmrelease -n `; you can also view the logs on this operator to see when changes are detected and which resources were attempted to be modified + +Both of these resources are created for all Helm charts in the Operator / System namespaces to avoid escalation of privileges to underprivileged users. + +### RBAC + +As described in the section on namespaces above, Helm Project Operator expects that Project Owners, Project Members, and other users in the cluster with Project-level permissions (e.g. permissions in a certain set of namespaces identified by a single label selector) have minimal permissions in any namespaces except the Project Registration Namespace (which is imported into the project by default) and those that already comprise their projects. Therefore, in order to allow Project Owners to assign specific chart permissions to other users in their Project namespaces, the Helm Project Operator will automatically watch the following bindings: +- ClusterRoleBindings +- RoleBindings in the Project Release Namespace + +On observing a change to one of those types of bindings, the Helm Project Operator will check whether the `roleRef` that the the binding points to matches a ClusterRole with the name provided under `helmProjectOperator.releaseRoleBindings.clusterRoleRefs.admin`, `helmProjectOperator.releaseRoleBindings.clusterRoleRefs.edit`, or `helmProjectOperator.releaseRoleBindings.clusterRoleRefs.view`; by default, these roleRefs correspond will correspond to `admin`, `edit`, and `view` respectively, which are the [default Kubernetes user-facing roles](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles). + +> Note: for Rancher RBAC users, these [default Kubernetes user-facing roles](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles) directly correlate to the `Project Owner`, `Project Member`, and `Read-Only` default Project Role Templates. + +If the `roleRef` matches, the Helm Project Operator will filter the `subjects` of the binding for all Users and Groups and use that to automatically construct a RoleBinding for each Role in the Project Release Namespace with the same name as the role and the following labels: +- `helm.cattle.io/project-helm-chart-role: {{ .Release.Name }}` +- `helm.cattle.io/project-helm-chart-role-aggregate-from: ` + +By default, the `example-chart` (the underlying chart deployed by Helm Project Operator) does not create any default roles; however, if a Cluster Admin would like to assign additional permissions to certain users, they can either directly assign RoleBindings in the Project Release Namespace to certain users or created Roles with the above two labels on them to allow Project Owners to control assigning those RBAC roles to users in their Project Registration namespaces. + +### Advanced Helm Project Operator Configuration + +|Value|Configuration| +|---|---------------------------| +|`valuesOverride`| Allows an Operator to override values that are set on each ProjectHelmChart deployment on an operator-level; user-provided options (specified on the `spec.values` of the ProjectHelmChart) are automatically overridden if operator-level values are provided. For an exmaple, see how the default value overrides `federate.targets` (note: when overriding list values like `federate.targets`, user-provided list values will **not** be concatenated) | +|`projectReleaseNamespaces.labelValues`| The value of the Project that all Project Release Namespaces should be auto-imported into (via label and annotation). Not recommended to be overridden on a Rancher setup. | +|`otherSystemProjectLabelValues`| Other namespaces that the operator should treat as a system namespace that should not be monitored. By default, all namespaces that match `global.cattle.systemProjectId` will not be matched. `kube-system` is explicitly marked as a system namespace as well, regardless of label or annotation. | +|`releaseRoleBindings.aggregate`| Whether to automatically create RBAC resources in Project Release namespaces +|`releaseRoleBindings.clusterRoleRefs.`| ClusterRoles to reference to discover subjects to create RoleBindings for in the Project Release Namespace for all corresponding Project Release Roles. See RBAC above for more information | +|`hardenedNamespaces.enabled`| Whether to automatically patch the default ServiceAccount with `automountServiceAccountToken: false` and create a default NetworkPolicy in all managed namespaces in the cluster; the default values ensure that the creation of the namespace does not break a CIS 1.16 hardened scan | +|`hardenedNamespaces.configuration`| The configuration to be supplied to the default ServiceAccount or auto-generated NetworkPolicy on managing a namespace | +|`helmController.enabled`| Whether to enable an embedded k3s-io/helm-controller instance within the Helm Project Operator. Should be disabled for RKE2 clusters since RKE2 clusters already run Helm Controller to manage internal Kubernetes components | +|`helmLocker.enabled`| Whether to enable an embedded rancher/helm-locker instance within the Helm Project Operator. | diff --git a/charts/prometheus-federator/3.0.2+up0.3.3/charts/helmProjectOperator/app-readme.md b/charts/prometheus-federator/3.0.2+up0.3.3/charts/helmProjectOperator/app-readme.md new file mode 100644 index 0000000000..fd551467d3 --- /dev/null +++ b/charts/prometheus-federator/3.0.2+up0.3.3/charts/helmProjectOperator/app-readme.md @@ -0,0 +1,20 @@ +# Helm Project Operator + +This chart installs the example [Helm Project Operator](https://github.com/rancher/helm-project-operator) onto your cluster. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + ​ +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. +​ +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. \ No newline at end of file diff --git a/charts/prometheus-federator/3.0.2+up0.3.3/charts/helmProjectOperator/questions.yaml b/charts/prometheus-federator/3.0.2+up0.3.3/charts/helmProjectOperator/questions.yaml new file mode 100644 index 0000000000..054361a7a4 --- /dev/null +++ b/charts/prometheus-federator/3.0.2+up0.3.3/charts/helmProjectOperator/questions.yaml @@ -0,0 +1,43 @@ +questions: +- variable: global.cattle.psp.enabled + default: "false" + description: "Flag to enable or disable the installation of PodSecurityPolicies by this chart in the target cluster. If the cluster is running Kubernetes 1.25+, you must update this value to false." + label: "Enable PodSecurityPolicies" + type: boolean + group: "Security Settings" +- variable: helmController.enabled + label: Enable Embedded Helm Controller + description: 'Note: If you are running this chart in an RKE2 cluster, this should be disabled.' + type: boolean + group: Helm Controller +- variable: helmLocker.enabled + label: Enable Embedded Helm Locker + type: boolean + group: Helm Locker +- variable: projectReleaseNamespaces.labelValue + label: Project Release Namespace Project ID + description: By default, the System Project is selected. This can be overriden to a different Project (e.g. p-xxxxx) + type: string + required: false + group: Namespaces +- variable: releaseRoleBindings.clusterRoleRefs.admin + label: Admin ClusterRole + description: By default, admin selects Project Owners. This can be overridden to a different ClusterRole (e.g. rt-xxxxx) + type: string + default: admin + required: false + group: RBAC +- variable: releaseRoleBindings.clusterRoleRefs.edit + label: Edit ClusterRole + description: By default, edit selects Project Members. This can be overridden to a different ClusterRole (e.g. rt-xxxxx) + type: string + default: edit + required: false + group: RBAC +- variable: releaseRoleBindings.clusterRoleRefs.view + label: View ClusterRole + description: By default, view selects Read-Only users. This can be overridden to a different ClusterRole (e.g. rt-xxxxx) + type: string + default: view + required: false + group: RBAC diff --git a/charts/prometheus-federator/3.0.2+up0.3.3/charts/helmProjectOperator/templates/NOTES.txt b/charts/prometheus-federator/3.0.2+up0.3.3/charts/helmProjectOperator/templates/NOTES.txt new file mode 100644 index 0000000000..32baeebcbe --- /dev/null +++ b/charts/prometheus-federator/3.0.2+up0.3.3/charts/helmProjectOperator/templates/NOTES.txt @@ -0,0 +1,2 @@ +{{ $.Chart.Name }} has been installed. Check its status by running: + kubectl --namespace {{ template "helm-project-operator.namespace" . }} get pods -l "release={{ $.Release.Name }}" diff --git a/charts/prometheus-federator/3.0.2+up0.3.3/charts/helmProjectOperator/templates/_helpers.tpl b/charts/prometheus-federator/3.0.2+up0.3.3/charts/helmProjectOperator/templates/_helpers.tpl new file mode 100644 index 0000000000..97dd6b368b --- /dev/null +++ b/charts/prometheus-federator/3.0.2+up0.3.3/charts/helmProjectOperator/templates/_helpers.tpl @@ -0,0 +1,66 @@ +# Rancher +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# Helm Project Operator + +{{/* vim: set filetype=mustache: */}} +{{/* Expand the name of the chart. This is suffixed with -alertmanager, which means subtract 13 from longest 63 available */}} +{{- define "helm-project-operator.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 50 | trimSuffix "-" -}} +{{- end }} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "helm-project-operator.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* Create chart name and version as used by the chart label. */}} +{{- define "helm-project-operator.chartref" -}} +{{- replace "+" "_" .Chart.Version | printf "%s-%s" .Chart.Name -}} +{{- end }} + +{{/* Generate basic labels */}} +{{- define "helm-project-operator.labels" -}} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/version: "{{ replace "+" "_" .Chart.Version }}" +app.kubernetes.io/part-of: {{ template "helm-project-operator.name" . }} +chart: {{ template "helm-project-operator.chartref" . }} +release: {{ $.Release.Name | quote }} +heritage: {{ $.Release.Service | quote }} +{{- if .Values.commonLabels}} +{{ toYaml .Values.commonLabels }} +{{- end }} +{{- end -}} diff --git a/charts/prometheus-federator/3.0.2+up0.3.3/charts/helmProjectOperator/templates/cleanup.yaml b/charts/prometheus-federator/3.0.2+up0.3.3/charts/helmProjectOperator/templates/cleanup.yaml new file mode 100644 index 0000000000..98675642d0 --- /dev/null +++ b/charts/prometheus-federator/3.0.2+up0.3.3/charts/helmProjectOperator/templates/cleanup.yaml @@ -0,0 +1,82 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ template "helm-project-operator.name" . }}-cleanup + namespace: {{ template "helm-project-operator.namespace" . }} + labels: {{ include "helm-project-operator.labels" . | nindent 4 }} + app: {{ template "helm-project-operator.name" . }} + annotations: + "helm.sh/hook": pre-delete + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed +spec: + template: + metadata: + name: {{ template "helm-project-operator.name" . }}-cleanup + labels: {{ include "helm-project-operator.labels" . | nindent 8 }} + app: {{ template "helm-project-operator.name" . }} + spec: + serviceAccountName: {{ template "helm-project-operator.name" . }} +{{- if .Values.cleanup.securityContext }} + securityContext: {{ toYaml .Values.cleanup.securityContext | nindent 8 }} +{{- end }} + initContainers: + - name: add-cleanup-annotations + image: {{ template "system_default_registry" . }}{{ .Values.cleanup.image.repository }}:{{ .Values.cleanup.image.tag }} + imagePullPolicy: "{{ .Values.image.pullPolicy }}" + command: + - /bin/sh + - -c + - > + echo "Labeling all ProjectHelmCharts with helm.cattle.io/helm-project-operator-cleanup=true"; + EXPECTED_HELM_API_VERSION={{ .Values.helmApiVersion }}; + IFS=$'\n'; + for namespace in $(kubectl get namespaces -l helm.cattle.io/helm-project-operated=true --no-headers -o=custom-columns=NAME:.metadata.name); do + for projectHelmChartAndHelmApiVersion in $(kubectl get projecthelmcharts -n ${namespace} --no-headers -o=custom-columns=NAME:.metadata.name,HELMAPIVERSION:.spec.helmApiVersion); do + projectHelmChartAndHelmApiVersion=$(echo ${projectHelmChartAndHelmApiVersion} | xargs); + projectHelmChart=$(echo ${projectHelmChartAndHelmApiVersion} | cut -d' ' -f1); + helmApiVersion=$(echo ${projectHelmChartAndHelmApiVersion} | cut -d' ' -f2); + if [[ ${helmApiVersion} != ${EXPECTED_HELM_API_VERSION} ]]; then + echo "Skipping marking ${namespace}/${projectHelmChart} with cleanup annotation since spec.helmApiVersion: ${helmApiVersion} is not ${EXPECTED_HELM_API_VERSION}"; + continue; + fi; + kubectl label projecthelmcharts -n ${namespace} ${projectHelmChart} helm.cattle.io/helm-project-operator-cleanup=true --overwrite; + done; + done; +{{- if .Values.cleanup.resources }} + resources: {{ toYaml .Values.cleanup.resources | nindent 12 }} +{{- end }} +{{- if .Values.cleanup.containerSecurityContext }} + securityContext: {{ toYaml .Values.cleanup.containerSecurityContext | nindent 12 }} +{{- end }} + containers: + - name: ensure-subresources-deleted + image: {{ template "system_default_registry" . }}{{ .Values.cleanup.image.repository }}:{{ .Values.cleanup.image.tag }} + imagePullPolicy: IfNotPresent + command: + - /bin/sh + - -c + - > + SYSTEM_NAMESPACE={{ .Release.Namespace }} + EXPECTED_HELM_API_VERSION={{ .Values.helmApiVersion }}; + HELM_API_VERSION_TRUNCATED=$(echo ${EXPECTED_HELM_API_VERSION} | cut -d'/' -f0); + echo "Ensuring HelmCharts and HelmReleases are deleted from ${SYSTEM_NAMESPACE}..."; + while [[ "$(kubectl get helmcharts,helmreleases -l helm.cattle.io/helm-api-version=${HELM_API_VERSION_TRUNCATED} -n ${SYSTEM_NAMESPACE} 2>&1)" != "No resources found in ${SYSTEM_NAMESPACE} namespace." ]]; do + echo "waiting for HelmCharts and HelmReleases to be deleted from ${SYSTEM_NAMESPACE}... sleeping 3 seconds"; + sleep 3; + done; + echo "Successfully deleted all HelmCharts and HelmReleases in ${SYSTEM_NAMESPACE}!"; +{{- if .Values.cleanup.resources }} + resources: {{ toYaml .Values.cleanup.resources | nindent 12 }} +{{- end }} +{{- if .Values.cleanup.containerSecurityContext }} + securityContext: {{ toYaml .Values.cleanup.containerSecurityContext | nindent 12 }} +{{- end }} + restartPolicy: OnFailure + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} + {{- if .Values.cleanup.nodeSelector }} + {{- toYaml .Values.cleanup.nodeSelector | nindent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} + {{- if .Values.cleanup.tolerations }} + {{- toYaml .Values.cleanup.tolerations | nindent 8 }} + {{- end }} diff --git a/charts/prometheus-federator/3.0.2+up0.3.3/charts/helmProjectOperator/templates/clusterrole.yaml b/charts/prometheus-federator/3.0.2+up0.3.3/charts/helmProjectOperator/templates/clusterrole.yaml new file mode 100644 index 0000000000..60ed263ba8 --- /dev/null +++ b/charts/prometheus-federator/3.0.2+up0.3.3/charts/helmProjectOperator/templates/clusterrole.yaml @@ -0,0 +1,57 @@ +{{- if and .Values.global.rbac.create .Values.global.rbac.userRoles.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "helm-project-operator.name" . }}-admin + labels: {{ include "helm-project-operator.labels" . | nindent 4 }} + {{- if .Values.global.rbac.userRoles.aggregateToDefaultRoles }} + rbac.authorization.k8s.io/aggregate-to-admin: "true" + {{- end }} +rules: +- apiGroups: + - helm.cattle.io + resources: + - projecthelmcharts + - projecthelmcharts/finalizers + - projecthelmcharts/status + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "helm-project-operator.name" . }}-edit + labels: {{ include "helm-project-operator.labels" . | nindent 4 }} + {{- if .Values.global.rbac.userRoles.aggregateToDefaultRoles }} + rbac.authorization.k8s.io/aggregate-to-edit: "true" + {{- end }} +rules: +- apiGroups: + - helm.cattle.io + resources: + - projecthelmcharts + - projecthelmcharts/status + verbs: + - 'get' + - 'list' + - 'watch' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "helm-project-operator.name" . }}-view + labels: {{ include "helm-project-operator.labels" . | nindent 4 }} + {{- if .Values.global.rbac.userRoles.aggregateToDefaultRoles }} + rbac.authorization.k8s.io/aggregate-to-view: "true" + {{- end }} +rules: +- apiGroups: + - helm.cattle.io + resources: + - projecthelmcharts + - projecthelmcharts/status + verbs: + - 'get' + - 'list' + - 'watch' +{{- end }} diff --git a/charts/prometheus-federator/3.0.2+up0.3.3/charts/helmProjectOperator/templates/configmap.yaml b/charts/prometheus-federator/3.0.2+up0.3.3/charts/helmProjectOperator/templates/configmap.yaml new file mode 100644 index 0000000000..d4def157dc --- /dev/null +++ b/charts/prometheus-federator/3.0.2+up0.3.3/charts/helmProjectOperator/templates/configmap.yaml @@ -0,0 +1,14 @@ +## Note: If you add another entry to this ConfigMap, make sure a corresponding env var is set +## in the deployment of the operator to ensure that a Helm upgrade will force the operator +## to reload the values in the ConfigMap and redeploy +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "helm-project-operator.name" . }}-config + namespace: {{ template "helm-project-operator.namespace" . }} + labels: {{ include "helm-project-operator.labels" . | nindent 4 }} +data: + hardened.yaml: |- +{{ .Values.hardenedNamespaces.configuration | toYaml | indent 4 }} + values.yaml: |- +{{ .Values.valuesOverride | toYaml | indent 4 }} diff --git a/charts/prometheus-federator/3.0.2+up0.3.3/charts/helmProjectOperator/templates/deployment.yaml b/charts/prometheus-federator/3.0.2+up0.3.3/charts/helmProjectOperator/templates/deployment.yaml new file mode 100644 index 0000000000..33b81e72e6 --- /dev/null +++ b/charts/prometheus-federator/3.0.2+up0.3.3/charts/helmProjectOperator/templates/deployment.yaml @@ -0,0 +1,126 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "helm-project-operator.name" . }} + namespace: {{ template "helm-project-operator.namespace" . }} + labels: {{ include "helm-project-operator.labels" . | nindent 4 }} + app: {{ template "helm-project-operator.name" . }} +spec: + {{- if .Values.replicas }} + replicas: {{ .Values.replicas }} + {{- end }} + selector: + matchLabels: + app: {{ template "helm-project-operator.name" . }} + release: {{ $.Release.Name | quote }} + template: + metadata: + labels: {{ include "helm-project-operator.labels" . | nindent 8 }} + app: {{ template "helm-project-operator.name" . }} + spec: + containers: + - name: {{ template "helm-project-operator.name" . }} + image: "{{ template "system_default_registry" . }}{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: "{{ .Values.image.pullPolicy }}" + args: + - {{ template "helm-project-operator.name" . }} + - --namespace={{ template "helm-project-operator.namespace" . }} + - --controller-name={{ template "helm-project-operator.name" . }} + - --values-override-file=/etc/helmprojectoperator/config/values.yaml +{{- if .Values.global.cattle.systemDefaultRegistry }} + - --system-default-registry={{ .Values.global.cattle.systemDefaultRegistry }} +{{- end }} +{{- if .Values.global.cattle.url }} + - --cattle-url={{ .Values.global.cattle.url }} +{{- end }} +{{- if .Values.global.cattle.projectLabel }} + - --project-label={{ .Values.global.cattle.projectLabel }} +{{- end }} +{{- if not .Values.projectReleaseNamespaces.enabled }} + - --system-project-label-values={{ join "," (append .Values.otherSystemProjectLabelValues .Values.global.cattle.systemProjectId) }} +{{- else if and (ne (len .Values.global.cattle.systemProjectId) 0) (ne (len .Values.projectReleaseNamespaces.labelValue) 0) (ne .Values.projectReleaseNamespaces.labelValue .Values.global.cattle.systemProjectId) }} + - --system-project-label-values={{ join "," (append .Values.otherSystemProjectLabelValues .Values.global.cattle.systemProjectId) }} +{{- else if len .Values.otherSystemProjectLabelValues }} + - --system-project-label-values={{ join "," .Values.otherSystemProjectLabelValues }} +{{- end }} +{{- if .Values.projectReleaseNamespaces.enabled }} +{{- if .Values.projectReleaseNamespaces.labelValue }} + - --project-release-label-value={{ .Values.projectReleaseNamespaces.labelValue }} +{{- else if .Values.global.cattle.systemProjectId }} + - --project-release-label-value={{ .Values.global.cattle.systemProjectId }} +{{- end }} +{{- end }} +{{- if .Values.global.cattle.clusterId }} + - --cluster-id={{ .Values.global.cattle.clusterId }} +{{- end }} +{{- if .Values.releaseRoleBindings.aggregate }} +{{- if .Values.releaseRoleBindings.clusterRoleRefs }} +{{- if .Values.releaseRoleBindings.clusterRoleRefs.admin }} + - --admin-cluster-role={{ .Values.releaseRoleBindings.clusterRoleRefs.admin }} +{{- end }} +{{- if .Values.releaseRoleBindings.clusterRoleRefs.edit }} + - --edit-cluster-role={{ .Values.releaseRoleBindings.clusterRoleRefs.edit }} +{{- end }} +{{- if .Values.releaseRoleBindings.clusterRoleRefs.view }} + - --view-cluster-role={{ .Values.releaseRoleBindings.clusterRoleRefs.view }} +{{- end }} +{{- end }} +{{- end }} +{{- if .Values.hardenedNamespaces.enabled }} + - --hardening-options-file=/etc/helmprojectoperator/config/hardening.yaml +{{- else }} + - --disable-hardening +{{- end }} +{{- if .Values.debug }} + - --debug + - --debug-level={{ .Values.debugLevel }} +{{- end }} +{{- if not .Values.helmController.enabled }} + - --disable-embedded-helm-controller +{{- else }} + - --helm-job-image={{ template "system_default_registry" . }}{{ .Values.helmController.job.image.repository }}:{{ .Values.helmController.job.image.tag }} +{{- end }} +{{- if not .Values.helmLocker.enabled }} + - --disable-embedded-helm-locker +{{- end }} +{{- if .Values.additionalArgs }} +{{- toYaml .Values.additionalArgs | nindent 10 }} +{{- end }} + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + ## Note: The below two values only exist to force Helm to upgrade the deployment on + ## a change to the contents of the ConfigMap during an upgrade. Neither serve + ## any practical purpose and can be removed and replaced with a configmap reloader + ## in a future change if dynamic updates are required. + - name: HARDENING_OPTIONS_SHA_256_HASH + value: {{ .Values.hardenedNamespaces.configuration | toYaml | sha256sum }} + - name: VALUES_OVERRIDE_SHA_256_HASH + value: {{ .Values.valuesOverride | toYaml | sha256sum }} +{{- if .Values.resources }} + resources: {{ toYaml .Values.resources | nindent 12 }} +{{- end }} +{{- if .Values.containerSecurityContext }} + securityContext: {{ toYaml .Values.containerSecurityContext | nindent 12 }} +{{- end }} + volumeMounts: + - name: config + mountPath: "/etc/helmprojectoperator/config" + serviceAccountName: {{ template "helm-project-operator.name" . }} +{{- if .Values.securityContext }} + securityContext: {{ toYaml .Values.securityContext | nindent 8 }} +{{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.nodeSelector }} +{{- toYaml .Values.nodeSelector | nindent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.tolerations }} +{{- toYaml .Values.tolerations | nindent 8 }} +{{- end }} + volumes: + - name: config + configMap: + name: {{ template "helm-project-operator.name" . }}-config diff --git a/charts/prometheus-federator/3.0.2+up0.3.3/charts/helmProjectOperator/templates/psp.yaml b/charts/prometheus-federator/3.0.2+up0.3.3/charts/helmProjectOperator/templates/psp.yaml new file mode 100644 index 0000000000..73dcc4560e --- /dev/null +++ b/charts/prometheus-federator/3.0.2+up0.3.3/charts/helmProjectOperator/templates/psp.yaml @@ -0,0 +1,68 @@ +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "helm-project-operator.name" . }}-psp + namespace: {{ template "helm-project-operator.namespace" . }} + labels: {{ include "helm-project-operator.labels" . | nindent 4 }} + app: {{ template "helm-project-operator.name" . }} +{{- if .Values.global.rbac.pspAnnotations }} + annotations: {{ toYaml .Values.global.rbac.pspAnnotations | nindent 4 }} +{{- end }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + # Permits the container to run with root privileges as well. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 0 + max: 65535 + readOnlyRootFilesystem: false +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "helm-project-operator.name" . }}-psp + labels: {{ include "helm-project-operator.labels" . | nindent 4 }} + app: {{ template "helm-project-operator.name" . }} +rules: +{{- if semverCompare "> 1.15.0-0" .Capabilities.KubeVersion.GitVersion }} +- apiGroups: ['policy'] +{{- else }} +- apiGroups: ['extensions'] +{{- end }} + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "helm-project-operator.name" . }}-psp +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "helm-project-operator.name" . }}-psp + labels: {{ include "helm-project-operator.labels" . | nindent 4 }} + app: {{ template "helm-project-operator.name" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "helm-project-operator.name" . }}-psp +subjects: + - kind: ServiceAccount + name: {{ template "helm-project-operator.name" . }} + namespace: {{ template "helm-project-operator.namespace" . }} +{{- end }} diff --git a/charts/prometheus-federator/3.0.2+up0.3.3/charts/helmProjectOperator/templates/rbac.yaml b/charts/prometheus-federator/3.0.2+up0.3.3/charts/helmProjectOperator/templates/rbac.yaml new file mode 100644 index 0000000000..b1c4092029 --- /dev/null +++ b/charts/prometheus-federator/3.0.2+up0.3.3/charts/helmProjectOperator/templates/rbac.yaml @@ -0,0 +1,32 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "helm-project-operator.name" . }} + labels: {{ include "helm-project-operator.labels" . | nindent 4 }} + app: {{ template "helm-project-operator.name" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: "cluster-admin" # see note below +subjects: +- kind: ServiceAccount + name: {{ template "helm-project-operator.name" . }} + namespace: {{ template "helm-project-operator.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "helm-project-operator.name" . }} + namespace: {{ template "helm-project-operator.namespace" . }} + labels: {{ include "helm-project-operator.labels" . | nindent 4 }} + app: {{ template "helm-project-operator.name" . }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: {{ toYaml .Values.global.imagePullSecrets | nindent 2 }} +{{- end }} +# --- +# NOTE: +# As of now, due to the fact that the k3s-io/helm-controller can only deploy jobs that are cluster-bound to the cluster-admin +# ClusterRole, the only way for this operator to be able to perform that binding is if it is also bound to the cluster-admin ClusterRole. +# +# As a result, this ClusterRoleBinding will be left as a work-in-progress until changes are made in k3s-io/helm-controller to allow us to grant +# only scoped down permissions to the Job that is deployed. diff --git a/charts/prometheus-federator/3.0.2+up0.3.3/charts/helmProjectOperator/templates/system-namespaces-configmap.yaml b/charts/prometheus-federator/3.0.2+up0.3.3/charts/helmProjectOperator/templates/system-namespaces-configmap.yaml new file mode 100644 index 0000000000..f4c85254e8 --- /dev/null +++ b/charts/prometheus-federator/3.0.2+up0.3.3/charts/helmProjectOperator/templates/system-namespaces-configmap.yaml @@ -0,0 +1,62 @@ +{{- if .Values.systemNamespacesConfigMap.create }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "helm-project-operator.name" . }}-system-namespaces + namespace: {{ template "helm-project-operator.namespace" . }} + labels: {{ include "helm-project-operator.labels" . | nindent 4 }} +data: + system-namespaces.json: |- + { +{{- if .Values.projectReleaseNamespaces.enabled }} +{{- if .Values.projectReleaseNamespaces.labelValue }} + "projectReleaseLabelValue": {{ .Values.projectReleaseNamespaces.labelValue | quote }}, +{{- else if .Values.global.cattle.systemProjectId }} + "projectReleaseLabelValue": {{ .Values.global.cattle.systemProjectId | quote }}, +{{- else }} + "projectReleaseLabelValue": "", +{{- end }} +{{- else }} + "projectReleaseLabelValue": "", +{{- end }} +{{- if not .Values.projectReleaseNamespaces.enabled }} + "systemProjectLabelValues": {{ append .Values.otherSystemProjectLabelValues .Values.global.cattle.systemProjectId | toJson }} +{{- else if and (ne (len .Values.global.cattle.systemProjectId) 0) (ne (len .Values.projectReleaseNamespaces.labelValue) 0) (ne .Values.projectReleaseNamespaces.labelValue .Values.global.cattle.systemProjectId) }} + "systemProjectLabelValues": {{ append .Values.otherSystemProjectLabelValues .Values.global.cattle.systemProjectId | toJson }} +{{- else if len .Values.otherSystemProjectLabelValues }} + "systemProjectLabelValues": {{ .Values.otherSystemProjectLabelValues | toJson }} +{{- else }} + "systemProjectLabelValues": [] +{{- end }} + } +--- +{{- if (and .Values.systemNamespacesConfigMap.rbac.enabled .Values.systemNamespacesConfigMap.rbac.subjects) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ template "helm-project-operator.name" . }}-system-namespaces + namespace: {{ template "helm-project-operator.namespace" . }} + labels: {{ include "helm-project-operator.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + resourceNames: + - "{{ template "helm-project-operator.name" . }}-system-namespaces" + verbs: + - 'get' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ template "helm-project-operator.name" . }}-system-namespaces + namespace: {{ template "helm-project-operator.namespace" . }} + labels: {{ include "helm-project-operator.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ template "helm-project-operator.name" . }}-system-namespaces +subjects: {{ .Values.systemNamespacesConfigMap.rbac.subjects | toYaml | nindent 2 }} +{{- end }} +{{- end }} diff --git a/charts/prometheus-federator/3.0.2+up0.3.3/charts/helmProjectOperator/templates/validate-psp-install.yaml b/charts/prometheus-federator/3.0.2+up0.3.3/charts/helmProjectOperator/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/prometheus-federator/3.0.2+up0.3.3/charts/helmProjectOperator/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/prometheus-federator/3.0.2+up0.3.3/charts/helmProjectOperator/values.yaml b/charts/prometheus-federator/3.0.2+up0.3.3/charts/helmProjectOperator/values.yaml new file mode 100644 index 0000000000..f856361453 --- /dev/null +++ b/charts/prometheus-federator/3.0.2+up0.3.3/charts/helmProjectOperator/values.yaml @@ -0,0 +1,228 @@ +# Default values for helm-project-operator. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Helm Project Operator Configuration + +global: + cattle: + clusterId: "" + psp: + enabled: false + projectLabel: field.cattle.io/projectId + systemDefaultRegistry: "" + systemProjectId: "" + url: "" + rbac: + ## Create RBAC resources for ServiceAccounts and users + ## + create: true + + userRoles: + ## Create default user ClusterRoles to allow users to interact with ProjectHelmCharts + create: true + ## Aggregate default user ClusterRoles into default k8s ClusterRoles + aggregateToDefaultRoles: true + + pspAnnotations: {} + ## Specify pod annotations + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#apparmor + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#seccomp + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#sysctl + ## + # seccomp.security.alpha.kubernetes.io/allowedProfileNames: '*' + # seccomp.security.alpha.kubernetes.io/defaultProfileName: 'docker/default' + # apparmor.security.beta.kubernetes.io/defaultProfileName: 'runtime/default' + + ## Reference to one or more secrets to be used when pulling images + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + imagePullSecrets: [] + # - name: "image-pull-secret" + +helmApiVersion: dummy.cattle.io/v1alpha1 + +## valuesOverride overrides values that are set on each ProjectHelmChart deployment on an operator-level +## User-provided values will be overwritten based on the values provided here +valuesOverride: {} + +## projectReleaseNamespaces are auto-generated namespaces that are created to host Helm Releases +## managed by this operator on behalf of a ProjectHelmChart +projectReleaseNamespaces: + ## Enabled determines whether Project Release Namespaces should be created. If false, the underlying + ## Helm release will be deployed in the Project Registration Namespace + enabled: true + ## labelValue is the value of the Project that the projectReleaseNamespace should be created within + ## If empty, this will be set to the value of global.cattle.systemProjectId + ## If global.cattle.systemProjectId is also empty, project release namespaces will be disabled + labelValue: "" + +## otherSystemProjectLabelValues are project labels that identify namespaces as those that should be treated as system projects +## i.e. they will be entirely ignored by the operator +## By default, the global.cattle.systemProjectId will be in this list +otherSystemProjectLabelValues: [] + +## releaseRoleBindings configures RoleBindings automatically created by the Helm Project Operator +## in Project Release Namespaces where underlying Helm charts are deployed +releaseRoleBindings: + ## aggregate enables creating these RoleBindings off aggregating RoleBindings in the + ## Project Registration Namespace or ClusterRoleBindings that bind users to the ClusterRoles + ## specified under clusterRoleRefs + aggregate: true + + ## clusterRoleRefs are the ClusterRoles whose RoleBinding or ClusterRoleBindings should determine + ## the RoleBindings created in the Project Release Namespace + ## + ## By default, these are set to create RoleBindings based on the RoleBindings / ClusterRoleBindings + ## attached to the default K8s user-facing ClusterRoles of admin, edit, and view. + ## ref: https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles + ## + clusterRoleRefs: + admin: admin + edit: edit + view: view + +hardenedNamespaces: + # Whether to automatically manage the configuration of the default ServiceAccount and + # auto-create a NetworkPolicy for each namespace created by this operator + enabled: true + + configuration: + # Values to be applied to each default ServiceAccount created in a managed namespace + serviceAccountSpec: + secrets: [] + imagePullSecrets: [] + automountServiceAccountToken: false + # Values to be applied to each default generated NetworkPolicy created in a managed namespace + networkPolicySpec: + podSelector: {} + egress: [] + ingress: [] + policyTypes: ["Ingress", "Egress"] + +## systemNamespacesConfigMap is a ConfigMap created to allow users to see valid entries +## for registering a ProjectHelmChart for a given Project on the Rancher Dashboard UI. +## It does not need to be enabled for a non-Rancher use case. +systemNamespacesConfigMap: + ## Create indicates whether the system namespaces configmap should be created + ## This is a required value for integration with Rancher Dashboard + create: true + + ## RBAC provides options around the RBAC created to allow users to be able to view + ## the systemNamespacesConfigMap; if not specified, only users with the ability to + ## view ConfigMaps in the namespace where this chart is deployed will be able to + ## properly view the system namespaces on the Rancher Dashboard UI + rbac: + ## enabled indicates that we should deploy a RoleBinding and Role to view this ConfigMap + enabled: true + ## subjects are the subjects that should be bound to this default RoleBinding + ## By default, we allow anyone who is authenticated to the system to be able to view + ## this ConfigMap in the deployment namespace + subjects: + - kind: Group + name: system:authenticated + +nameOverride: "" + +namespaceOverride: "" + +replicas: 1 + +image: + repository: rancher/helm-project-operator + tag: v0.2.1 + pullPolicy: IfNotPresent + +helmController: + # Note: should be disabled for RKE2 clusters since they already run Helm Controller to manage internal Kubernetes components + enabled: true + + job: + image: + repository: rancher/klipper-helm + tag: v0.7.0-build20220315 + +helmLocker: + enabled: true + +# Additional arguments to be passed into the Helm Project Operator image +additionalArgs: [] + +## Define which Nodes the Pods are scheduled on. +## ref: https://kubernetes.io/docs/user-guide/node-selection/ +## +nodeSelector: {} + +## Tolerations for use with node taints +## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +## +tolerations: [] +# - key: "key" +# operator: "Equal" +# value: "value" +# effect: "NoSchedule" + +resources: {} + # limits: + # memory: 500Mi + # cpu: 1000m + # requests: + # memory: 100Mi + # cpu: 100m + +containerSecurityContext: {} + # allowPrivilegeEscalation: false + # capabilities: + # drop: + # - ALL + # privileged: false + # readOnlyRootFilesystem: true + +securityContext: {} + # runAsGroup: 1000 + # runAsUser: 1000 + # supplementalGroups: + # - 1000 + +debug: false +debugLevel: 0 + +cleanup: + image: + repository: rancher/shell + tag: v0.1.25 + pullPolicy: IfNotPresent + + ## Define which Nodes the Pods are scheduled on. + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + ## Tolerations for use with node taints + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + + containerSecurityContext: {} + # allowPrivilegeEscalation: false + # capabilities: + # drop: + # - ALL + # privileged: false + # readOnlyRootFilesystem: true + + securityContext: + runAsNonRoot: false + runAsUser: 0 + + resources: {} + # limits: + # memory: 500Mi + # cpu: 1000m + # requests: + # memory: 100Mi + # cpu: 100m diff --git a/charts/prometheus-federator/3.0.2+up0.3.3/questions.yaml b/charts/prometheus-federator/3.0.2+up0.3.3/questions.yaml new file mode 100644 index 0000000000..87cf1339f8 --- /dev/null +++ b/charts/prometheus-federator/3.0.2+up0.3.3/questions.yaml @@ -0,0 +1,43 @@ +questions: +- variable: global.cattle.psp.enabled + default: "false" + description: "Flag to enable or disable the installation of PodSecurityPolicies by this chart in the target cluster. If the cluster is running Kubernetes 1.25+, you must update this value to false." + label: "Enable PodSecurityPolicies" + type: boolean + group: "Security Settings" +- variable: helmProjectOperator.helmController.enabled + label: Enable Embedded Helm Controller + description: 'Note: If you are running Prometheus Federator in an RKE2 / K3s cluster before v1.23.14 / v1.24.8 / v1.25.4, this should be disabled.' + type: boolean + group: Helm Controller +- variable: helmProjectOperator.helmLocker.enabled + label: Enable Embedded Helm Locker + type: boolean + group: Helm Locker +- variable: helmProjectOperator.projectReleaseNamespaces.labelValue + label: Project Release Namespace Project ID + description: By default, the System Project is selected. This can be overriden to a different Project (e.g. p-xxxxx) + type: string + required: false + group: Namespaces +- variable: helmProjectOperator.releaseRoleBindings.clusterRoleRefs.admin + label: Admin ClusterRole + description: By default, admin selects Project Owners. This can be overridden to a different ClusterRole (e.g. rt-xxxxx) + type: string + default: admin + required: false + group: RBAC +- variable: helmProjectOperator.releaseRoleBindings.clusterRoleRefs.edit + label: Edit ClusterRole + description: By default, edit selects Project Members. This can be overridden to a different ClusterRole (e.g. rt-xxxxx) + type: string + default: edit + required: false + group: RBAC +- variable: helmProjectOperator.releaseRoleBindings.clusterRoleRefs.view + label: View ClusterRole + description: By default, view selects Read-Only users. This can be overridden to a different ClusterRole (e.g. rt-xxxxx) + type: string + default: view + required: false + group: RBAC \ No newline at end of file diff --git a/charts/prometheus-federator/3.0.2+up0.3.3/templates/NOTES.txt b/charts/prometheus-federator/3.0.2+up0.3.3/templates/NOTES.txt new file mode 100644 index 0000000000..f551f36613 --- /dev/null +++ b/charts/prometheus-federator/3.0.2+up0.3.3/templates/NOTES.txt @@ -0,0 +1,3 @@ +{{ $.Chart.Name }} has been installed. Check its status by running: + kubectl --namespace {{ template "prometheus-federator.namespace" . }} get pods -l "release={{ $.Release.Name }}" + diff --git a/charts/prometheus-federator/3.0.2+up0.3.3/templates/_helpers.tpl b/charts/prometheus-federator/3.0.2+up0.3.3/templates/_helpers.tpl new file mode 100644 index 0000000000..15ea4e5c88 --- /dev/null +++ b/charts/prometheus-federator/3.0.2+up0.3.3/templates/_helpers.tpl @@ -0,0 +1,66 @@ +# Rancher +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# Helm Project Operator + +{{/* vim: set filetype=mustache: */}} +{{/* Expand the name of the chart. This is suffixed with -alertmanager, which means subtract 13 from longest 63 available */}} +{{- define "prometheus-federator.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 50 | trimSuffix "-" -}} +{{- end }} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "prometheus-federator.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* Create chart name and version as used by the chart label. */}} +{{- define "prometheus-federator.chartref" -}} +{{- replace "+" "_" .Chart.Version | printf "%s-%s" .Chart.Name -}} +{{- end }} + +{{/* Generate basic labels */}} +{{- define "prometheus-federator.labels" }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/version: "{{ replace "+" "_" .Chart.Version }}" +app.kubernetes.io/part-of: {{ template "prometheus-federator.name" . }} +chart: {{ template "prometheus-federator.chartref" . }} +release: {{ $.Release.Name | quote }} +heritage: {{ $.Release.Service | quote }} +{{- if .Values.commonLabels}} +{{ toYaml .Values.commonLabels }} +{{- end }} +{{- end }} diff --git a/charts/prometheus-federator/3.0.2+up0.3.3/values.yaml b/charts/prometheus-federator/3.0.2+up0.3.3/values.yaml new file mode 100644 index 0000000000..7e81173aff --- /dev/null +++ b/charts/prometheus-federator/3.0.2+up0.3.3/values.yaml @@ -0,0 +1,94 @@ +# Default values for helm-project-operator. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Prometheus Federator Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + projectLabel: field.cattle.io/projectId + clusterId: "" + systemProjectId: "" + url: "" + rbac: + pspEnabled: true + pspAnnotations: {} + ## Specify pod annotations + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#apparmor + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#seccomp + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#sysctl + ## + # seccomp.security.alpha.kubernetes.io/allowedProfileNames: '*' + # seccomp.security.alpha.kubernetes.io/defaultProfileName: 'docker/default' + # apparmor.security.beta.kubernetes.io/defaultProfileName: 'runtime/default' + + ## Reference to one or more secrets to be used when pulling images + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + imagePullSecrets: [] + # - name: "image-pull-secret" + +helmProjectOperator: + enabled: true + + # ensures that all resources created by subchart show up as prometheus-federator + helmApiVersion: monitoring.cattle.io/v1alpha1 + + nameOverride: prometheus-federator + + helmController: + # Note: should be disabled for RKE2 clusters since they already run Helm Controller to manage internal Kubernetes components + enabled: true + + helmLocker: + enabled: true + + ## valuesOverride overrides values that are set on each Project Prometheus Stack Helm Chart deployment on an operator level + ## all values provided here will override any user-provided values automatically + valuesOverride: + + federate: + # Change this to point at all Prometheuses you want all your Project Prometheus Stacks to federate from + # By default, this matches the default deployment of Rancher Monitoring + targets: + - rancher-monitoring-prometheus.cattle-monitoring-system.svc:9090 + + image: + repository: rancher/prometheus-federator + tag: v0.3.2 + pullPolicy: IfNotPresent + + # Additional arguments to be passed into the Prometheus Federator image + additionalArgs: [] + + ## Define which Nodes the Pods are scheduled on. + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + ## Tolerations for use with node taints + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + + resources: {} + # limits: + # memory: 500Mi + # cpu: 1000m + # requests: + # memory: 100Mi + # cpu: 100m + + securityContext: {} + # allowPrivilegeEscalation: false + # readOnlyRootFilesystem: true + + debug: false + debugLevel: 0 \ No newline at end of file diff --git a/charts/rancher-alerting-drivers/102.1.2/Chart.yaml b/charts/rancher-alerting-drivers/102.1.2/Chart.yaml new file mode 100644 index 0000000000..ad66b7dcfb --- /dev/null +++ b/charts/rancher-alerting-drivers/102.1.2/Chart.yaml @@ -0,0 +1,29 @@ +annotations: + catalog.cattle.io/certified: rancher + catalog.cattle.io/display-name: Alerting Drivers + catalog.cattle.io/kube-version: '>= 1.16.0-0 < 1.28.0-0' + catalog.cattle.io/os: linux + catalog.cattle.io/permits-os: linux,windows + catalog.cattle.io/rancher-version: '>= 2.7.0-0 < 2.8.0-0' + catalog.cattle.io/release-name: rancher-alerting-drivers + catalog.cattle.io/type: cluster-tool + catalog.cattle.io/upstream-version: 100.0.1 +apiVersion: v2 +appVersion: 1.16.0 +dependencies: +- condition: prom2teams.enabled + name: prom2teams + repository: file://./charts/prom2teams + version: 0.2.0 +- condition: sachet.enabled + name: sachet + repository: file://./charts/sachet + version: 1.0.1 +description: The manager for third-party webhook receivers used in Prometheus Alertmanager +icon: https://charts.rancher.io/assets/logos/alerting-drivers.svg +keywords: +- monitoring +- alertmanger +- webhook +name: rancher-alerting-drivers +version: 102.1.2 diff --git a/charts/rancher-alerting-drivers/102.1.2/README.md b/charts/rancher-alerting-drivers/102.1.2/README.md new file mode 100644 index 0000000000..ea3f118015 --- /dev/null +++ b/charts/rancher-alerting-drivers/102.1.2/README.md @@ -0,0 +1,11 @@ +# Rancher Alerting Drivers + +This chart installs one or more [Alertmanager Webhook Receiver Integrations](https://prometheus.io/docs/operating/integrations/#alertmanager-webhook-receiver) (i.e. Drivers). + +Those Drivers can be targeted by an existing deployment of Alertmanager to send alerts to notification mechanisms that are not natively supported. + +Currently, this chart supports the following Drivers: +- Microsoft Teams, based on [prom2teams](https://github.com/idealista/prom2teams) +- SMS, based on [Sachet](https://github.com/messagebird/sachet) + +After installing rancher-alerting-drivers, please refer to the upstream documentation for each Driver for configuration options. \ No newline at end of file diff --git a/charts/rancher-alerting-drivers/102.1.2/app-readme.md b/charts/rancher-alerting-drivers/102.1.2/app-readme.md new file mode 100644 index 0000000000..fe228d96f7 --- /dev/null +++ b/charts/rancher-alerting-drivers/102.1.2/app-readme.md @@ -0,0 +1,29 @@ +# Rancher Alerting Drivers + +This chart installs one or more [Alertmanager Webhook Receiver Integrations](https://prometheus.io/docs/operating/integrations/#alertmanager-webhook-receiver) (i.e. Drivers). + +Those Drivers can be targeted by an existing deployment of Alertmanager to send alerts to notification mechanisms that are not natively supported. + +Currently, this chart supports the following Drivers: +- Microsoft Teams, based on [prom2teams](https://github.com/idealista/prom2teams) +- SMS, based on [Sachet](https://github.com/messagebird/sachet) + +After installing rancher-alerting-drivers, please refer to the upstream documentation for each Driver for configuration options. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + ​ +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. +​ +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. \ No newline at end of file diff --git a/charts/rancher-alerting-drivers/102.1.2/charts/prom2teams/.helmignore b/charts/rancher-alerting-drivers/102.1.2/charts/prom2teams/.helmignore new file mode 100644 index 0000000000..50af031725 --- /dev/null +++ b/charts/rancher-alerting-drivers/102.1.2/charts/prom2teams/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-alerting-drivers/102.1.2/charts/prom2teams/Chart.yaml b/charts/rancher-alerting-drivers/102.1.2/charts/prom2teams/Chart.yaml new file mode 100644 index 0000000000..aeae0df709 --- /dev/null +++ b/charts/rancher-alerting-drivers/102.1.2/charts/prom2teams/Chart.yaml @@ -0,0 +1,10 @@ +annotations: + catalog.cattle.io/certified: rancher + catalog.cattle.io/hidden: "true" + catalog.cattle.io/os: linux + catalog.cattle.io/release-name: rancher-prom2teams +apiVersion: v1 +appVersion: 4.2.1 +description: A Helm chart for Prom2Teams based on the upstream https://github.com/idealista/prom2teams +name: prom2teams +version: 0.2.0 diff --git a/charts/rancher-alerting-drivers/102.1.2/charts/prom2teams/files/teams.j2 b/charts/rancher-alerting-drivers/102.1.2/charts/prom2teams/files/teams.j2 new file mode 100644 index 0000000000..f1cf61d4ef --- /dev/null +++ b/charts/rancher-alerting-drivers/102.1.2/charts/prom2teams/files/teams.j2 @@ -0,0 +1,44 @@ +{%- set + theme_colors = { + 'resolved' : '2DC72D', + 'critical' : '8C1A1A', + 'severe' : '8C1A1A', + 'warning' : 'FF9A0B', + 'unknown' : 'CCCCCC' + } +-%} + +{ + "@type": "MessageCard", + "@context": "http://schema.org/extensions", + "themeColor": "{% if status=='resolved' %} {{ theme_colors.resolved }} {% else %} {{ theme_colors[msg_text.severity] }} {% endif %}", + "summary": "{% if status=='resolved' %}(Resolved) {% endif %}{{ msg_text.summary }}", + "title": "Prometheus alert {% if status=='resolved' %}(Resolved) {% elif status=='unknown' %} (status unknown) {% endif %}", + "sections": [{ + "activityTitle": "{{ msg_text.summary }}", + "facts": [{% if msg_text.name %}{ + "name": "Alert", + "value": "{{ msg_text.name }}" + },{% endif %}{% if msg_text.instance %}{ + "name": "In host", + "value": "{{ msg_text.instance }}" + },{% endif %}{% if msg_text.severity %}{ + "name": "Severity", + "value": "{{ msg_text.severity }}" + },{% endif %}{% if msg_text.description %}{ + "name": "Description", + "value": "{{ msg_text.description }}" + },{% endif %}{ + "name": "Status", + "value": "{{ msg_text.status }}" + }{% if msg_text.extra_labels %}{% for key in msg_text.extra_labels %},{ + "name": "{{ key }}", + "value": "{{ msg_text.extra_labels[key] }}" + }{% endfor %}{% endif %} + {% if msg_text.extra_annotations %}{% for key in msg_text.extra_annotations %},{ + "name": "{{ key }}", + "value": "{{ msg_text.extra_annotations[key] }}" + }{% endfor %}{% endif %}], + "markdown": true + }] +} diff --git a/charts/rancher-alerting-drivers/102.1.2/charts/prom2teams/templates/NOTES.txt b/charts/rancher-alerting-drivers/102.1.2/charts/prom2teams/templates/NOTES.txt new file mode 100644 index 0000000000..a94c4132b6 --- /dev/null +++ b/charts/rancher-alerting-drivers/102.1.2/charts/prom2teams/templates/NOTES.txt @@ -0,0 +1,2 @@ +Prom2Teams has been installed. Check its status by running: + kubectl --namespace {{ .Release.Namespace }} get pods -l "app.kubernetes.io/instance={{ .Release.Name }}" diff --git a/charts/rancher-alerting-drivers/102.1.2/charts/prom2teams/templates/_helpers.tpl b/charts/rancher-alerting-drivers/102.1.2/charts/prom2teams/templates/_helpers.tpl new file mode 100644 index 0000000000..ffc0fa3567 --- /dev/null +++ b/charts/rancher-alerting-drivers/102.1.2/charts/prom2teams/templates/_helpers.tpl @@ -0,0 +1,73 @@ +{{/* vim: set filetype=mustache: */}} + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +{{/* +Expand the name of the chart. +*/}} +{{- define "prom2teams.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "prom2teams.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "prom2teams.namespace" -}} +{{ default .Release.Namespace .Values.global.namespaceOverride }} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "prom2teams.labels" -}} +app.kubernetes.io/name: {{ include "prom2teams.name" . }} +helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +app.kubernetes.io/instance: {{ .Release.Name }} +release: {{ .Release.Name }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} diff --git a/charts/rancher-alerting-drivers/102.1.2/charts/prom2teams/templates/configmap.yaml b/charts/rancher-alerting-drivers/102.1.2/charts/prom2teams/templates/configmap.yaml new file mode 100644 index 0000000000..ccf38953e2 --- /dev/null +++ b/charts/rancher-alerting-drivers/102.1.2/charts/prom2teams/templates/configmap.yaml @@ -0,0 +1,39 @@ +{{- $valid := list "DEBUG" "INFO" "WARNING" "ERROR" "CRITICAL" -}} +{{- if not (has .Values.prom2teams.loglevel $valid) -}} +{{- fail "Invalid log level"}} +{{- end -}} +{{- if and .Values.prom2teams.connector (hasKey .Values.prom2teams.connectors "Connector") -}} +{{- fail "Invalid configuration: prom2teams.connectors can't have a connector named Connector when prom2teams.connector is set"}} +{{- end -}} +{{/* Create the configmap when the operation is helm install and the target configmap does not exist. */}} +{{- if not (lookup "v1" "ConfigMap" (include "prom2teams.namespace" . ) (include "prom2teams.fullname" .)) }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ include "prom2teams.namespace" . }} + name: {{ include "prom2teams.fullname" . }} + labels: {{ include "prom2teams.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": pre-install, pre-upgrade + "helm.sh/hook-weight": "3" + "helm.sh/resource-policy": keep +data: + config.ini: |- + [HTTP Server] + Host: {{ .Values.prom2teams.host }} + Port: {{ .Values.prom2teams.port }} + [Microsoft Teams] + {{- with .Values.prom2teams.connector }} + Connector: {{ . }} + {{- end }} + {{- range $key, $val := .Values.prom2teams.connectors }} + {{ $key }}: {{ $val }} + {{- end }} + [Group Alerts] + Field: {{ .Values.prom2teams.group_alerts_by }} + [Log] + Level: {{ .Values.prom2teams.loglevel }} + [Template] + Path: {{ .Values.prom2teams.templatepath }} + teams.j2: {{ .Files.Get "files/teams.j2" | quote }} + {{- end -}} diff --git a/charts/rancher-alerting-drivers/102.1.2/charts/prom2teams/templates/deployment.yaml b/charts/rancher-alerting-drivers/102.1.2/charts/prom2teams/templates/deployment.yaml new file mode 100644 index 0000000000..34f7d0f465 --- /dev/null +++ b/charts/rancher-alerting-drivers/102.1.2/charts/prom2teams/templates/deployment.yaml @@ -0,0 +1,83 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "prom2teams.fullname" . }} + namespace: {{ include "prom2teams.namespace" . }} + labels: {{ include "prom2teams.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + app.kubernetes.io/name: {{ include "prom2teams.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + template: + metadata: + labels: + app.kubernetes.io/name: {{ include "prom2teams.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + spec: + serviceAccountName: {{ include "prom2teams.fullname" . }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: {{ toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: config + configMap: + name: {{ include "prom2teams.fullname" . }} + containers: + - name: {{ .Chart.Name }} + image: {{ include "system_default_registry" . }}{{ .Values.image.repository }}:{{ .Values.image.tag }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: http + containerPort: 8089 + protocol: TCP + volumeMounts: + - name: config + mountPath: /opt/prom2teams/helmconfig/ + env: + - name: APP_CONFIG_FILE + value: {{ .Values.prom2teams.config | quote }} + - name: PROM2TEAMS_PORT + value: {{ .Values.prom2teams.port | quote }} + - name: PROM2TEAMS_HOST + value: {{ .Values.prom2teams.host | quote }} + - name: PROM2TEAMS_CONNECTOR + value: {{ .Values.prom2teams.connector | quote }} + - name: PROM2TEAMS_GROUP_ALERTS_BY + value: {{ .Values.prom2teams.group_alerts_by | quote }} + - name: PROM2TEAMS_LOGLEVEL + value: {{ .Values.prom2teams.loglevel }} + {{- range $key, $value := .Values.prom2teams.extraEnv }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + resources: {{ toYaml .Values.resources | nindent 12 }} + {{- if .Values.securityContext.enabled }} + securityContext: + privileged: false + readOnlyRootFilesystem: false + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} + {{- if .Values.nodeSelector }} + {{- toYaml .Values.nodeSelector | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: {{ toYaml . | nindent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} + {{- if .Values.tolerations }} + {{- toYaml .Values.tolerations | nindent 8 }} + {{- end }} + {{- if .Values.securityContext.enabled }} + securityContext: + runAsNonRoot: {{ if eq (int .Values.securityContext.runAsUser) 0 }}false{{ else }}true{{ end }} + runAsUser: {{ .Values.securityContext.runAsUser }} + runAsGroup: {{ .Values.securityContext.runAsGroup }} + fsGroup: {{ .Values.securityContext.fsGroup }} + {{- end }} + diff --git a/charts/rancher-alerting-drivers/102.1.2/charts/prom2teams/templates/psp.yaml b/charts/rancher-alerting-drivers/102.1.2/charts/prom2teams/templates/psp.yaml new file mode 100644 index 0000000000..3e49a6c5d4 --- /dev/null +++ b/charts/rancher-alerting-drivers/102.1.2/charts/prom2teams/templates/psp.yaml @@ -0,0 +1,61 @@ +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ include "prom2teams.fullname" . }}-psp-{{ include "prom2teams.namespace" . }} + labels: {{ include "prom2teams.labels" . | nindent 4 }} +spec: + privileged: false + allowPrivilegeEscalation: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'configMap' + - 'secret' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "prom2teams.fullname" . }}-psp + namespace: {{ include "prom2teams.namespace" . }} + labels: {{ include "prom2teams.labels" . | nindent 4 }} +rules: + - apiGroups: + - policy + resourceNames: + - {{ include "prom2teams.fullname" . }}-psp-{{ include "prom2teams.namespace" . }} + resources: + - podsecuritypolicies + verbs: + - use +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "prom2teams.fullname" . }}-psp + namespace: {{ include "prom2teams.namespace" . }} + labels: {{ include "prom2teams.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "prom2teams.fullname" . }}-psp +subjects: + - kind: ServiceAccount + name: {{ include "prom2teams.fullname" . }} +{{- end }} diff --git a/charts/rancher-alerting-drivers/102.1.2/charts/prom2teams/templates/service-account.yaml b/charts/rancher-alerting-drivers/102.1.2/charts/prom2teams/templates/service-account.yaml new file mode 100644 index 0000000000..a9572c5cd9 --- /dev/null +++ b/charts/rancher-alerting-drivers/102.1.2/charts/prom2teams/templates/service-account.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "prom2teams.fullname" . }} + namespace: {{ include "prom2teams.namespace" . }} + labels: {{ include "prom2teams.labels" . | nindent 4 }} diff --git a/charts/rancher-alerting-drivers/102.1.2/charts/prom2teams/templates/service.yaml b/charts/rancher-alerting-drivers/102.1.2/charts/prom2teams/templates/service.yaml new file mode 100644 index 0000000000..cc95cad355 --- /dev/null +++ b/charts/rancher-alerting-drivers/102.1.2/charts/prom2teams/templates/service.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "prom2teams.fullname" . }} + namespace: {{ include "prom2teams.namespace" . }} + labels: +{{ include "prom2teams.labels" . | indent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: 8089 + protocol: TCP + name: http + selector: + app.kubernetes.io/name: {{ include "prom2teams.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} diff --git a/charts/rancher-alerting-drivers/102.1.2/charts/prom2teams/values.yaml b/charts/rancher-alerting-drivers/102.1.2/charts/prom2teams/values.yaml new file mode 100644 index 0000000000..e53d361eea --- /dev/null +++ b/charts/rancher-alerting-drivers/102.1.2/charts/prom2teams/values.yaml @@ -0,0 +1,69 @@ +# Default values for prom2teams. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + namespaceOverride: "" + +nameOverride: "prom2teams" +fullnameOverride: "" + +replicaCount: 1 + +image: + repository: rancher/mirrored-idealista-prom2teams + tag: 4.2.1 + pullPolicy: IfNotPresent + +resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 200m + memory: 200Mi + +service: + type: ClusterIP + port: 8089 + +prom2teams: + host: 0.0.0.0 + port: 8089 + connector: the-connector-url + connectors: {} + # group_alerts_by can be one of + # ("name" | "description" | "instance" | "severity" | "status" | "summary" | "fingerprint" | "runbook_url") + group_alerts_by: + # loglevel can be one of (DEBUG | INFO | WARNING | ERROR | CRITICAL) + loglevel: INFO + templatepath: /opt/prom2teams/helmconfig/teams.j2 + config: /opt/prom2teams/helmconfig/config.ini + extraEnv: {} + +# Security Context properties +securityContext: + # enabled is a flag to enable Security Context + enabled: true + # runAsUser is the user ID used to run the container + runAsUser: 101 + # runAsGroup is the primary group ID used to run all processes within any container of the pod + runAsGroup: 101 + # fsGroup is the group ID associated with the container + fsGroup: 101 + # readOnlyRootFilesystem is a flag to enable readOnlyRootFilesystem for the Hazelcast security context + readOnlyRootFilesystem: true + +## Node labels for pod assignment +## Ref: https://kubernetes.io/docs/user-guide/node-selection/ +## +nodeSelector: {} + +## List of node taints to tolerate (requires Kubernetes >= 1.6) +tolerations: [] + +affinity: {} diff --git a/charts/rancher-alerting-drivers/102.1.2/charts/sachet/.helmignore b/charts/rancher-alerting-drivers/102.1.2/charts/sachet/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-alerting-drivers/102.1.2/charts/sachet/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-alerting-drivers/102.1.2/charts/sachet/Chart.yaml b/charts/rancher-alerting-drivers/102.1.2/charts/sachet/Chart.yaml new file mode 100644 index 0000000000..dd0d706a60 --- /dev/null +++ b/charts/rancher-alerting-drivers/102.1.2/charts/sachet/Chart.yaml @@ -0,0 +1,11 @@ +annotations: + catalog.cattle.io/certified: rancher + catalog.cattle.io/hidden: "true" + catalog.cattle.io/os: linux + catalog.cattle.io/release-name: rancher-sachet +apiVersion: v2 +appVersion: 0.3.1 +description: A Helm chart for Sachet based on the upstream https://github.com/messagebird/sachet +name: sachet +type: application +version: 1.0.1 diff --git a/charts/rancher-alerting-drivers/102.1.2/charts/sachet/files/template.tmpl b/charts/rancher-alerting-drivers/102.1.2/charts/sachet/files/template.tmpl new file mode 100644 index 0000000000..08f24e1387 --- /dev/null +++ b/charts/rancher-alerting-drivers/102.1.2/charts/sachet/files/template.tmpl @@ -0,0 +1 @@ +# reference: https://github.com/messagebird/sachet/blob/master/examples/telegram.tmpl diff --git a/charts/rancher-alerting-drivers/102.1.2/charts/sachet/templates/NOTES.txt b/charts/rancher-alerting-drivers/102.1.2/charts/sachet/templates/NOTES.txt new file mode 100644 index 0000000000..247a91fc13 --- /dev/null +++ b/charts/rancher-alerting-drivers/102.1.2/charts/sachet/templates/NOTES.txt @@ -0,0 +1,3 @@ +rancher-sachet is now installed on the cluster! +Please refer to the upstream documentation for configuration options: +https://github.com/messagebird/sachet diff --git a/charts/rancher-alerting-drivers/102.1.2/charts/sachet/templates/_helpers.tpl b/charts/rancher-alerting-drivers/102.1.2/charts/sachet/templates/_helpers.tpl new file mode 100644 index 0000000000..eaa61fee50 --- /dev/null +++ b/charts/rancher-alerting-drivers/102.1.2/charts/sachet/templates/_helpers.tpl @@ -0,0 +1,79 @@ +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "sachet.namespace" -}} +{{ default .Release.Namespace .Values.global.namespaceOverride }} +{{- end }} + +{{/* +Expand the name of the chart. +*/}} +{{- define "sachet.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "sachet.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "sachet.labels" -}} +helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{ include "sachet.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "sachet.selectorLabels" -}} +app.kubernetes.io/name: {{ include "sachet.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + + diff --git a/charts/rancher-alerting-drivers/102.1.2/charts/sachet/templates/configmap-pre-install.yaml b/charts/rancher-alerting-drivers/102.1.2/charts/sachet/templates/configmap-pre-install.yaml new file mode 100644 index 0000000000..e8c63ac039 --- /dev/null +++ b/charts/rancher-alerting-drivers/102.1.2/charts/sachet/templates/configmap-pre-install.yaml @@ -0,0 +1,34 @@ +{{/*This file is applied when the operation is helm install and the target confimap does not exist. */}} +{{- if not (lookup "v1" "ConfigMap" (include "sachet.namespace" . ) (include "sachet.fullname" .)) }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ include "sachet.namespace" . }} + name: {{ include "sachet.fullname" . }} + labels: {{ include "sachet.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": pre-install, pre-upgrade + "helm.sh/hook-weight": "3" + "helm.sh/resource-policy": keep +data: + config.yaml: |- + {{- if and (not .Values.sachet.providers) (not .Values.sachet.receivers) }} + # please refer to the upstream documentation for configuration options: + # https://github.com/messagebird/sachet + # + # providers: + # aliyun: + # region_id: + # ... + # receivers: + # - name: 'team-sms' + # provider: 'aliyu' + # ... + {{- end }} + {{- with .Values.sachet.providers }} + providers: {{ toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sachet.receivers }} + receivers: {{ toYaml . | nindent 6 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-alerting-drivers/102.1.2/charts/sachet/templates/deployment.yaml b/charts/rancher-alerting-drivers/102.1.2/charts/sachet/templates/deployment.yaml new file mode 100644 index 0000000000..17215eebd0 --- /dev/null +++ b/charts/rancher-alerting-drivers/102.1.2/charts/sachet/templates/deployment.yaml @@ -0,0 +1,75 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "sachet.fullname" . }} + namespace: {{ include "sachet.namespace" . }} + labels: {{ include "sachet.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: {{ include "sachet.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: {{ toYaml . | nindent 8 }} + {{- end }} + labels: {{ include "sachet.selectorLabels" . | nindent 8 }} + spec: + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} + {{- if .Values.nodeSelector }} + {{- toYaml .Values.nodeSelector | nindent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} + {{- if .Values.tolerations }} + {{- toYaml .Values.tolerations | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: {{ toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: {{ toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "sachet.fullname" . }} + {{- with .Values.podSecurityContext }} + securityContext: {{ toYaml .Values.podSecurityContext | nindent 8 }} + {{- end }} + containers: + - name: {{ .Chart.Name }} + securityContext: {{ toYaml .Values.securityContext | nindent 12 }} + image: {{ include "system_default_registry" . }}{{ .Values.image.repository }}:{{ .Values.image.tag }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: http + containerPort: 9876 + protocol: TCP + livenessProbe: + httpGet: + path: /-/live + port: http + readinessProbe: + httpGet: + path: /-/ready + port: http + volumeMounts: + - mountPath: /etc/sachet/ + name: config-volume + {{- with .Values.resources }} + resources: {{ toYaml .Values.resources | nindent 12 }} + {{- end }} + - name: config-reloader + securityContext: {{ toYaml .Values.securityContext | nindent 12 }} + image: {{ include "system_default_registry" . }}{{ .Values.configReloader.repository }}:{{ .Values.configReloader.tag }} + imagePullPolicy: {{ .Values.configReloader.pullPolicy }} + args: + - -volume-dir=/watch-config + - -webhook-method=POST + - -webhook-status-code=200 + - -webhook-url=http://127.0.0.1:{{ .Values.service.port }}/-/reload + volumeMounts: + - mountPath: /watch-config + name: config-volume + volumes: + - name: config-volume + configMap: + name: {{ include "sachet.fullname" . }} + defaultMode: 0777 diff --git a/charts/rancher-alerting-drivers/102.1.2/charts/sachet/templates/psp.yaml b/charts/rancher-alerting-drivers/102.1.2/charts/sachet/templates/psp.yaml new file mode 100644 index 0000000000..16ec9ba8e7 --- /dev/null +++ b/charts/rancher-alerting-drivers/102.1.2/charts/sachet/templates/psp.yaml @@ -0,0 +1,61 @@ +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ include "sachet.fullname" . }}-psp-{{ include "sachet.namespace" . }} + labels: {{ include "sachet.labels" . | nindent 4 }} +spec: + privileged: false + allowPrivilegeEscalation: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'configMap' + - 'secret' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "sachet.fullname" . }}-psp + namespace: {{ include "sachet.namespace" . }} + labels: {{ include "sachet.labels" . | nindent 4 }} +rules: + - apiGroups: + - policy + resourceNames: + - {{ include "sachet.fullname" . }}-psp-{{ include "sachet.namespace" . }} + resources: + - podsecuritypolicies + verbs: + - use +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "sachet.fullname" . }}-psp + namespace: {{ include "sachet.namespace" . }} + labels: {{ include "sachet.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "sachet.fullname" . }}-psp +subjects: + - kind: ServiceAccount + name: {{ include "sachet.fullname" . }} +{{- end }} diff --git a/charts/rancher-alerting-drivers/102.1.2/charts/sachet/templates/service-account.yaml b/charts/rancher-alerting-drivers/102.1.2/charts/sachet/templates/service-account.yaml new file mode 100644 index 0000000000..8833f1b3b2 --- /dev/null +++ b/charts/rancher-alerting-drivers/102.1.2/charts/sachet/templates/service-account.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "sachet.fullname" . }} + namespace: {{ include "sachet.namespace" . }} + labels: {{ include "sachet.labels" . | nindent 4 }} diff --git a/charts/rancher-alerting-drivers/102.1.2/charts/sachet/templates/service.yaml b/charts/rancher-alerting-drivers/102.1.2/charts/sachet/templates/service.yaml new file mode 100644 index 0000000000..216e8322ca --- /dev/null +++ b/charts/rancher-alerting-drivers/102.1.2/charts/sachet/templates/service.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "sachet.fullname" . }} + namespace: {{ include "sachet.namespace" . }} + labels: {{ include "sachet.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + {{- if contains "NodePort" .Values.service.type }} + nodePort: {{ .Values.service.nodePort }} + {{- end }} + selector: {{ include "sachet.selectorLabels" . | nindent 4 }} diff --git a/charts/rancher-alerting-drivers/102.1.2/charts/sachet/values.yaml b/charts/rancher-alerting-drivers/102.1.2/charts/sachet/values.yaml new file mode 100644 index 0000000000..c9180b1430 --- /dev/null +++ b/charts/rancher-alerting-drivers/102.1.2/charts/sachet/values.yaml @@ -0,0 +1,69 @@ +# Default values for sachet. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + namespaceOverride: "" + +nameOverride: "sachet" +fullnameOverride: "" + +configReloader: + repository: rancher/mirrored-jimmidyson-configmap-reload + pullPolicy: IfNotPresent + tag: v0.8.0 + +sachet: + # reference: https://github.com/messagebird/sachet/blob/master/examples/config.yaml + providers: {} + + receivers: [] + +replicaCount: 1 + +image: + repository: rancher/mirrored-messagebird-sachet + pullPolicy: IfNotPresent + tag: 0.3.1 + +imagePullSecrets: [] + +podAnnotations: {} + +podSecurityContext: + +securityContext: + runAsUser: 1000 + runAsNonRoot: true + runAsGroup: 1000 + +service: + type: ClusterIP + port: 9876 + nodePort: 30001 + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +## Node labels for pod assignment +## Ref: https://kubernetes.io/docs/user-guide/node-selection/ +## +nodeSelector: {} + +## List of node taints to tolerate (requires Kubernetes >= 1.6) +tolerations: [] + +affinity: {} diff --git a/charts/rancher-alerting-drivers/102.1.2/questions.yml b/charts/rancher-alerting-drivers/102.1.2/questions.yml new file mode 100644 index 0000000000..0eb043efdc --- /dev/null +++ b/charts/rancher-alerting-drivers/102.1.2/questions.yml @@ -0,0 +1,17 @@ +categories: + - monitoring +namespace: cattle-monitoring-system +questions: + - variable: prom2teams.enabled + label: Enable Microsoft Teams + type: boolean + group: "General" + - variable: sachet.enabled + label: Enable SMS + type: boolean + group: "General" + - variable: global.cattle.psp.enabled + description: "Flag to enable or disable the installation of PodSecurityPolicies by this chart in the target cluster. If the cluster is running Kubernetes 1.25+, you must update this value to false." + label: "Enable PodSecurityPolicies" + type: boolean + group: "Security Settings" diff --git a/charts/rancher-alerting-drivers/102.1.2/templates/NOTES.txt b/charts/rancher-alerting-drivers/102.1.2/templates/NOTES.txt new file mode 100644 index 0000000000..59c1415e09 --- /dev/null +++ b/charts/rancher-alerting-drivers/102.1.2/templates/NOTES.txt @@ -0,0 +1,2 @@ +rancher-alerting-drivers is now installed on the cluster! +Please refer to the upstream documentation for each Driver for configuration options. \ No newline at end of file diff --git a/charts/rancher-alerting-drivers/102.1.2/templates/_helpers.tpl b/charts/rancher-alerting-drivers/102.1.2/templates/_helpers.tpl new file mode 100644 index 0000000000..e1dbe3370d --- /dev/null +++ b/charts/rancher-alerting-drivers/102.1.2/templates/_helpers.tpl @@ -0,0 +1,117 @@ +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +{{/* +Expand the name of the chart. +*/}} +{{- define "drivers.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "drivers.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "drivers.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "drivers.labels" -}} +helm.sh/chart: {{ include "drivers.chart" . }} +{{ include "drivers.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "drivers.selectorLabels" -}} +app.kubernetes.io/name: {{ include "drivers.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "drivers.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "drivers.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +https://github.com/helm/helm/issues/4535#issuecomment-477778391 +Usage: {{ include "call-nested" (list . "SUBCHART_NAME" "TEMPLATE") }} +e.g. {{ include "call-nested" (list . "grafana" "grafana.fullname") }} +*/}} +{{- define "call-nested" }} +{{- $dot := index . 0 }} +{{- $subchart := index . 1 | splitList "." }} +{{- $template := index . 2 }} +{{- $values := $dot.Values }} +{{- range $subchart }} +{{- $values = index $values . }} +{{- end }} +{{- include $template (dict "Chart" (dict "Name" (last $subchart)) "Values" $values "Release" $dot.Release "Capabilities" $dot.Capabilities) }} +{{- end }} + + +{{/* +Get the list of configMaps to be managed +*/}} +{{- define "drivers.configmapList" -}} +{{- if .Values.sachet.enabled -}} +- {{ include "call-nested" (list . "sachet" "sachet.fullname") }} +{{- end }} +{{- if .Values.prom2teams.enabled -}} +- {{ include "call-nested" (list . "prom2teams" "prom2teams.fullname") }} +{{- end }} +{{- end }} diff --git a/charts/rancher-alerting-drivers/102.1.2/templates/cluster-role.yaml b/charts/rancher-alerting-drivers/102.1.2/templates/cluster-role.yaml new file mode 100644 index 0000000000..9fa501af08 --- /dev/null +++ b/charts/rancher-alerting-drivers/102.1.2/templates/cluster-role.yaml @@ -0,0 +1,50 @@ +{{- if and (not .Values.sachet.enabled) (not .Values.prom2teams.enabled) -}} +{{- fail "At least one Driver must be enabled to install the chart. " }} +{{- end -}} + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "drivers.fullname" . }}-admin-{{ .Release.Namespace }} + labels: {{ include "drivers.labels" . | nindent 4 }} + rbac.authorization.k8s.io/aggregate-to-admin: "true" +rules: + - apiGroups: + - "" + resources: + - configmaps + resourceNames: {{ include "drivers.configmapList" . | nindent 6 }} + verbs: + - "*" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "drivers.fullname" . }}-edit-{{ .Release.Namespace }} + labels: {{ include "drivers.labels" . | nindent 4 }} + rbac.authorization.k8s.io/aggregate-to-edit: "true" +rules: + - apiGroups: + - "" + resources: + - configmaps + resourceNames: {{ include "drivers.configmapList" . | nindent 6 }} + verbs: + - "*" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "drivers.fullname" . }}-view-{{ .Release.Namespace }} + labels: {{ include "drivers.labels" . | nindent 4 }} + rbac.authorization.k8s.io/aggregate-to-view: "true" +rules: + - apiGroups: + - "" + resources: + - configmaps + resourceNames: {{ include "drivers.configmapList" . | nindent 6 }} + verbs: + - 'get' + - 'list' + - 'watch' diff --git a/charts/rancher-alerting-drivers/102.1.2/templates/hardened.yaml b/charts/rancher-alerting-drivers/102.1.2/templates/hardened.yaml new file mode 100644 index 0000000000..be1ddc12a5 --- /dev/null +++ b/charts/rancher-alerting-drivers/102.1.2/templates/hardened.yaml @@ -0,0 +1,126 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "drivers.fullname" . }}-patch-sa + namespace: {{ .Release.Namespace }} + labels: {{ include "drivers.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": post-install, post-upgrade + "helm.sh/hook-delete-policy": hook-succeeded, before-hook-creation +spec: + backoffLimit: 1 + template: + spec: + serviceAccountName: {{ include "drivers.fullname" . }}-patch-sa + securityContext: + runAsNonRoot: true + runAsUser: 1000 + restartPolicy: Never + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.nodeSelector }} +{{ toYaml .Values.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.tolerations }} +{{ toYaml .Values.tolerations | indent 8 }} +{{- end }} + containers: + - name: {{ include "drivers.fullname" . }}-patch-sa + image: "{{ include "system_default_registry" . }}{{ .Values.global.kubectl.repository }}:{{ .Values.global.kubectl.tag }}" + imagePullPolicy: IfNotPresent + command: ["kubectl", "-n", {{ .Release.Namespace | quote }}, "patch", "serviceaccount", "default", "-p", "{\"automountServiceAccountToken\": false}"] +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "drivers.fullname" . }}-patch-sa + namespace: {{ .Release.Namespace }} + labels: {{ include "drivers.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": post-install, post-upgrade + "helm.sh/hook-delete-policy": hook-succeeded, before-hook-creation +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "drivers.fullname" . }}-patch-sa + labels: {{ include "drivers.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": post-install, post-upgrade + "helm.sh/hook-delete-policy": hook-succeeded, before-hook-creation +rules: + - apiGroups: [""] + resources: ["serviceaccounts"] + verbs: ["get", "patch"] + {{- if .Values.global.cattle.psp.enabled }} + - apiGroups: ["policy"] + resources: ["podsecuritypolicies"] + verbs: ["use"] + resourceNames: + - {{ include "drivers.fullname" . }}-patch-sa + {{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "drivers.fullname" . }}-patch-sa + labels: {{ include "drivers.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": post-install, post-upgrade + "helm.sh/hook-delete-policy": hook-succeeded, before-hook-creation +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "drivers.fullname" . }}-patch-sa +subjects: + - kind: ServiceAccount + name: {{ include "drivers.fullname" . }}-patch-sa + namespace: {{ .Release.Namespace }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ include "drivers.fullname" . }}-patch-sa + labels: {{ include "drivers.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": post-install, post-upgrade + "helm.sh/hook-delete-policy": hook-succeeded, before-hook-creation +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "drivers.fullname" . }}-default-allow-all + namespace: {{ .Release.Namespace }} +spec: + podSelector: {} + ingress: + - {} + egress: + - {} + policyTypes: + - Ingress + - Egress diff --git a/charts/rancher-alerting-drivers/102.1.2/templates/validate-psp-install.yaml b/charts/rancher-alerting-drivers/102.1.2/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-alerting-drivers/102.1.2/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-alerting-drivers/102.1.2/values.yaml b/charts/rancher-alerting-drivers/102.1.2/values.yaml new file mode 100644 index 0000000000..83d12f175a --- /dev/null +++ b/charts/rancher-alerting-drivers/102.1.2/values.yaml @@ -0,0 +1,29 @@ +# Default values for rancher-alerting-driver. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +global: + cattle: + psp: + enabled: false + # the registry where all images will be pulled from + systemDefaultRegistry: "" + kubectl: + repository: rancher/kubectl + tag: v1.20.2 + # set this value if you want the sub-charts to be installed into + # a namespace rather than where this chart is installed + namespaceOverride: "" + +prom2teams: + enabled: false + +sachet: + enabled: true + +## Node labels for pod assignment +## Ref: https://kubernetes.io/docs/user-guide/node-selection/ +## +nodeSelector: {} +## List of node taints to tolerate (requires Kubernetes >= 1.6) +tolerations: [] diff --git a/charts/rancher-monitoring-crd/102.0.5+up40.1.2/Chart.yaml b/charts/rancher-monitoring-crd/102.0.5+up40.1.2/Chart.yaml new file mode 100644 index 0000000000..0246882b76 --- /dev/null +++ b/charts/rancher-monitoring-crd/102.0.5+up40.1.2/Chart.yaml @@ -0,0 +1,10 @@ +annotations: + catalog.cattle.io/certified: rancher + catalog.cattle.io/hidden: "true" + catalog.cattle.io/namespace: cattle-monitoring-system + catalog.cattle.io/release-name: rancher-monitoring-crd +apiVersion: v1 +description: Installs the CRDs for rancher-monitoring. +name: rancher-monitoring-crd +type: application +version: 102.0.5+up40.1.2 diff --git a/charts/rancher-monitoring-crd/102.0.5+up40.1.2/README.md b/charts/rancher-monitoring-crd/102.0.5+up40.1.2/README.md new file mode 100644 index 0000000000..e0b63e0268 --- /dev/null +++ b/charts/rancher-monitoring-crd/102.0.5+up40.1.2/README.md @@ -0,0 +1,24 @@ +# rancher-monitoring-crd +A Rancher chart that installs the CRDs used by rancher-monitoring. + +## How does this chart work? + +This chart marshalls all of the CRD files placed in the `crd-manifest` directory into a ConfigMap that is installed onto a cluster alongside relevant RBAC (ServiceAccount, ClusterRoleBinding, ClusterRole, and PodSecurityPolicy). + +Once the relevant dependent resourcees are installed / upgraded / rolled back, this chart executes a post-install / post-upgrade / post-rollback Job that: +- Patches any existing versions of the CRDs contained within the `crd-manifest` on the cluster to set `spec.preserveUnknownFields=false`; this step is required since, based on [Kubernetes docs](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#field-pruning) and a [known workaround](https://github.com/kubernetes-sigs/controller-tools/issues/476#issuecomment-691519936), such CRDs cannot be upgraded normally from `apiextensions.k8s.io/v1beta1` to `apiextensions.k8s.io/v1`. +- Runs a `kubectl apply` on the CRDs that are contained within the crd-manifest ConfigMap to upgrade CRDs in the cluster + +On an uninstall, this chart executes a separate post-delete Job that: +- Patches any existing versions of the CRDs contained within `crd-manifest` on the cluster to set `metadata.finalizers=[]` +- Runs a `kubectl delete` on the CRDs that are contained within the crd-manifest ConfigMap to clean up the CRDs from the cluster + +Note: If the relevant CRDs already existed in the cluster at the time of install, this chart will absorb ownership of the lifecycle of those CRDs; therefore, on a `helm uninstall`, those CRDs will also be removed from the cluster alongside this chart. + +## Why can't we just place the CRDs in the templates/ directory of the main chart? + +In Helm today, you cannot declare a CRD and declare a resource of that CRD's kind in templates/ without encountering a failure on render. + +## [Helm 3] Why can't we just place the CRDs in the crds/ directory of the main chart? + +The Helm 3 `crds/` directory only supports the installation of CRDs, but does not support the upgrade and removal of CRDs, unlike what this chart facilitiates. \ No newline at end of file diff --git a/charts/rancher-monitoring-crd/102.0.5+up40.1.2/files/crd-manifest.tgz b/charts/rancher-monitoring-crd/102.0.5+up40.1.2/files/crd-manifest.tgz new file mode 100644 index 0000000000000000000000000000000000000000..9d87ba488d7e891a5ed92a96befd2ac779846475 GIT binary patch literal 191662 zcma%iV~{36v+mfoZQHhYY}>YNYsa>2&yKlc&+OPX@9duM#<}Oujq^ryzFkjMW_Cw+ zS3Q-Lg&ze0@bC9^-tFat!=6aM=l7=&VH+H%`eFHty&n^iY6NB`LV_dncY9Wiea_|~w&_RjsovMf zNL=sx>D|cYgWv7`jau#(q))u$`{?qQQj>f|Jh`|y(6-ExGVOS!Vn(xCaJj|6U7^r* zRv9SeK`^?A{*dU^RqDt*eNPS_AJLV)u5r;-x+KwDVHR^mTb?zaY#V|0Y3(~((;Z*B z3|a0@;HOA%QC2pAw*0LeV#yb2>GBs*X)-=7zMrG4hG$?-s^ZN%Y(f_bq=%SkvVth* z8woj7srsL!9|!}{4j~V<#h>Ito^63gPk=iLN(h&@!l|Sg?O7(u^dI!oA zg?oG?9*R(9p_7NFXgop+!5r;_JiwN?t>lgdl4OLo@0d6QzT!8SQ)|`}InkLvTtS|u zo8O^x6PMCz=3r?~;xkh%!UMy1#{bzbH*Xu6T8VlZ-lv4}@HrKWL_AV~tQ zVL3XYgBVx1LSl*{B4afox=?L;#m{*mtHD;^PPPVKUiUX;UXo^lMV7>cDfC_DA~a

    9U763->(UO=LZ1yLoHDMu0Mj(^2ZPa5PNo|F#MgS9W`t5o0HCQH?7p1~hN z#8mn735K6$8|yU}nv-dMNzF`W9-;p%du`H)epqF{+v=joB%MVTTaAY33}^0l3QDSL z+DWFoiz6uvuKv+WQ|nl9-|$fuf?H83BSG{(M*;gU{7p=Y^5?5gaC-&ng!Ur2_D0}mtNSfCk_DhTEE_vIgn7yx4=fvb$o4qz4SI<9y{;ZR7h+IA7ho9O_gCK^e<70 zTLz#GXbjse!8d~rDdFxz&JD$6uiQsy~YLZIER=nK=?TeO8C5Dz0|<#k0W~+ zZxvbl8n0d|m~G?uoJzdQ&MynkUkhn6ycK`_IIApY&uUVN|EZV{s*A)c={lidE0L~& z)5!v?o{#_w6~(FGsyoTHeARrYWoO_mi;zCml#JEK+SXyq`}#6@NURRgb?P$!cTl|L zDL8MOv(VMW*`dk219YIhz&$vby&K+Dh@IF3_|=q_q-+rwUWd7SJ54ipT||R}`6$3C zOLWhpaXsm^9lda5*>5)~_`6puOGERq=)m0ZIO?<7x<9XbA&To(9zVnu>w#pCv;?zf z9HJxN(xCltLYJ%u?(4C3&C&hV*z=t2g*BlJfrOCKAOoI0Lcne`t4sK;Tos@xG8fOi zZVuBhOIAX+Z1MWtoSubCB(gW{BZF~D`;w|P2>r1R}Y(XC;=sJXjr(#Qc zPbo#|F|e*(Gb?YTlR)KIB_{9T2$V4~-v?ZOq zdZ!){gA=KHU-dPZ)_c~|l2@}bXfRp5a*^1Rt!ZgWJXACeH9;#JkGPN;(Mfc+YSNTt zLPB%Qj5;LLg5AzSmJBzEAv4if`OrTY5K@UB2#G95Ii7*QfUU<78j?9u|5=3=oz6mT zI*rIMqEVi`Z=(4r3){4q*3*a*o3LM6luFMSeyUk}ZEhByAdri?j3Dq<8H-cW$?*6z zTWeV5LeNn+IY9;kf$$4;+R7}JFuubf+SWfIyKES}AuM=$PRmd8P_^x&6gT`~ znrpRJ8&Ly()gDLoi}t`HQtkHPoYc%xe(IF)QsYk54JqeO%G(F_63L|9%Vf}eY{!~pJLV8-y3+8y#jkPVI-$}fAPu)F@aQ{EY3kD@Fm!*) zkt=iEL-J$hZH9Q%`NtE<*}X@J6WL?6C4@vMXjI-;=#M3Q{dYj3ijSQ-%6*E(w}r+( z9UEI+v%I-(t}MWE#n=IEshi`q5NOp#l9;dfX({Se6#=Aa7WGPFb$q|)Ln>1eSj}@~ zj?8Ba*0~va>79to1DQwQ&WYomyi0)Rs7Nkx?T@I*>FXuXyu!?{ZpO^K!gQ;=;x@7Q zfS{^S5YItAjy^xx?{}^qovzIGrPA;nV-9%fK&G~i+#f!#NQtk$bWJOt+OmEscK8;g^v3>+^bIZ)}ge7w_E9siD`;@oAg$`L6$s{|yY!oGmzqj#kPD z?&nzeiZa0EuKA+7){Xa|MhvV>6E05)_yWG^@z@b`fT{LAF`OfG0+L)(TTF5%i)L2U zKv#mHiNSxotf&1-4E%?<_?z%@B2A$+oJDrm`3KUf?wStgd6s(Y}@-YzId@ne-#y(Rnvj zgtwotqS(GyGew9uKk|L|uL=}!N?lGs)aUyvLenNi_c5-s(Kk(ff{OI8m2q5#N=UOMbbMjwbY`VGZZ~`@X&BHd2>LSV6O1xe7t$HrBw2@#UWaVwX7ovjh9z&i2+M}$_A~q}CF^~zTk7GpD~2?r z(h4k53#lxbm%2ky*ta&*4@-(QkGD0s+;)_kQF4qZ?b@#0aoH%6Nf1RYB%Nhd+4@rK zOBhV1*^ld9<*s+GHSqe^J%slZ(1W?xi+l%f* zo?VCSM)FG$5g`lT_tR`zxmE3+GUH9vYe<0%SWt!IjQL0A84B93S?$6i-P%x2qB+@V zYK?zL*KQ?D)l)EBG=g+qdrAi9l}pa+En+|mrY8Sn(jt^pE?#P}A%xe=K*t=bv{Kp= z`SphGOHi3RiGWT2_{pk)ySASQTr^IEEfZczFiek2qQ@3d#S9~KEXbHHhgQHd4tT=F zTUd98!yOATFwmw5p*J^^Cg`g!eiLYn8D~rkO{X%nQ^m`#rSda|PFFXF*n+O%BDA+` zgLr~%^zHNZ0NQzI-V<00Zl3Rpn{cZ7;~*+Z0%!_Hr>CuqVPCA#V<@Fl^Bz~6^2w4U znwn&yU`HrF5ivDfB|p5qeN4iTmoJi7&@pDkyRkz6ioB@7+U+gORIdU?-n+eYPAUL)qTc2(&9iI;)RQ%0|eR%LaJ)yk6N z7iT`v%bs_ZKQCiR4sR6p)Ks^0uEwt^@D5VSWYssQ!rM_%HF3`3p{jvKi-x4*MJ-m9 z(qTV-C2yBcG|-ZR72)?u(2-A&_9@BCAUHVvb#8J06!hmB+sJ!GIZPGJ~ysr)~R!@Ymww5m8JRljo|t;W$HYFgXM=ujuDAuKuGy-6`kB)DzURb z)Jh$rPLIt!^2?Q#&!L!awzk%&_u#pvC|;oXn%Xm?ax*j0m%wfaJOFM8J8(c7g3tDD z2vDzkndk{LHzVxymepmc^%;ZBUP9-A2Ccm~a=VR(_|@a8WKr(o%9Ju;Rx1zk&X#nL z<^dEnWk~8(OLm0nV~P+xMdfj#GaDCR(qYmTu=u1KWMzQ1$S?!l-LeQeNIT8QsH*;) zsMAXpl-aANr$L@zhK=<~IYy^cyu|Zg6wF*#vFbUgkH5zzxAk;$meZlLRT;SzH|@*O zv}gSc`Kh-EE}K4VT_7vhlgMbUm)bO&v1Db-Q@fN&ryEd-t}39YN~RyS>xv>i9RYN$ z*J)R`7@A)UB83##T1ze^yt4Ti4#K}0?qm=ICsooCfGTvtSxkYl&1spih87_{2|y5_ zLgXoeg+>5`@&S{*`<#&vZ2INK;3{ zu@e12yJg+sDoq9`SXj>8m`#vi_&6E}wUxx)oz+wjZheu9=fs3uK5aMBzr9py;orx^fn@trHT^EYYy?6 z>x}+eU;&cnb^c??)>vJEDTl4w6_`Cj}fE z6UGAbN<>kOSaa+%#EZVtFe3W+!ZPW7M%${eQT^qZ;?avP3r!m=+4^_F;q5U0Rp3Uu zo!GRGA+|RRjt{PloOyBICCjJ$OCc6R`>IWZ)+OOG*qy4eYgrHj9Ed#u3^N#lXk>vg zaF{AJliiv6zRz_EH2Vz2&da0U?{uC?E^N{Hi4;UK@u27-C%M+0h#2(h1gQKy9uDXe zgvh$Va-zBHvpsga{>=(Vj8$ppS9$B=s3NtRE2xRD7QDmwLu_!bOROAw3^s?8;mriW z39WHNrNqOOdYl?T_^`_M?XgHWh%OnuavRSPE7M)|aDq@vx-hT|q3w{|Q_vKJ&e{YcO>4C^GZ_ zD}*7Fs}e)CogmHxq#54EMcO9|s6wh+x%}wEg;X)oIDT%iF(BwH3L+k0cpQd3s>m8H zugOK*#jC5}cp-VZUCX(gPBnS4R1)TteA%DC&eNL@o=O_aBkJ`8$L}|UTz-z>)BM~b zN4fYJRaU>^IkZ0aE+v;{rN1;? zYdkXkoKi2x3UEZL)__8K_X<&YP$puef3at2n2UXym zhIWuyus~3UJe=HeTQ7$-U|&?wC{~3l92yQQ+dvQDhDrsocuJlr%;aq#wNt%SSuhKk zg;NDH3)*)G?c=6M74%j&aSWN1c&i#gmy1NxG=o|m%dG2hHYG{fQl0uNWX#=&3f9GA z!ONlOU_HdnxYa+n&63yjIT6h%R<~Fbxz_0Us#YxGIxAK>fHx^(r;lSz-KTahoJOq4 zGB=Y9G93cwV4>rs-b!$?3fqY`Y^dB}o#93QQB)a19&nwcqos7X2n*$Woo@FSxC?A- z8|z>(NerRp*xRa2U$quPnzGoJqS~Fmk!k0o`kk_~Y|KvlI@qKHby}eOAbE=wH!BL+ zK_QCtxYJ271xnA0X=Q~wnOK!PPW66hGJfb8S}oUuosH~?4-0h97q}(v>mN=jA~Og; zRD!@5vL*f*mj8tKuY%x3@Vl@58YSVs=hFoLug(=%{8ApMJC4U#_=@=0hXro-Z4`R; z4Z;(@8u*?s5}Wa#A|ODDav%WRGCRP}v;4$y;C4OuzW`OaGeDXTaNqeiCcJY$`(_S6 z5DolaT}-Jk=RFAU&K!U z1|55E4`BM0z~p6L^rYl+g?sXC5TtJw;QQm9A={u|1r8l1P|-K%`ED92#sSdyC6UgH zf%T!h=_U;P*AKA-cVN>$Q3wDt`$lowUqD6ao4Ut&vmo1Ar^?)1ar}cduwu4evXE)< zgZm(ochG4ly7M4%eab>?JFZsMdyDzGf-;~9IFS%cB}!zbkr4^iOi*f2(}+sxhnC%D zWV!{92t(CzVp^(-U}nP{`XtGD7wMExyvF^?>3a_TcGvUOcMN9D5(#lBQo*tMp(_v2 zG+F1$TU(Lj_h1Ktq>kf8^nRqd3Vj!QXknwuqF3wSnuE$=#$H$6`70aDNb;y(+ItD6 z=b)`NJE!Z#a!Utor>|M6Q~3#7lC7W5L9B^FWT15Q4A_+(ujl#(ZGTlwaiGb^2rswn zQxLun{b-e8%Xc@efV*4a6vW26f!VWsq;RdM!1mQ|);@$bQ_kLL#*s|PF&os=}_3#6(@-QTG39%{HhI?#fFtWMz zFxJE$F~>BAbM{E!mq+1K6}85tZEhcN##MST z9tq;8cF<=mBB=K22t;^81ST+nQU7?2F#h;>jl@ycE%eJ2?vpsxSGp5eo$2|xhK}5_ z^i#O9RZ-$`MY{XdjSBc;uP3@SdDl^_3n2}DZnpeZYoU{#x=g&}eH}qjneC0M7ua!* z%tBQ9&WkmjE7)lilg}Izu~_H=kmgn^c!e=0qD-JI2=e9VZDW$`B!qsvt^N8wDa3xc zS-;DxukSd;>vAtn6^c}=MvLG)8^1sWc#&r?V3Ftd35(4g|94fM1vZNf z@ECj|KFOCj*-O5~Iv0RASS$<JpZpE zGynx4Du_I45nve<5NRHsgaN`o0I1^obHxTQh5yEYO2bNQfC2vNdj%ihz?J&e2cB($ z&1iA?cU>*@;`tVt=y?|FK{t2)#JJwnaF;*-za#IrL1KVVFbpgLFSb9vV|WZKVC6u< zd*6dJ1Iqwl+K2e~dH?Kj>1hu();~XjYlua-{{*Q1_dO)QcX*Z|HcRL|Br9!7{y7wq z$J3B_-RO1l# zc{uLp^{Nx8Ng0vG2J|C8THmf?$!tWE#4**gXc({~M4%Pn811&d zBjlIM5eF*}9Wkc?6m-&j)PCSLLooC7VFQwGr?Rkgf{s{Zp%QgabV8lrb^hie`vP{v zbEocp1t>|1YY{hQB8lNqb~A{ix!{Ma@WV)q2s11V3ph-?B%t@qJXyUSna8zI<}W~i zbPy&B+)n}~3vk{H<4|P1uahk>S@hQ-?&E3a-v}&Q8Dl)|Za(Rgzs^f2k>y|(6%;wo zWEuzcMKRGbQkYDj#=KhS`p1Y0QWU+J?S!|>32VExF(k(iF;8Z%_)R_CHNDEjmWBPy zG#ms=P4i12XD%AJd@Kz+<*ZD^d@cFEG7tQ9shU<`pZiqGlXa8g!2~@>;Y9FI>Y^fa z(>2gN!sp$0@;D26SatZI`%_Y*;3ykuA`gFcR#GU5=reao zHUmz&W=u+FB>qTR;t9QE-fwUPlBkSE8gJ>)hcfJ|;lv?NPd?0`jm!1UlJZ?^D)ih{(-EuqrF?T@emne zdzlx*%c%D#nzu_7*TenVmbXQ{RZj;=9K$=xvVQJ9lp4Ohe3Cra4X!|Fk4&t(scp#&0&=*4Z!*&B<=Ajmb&naAZ(wX)Gu_biyd$kKL z#=1I9zJ*JKgX3+AVAdt^f>o?GCG`vg!WKZln&S7UZU>G$&VKurH=Pqd{LiOfeZXzNYKwhJ* z(#d$fDq4m;?SxGRPxD$nh*@3SwaPa@I+MQQXa;|2OnXY;?=pVq>CT17xsDp^k;wzcKAz2vfxN(b-Mx0YvC>yI|2UL zUMZZ_-GrXi{pf%8=u3d>K?Q%BXa$PB7p&+@`1Hr1;g}ozPiK{|^<%Nyt?1hz;x`Y0 zfsNB$H4lYeB|GJH^spE12FkXZ1SjYey`-R4Ga6v58(yGTr9y~0mLBS^)L!< zF|c}<^_6{D)K)%xD7yhMl(NUUirNs}VAV~{6G(HvzrV}XZD5uoRW4XA-(QKJ`l<}E zdzFmzJQcRkiYX4eyrLP zw?|WY0ZY-V01LOL1H!_|{q-)y_Z|)#m+52;2aeh)zz44NB&zV}@&0|14K63qf){CQ zzbP3xusDM!eHZ5|iP@XD6mrIA?#|luS=yzPpSKB0?M%9)!RcWl%8Wm)AOo+eCU8P- zO<2Deq?;<`$GLUVGY$>=QmfHoREMa}sD5__2nriM2!%CV$kYeGC4zleFkqW-A1=^$ z5nkB9>0iZP9tuA8l_2R$Vci4wcas124cN#Bfy(q64IMa?U-8Dtt7}KktK&pr(*vil z#$z}9rw9yCBMAn$k!1m3JKe`R2JYwq_7~7jItO5Q1M^=UWE~6ZJISwm0T96W9?}qo z0~qAOkSMIT08rS%1`c)n|JgoQps>CKrLg|gf9Khe0N;)bdOOep6n`a9*^%%QfI-7K z`Oib$k+>>7-Qu46HVEOL4;TtInj_n$*8mO^^ZlyKx_>u?iTMv{Wng(N@3;yD`{(Ds zXN`aud37H9K{ewR#@TBXY!zs-ni!lpE;pwIN=tV_gKmC*(m(QIgxzkI$RLcCz?HvSGE}Ki?55Jz3q~pM8 zOx0q7(c>5um!28kO5&4^GR=@3$OJNc`LBxcfV*hAg58?(rfcEwo`vcbt?ts+2Fl~> z#_IhlRBR<6`oVK9k+##HkXnlU8a41YNYIuG`>mz?dhf2U_a}EzQQN-mpI5!vyk1e} z;cz>(5HWf8LFjq+z{$Q4-?5mJuyu=|ke#*bnEjfg$Lh$|Q`#&nxU*SIH1q*2kItPR z0))TAKgdUUdL-Vk?-aF6)mdm05X-jLo-#@R%W0k0%UKg>7)%0o2~78Sh)an)j~}bO zx=U1xepvydFM6G=`HQwmhwSXW%(KtXYkBssBRpk;6zAa!P+kL3faalC4?IcDp(w}2f6h~ zL07c`a=RE>6Fg!TWnx0-*e>~4E%tJ&249=)Gt9`-`6jkk!K%F9$ZUnA}mVuMMNkxS9N%nBV(LhuxdITs^`3u(a-Funl>g-OQT_(n3r*P&00uo3Xn6S4qy|* zk*L1ctkg47!&Yhl?3#0M%2&6teY^4^89jW>e;;OsVQfzzgHV?n2m{T$itZ)4&3{O=Va1}_F~=?GlA|a zXeEe4<;@UO29G87vq8G@821STj3mrqPQWF|gPa=}{GFM;Pln^aC#;MlETO@}@&7$x zWeL^(e3;FFdV>tjS=5bF=vW^;dt187qkj$4g(oOID!{49IyLr z6@GXIIM(^Pnxjpx2i3j^UJ*3*{%;5=0S~V4oL|utKK9-hncj7c89%J(uT+o7kAtpSHdhVp8-3#F7uqf63HT=bw_P1 zpT3NBeRW2;DYo?X;L5i6nNnCYPG~2dmXVtqUq7}kmFrblw3nsetr~lQJwQ}$CV9GNX<&g)2Jjx8g7RIhX%aY(-ZkPYSE0N=)FJu~aIgoJEOyyQ! zwN*({qR#ZPcG3jq&2wFC8Q5bl9;^I`=?P-$<;MREsPbj{2tFr^hba88YT#8l>(8ZO7YdeT}8L!-?`7#5jm4@{H3pF>Z=dx;D036cD=0Qbms7g|s2b zixGjH6$8qe+405$vgZrB!(5Yi#t&sf`Uck|{QsMbh#CG5GLmo|5Bpd97l!~gB>nec zPFxQ+D@OE9M*b>-0BB3W1GSZ|0Bk4)OP|2+R}lRJlDGatM!vCcGJ+lPKgoz#0iZcC zpbbd}z(3pHWJGKLdRA=re@#ZPBmUQ9($G|5x+oN=@_8(q;T zN9m~IQq<744G0g6BE*5Nj0oR(9IHGvO<(1I@D_b$aL}t&6mSfRAhuvVTCFV%DwX}h zr_n}hb|#KXC|v?}q8mAht|DQWGl%A@QO;KpIxC60g79$J_bvNh9@ucXHY(j_bN7ze z6bd&vXVv{;ISoW+K_C+mI!J)V8YtRm?<;>+VhIGOVUC>Gbh>jmcb@#IR|z;)bug!Z zg)Ppv@}Ha4mk$)=1oJ++%K59?KIakEIDckA)>U zR2Q5A7WbQb{Iv-TK$iguxWQ`ad?~M=@77*pOe`my8e)HHoV zYeuM6O-)$yGwo?x?zfdGi5@iez~sp7Wanb_MySGN(W#59#D21elvHqbM2rW1#%+w` z1Ht2u7uO^#ooXWgbTQ=$^-AGCf;g(hMA|Q~ik??>Ohc(sw!s^aN|t-ctMIi2TY^3j z3@I#zC4(xnZJtO0R9aQzm^GH65c4OQv&TuA=&ogP}dqh1TUte`sgtV)kH@Lh^D{XGNz3;}!jW z$Phqc14y(75J1|JmNa)Sc%mB`PH<+DxnsHFjqGEUZKLqE_6>sjKq|OFEeEy;G^Ggw zm{QNfjKU`f1}4m6@JE0Pw*M}E8JG_P{jGqO@Qr^bc;ROngP&4g0_0$_H~t%yK2J=@@8bH7zjKF`r2e!{fkK*E5fbL!BFE9s( z3t+kpz~J?bA#m`r7-0hZUGMqFgiqwZePa2W(#%E|L;sg;UZ^Scu5YV&(YfG%S5_SC zd%dZ5B=C!%a8BUE`vZx2ar{^PiYD=JPQJseUgBKe1_2pZzGJehjgU{$kN)|L5~vth zf1Bnb{%0%<72{WV+MNC82lPLaAc6}DfbaW=@SbfJiYOEH^z>CF1<{~}p0Es4UEUIo zQoAb3tHACrZVlBU=mB5Q5xsGuNfzG)gVVZ^K9{tC9KJ6kF$6EO)lu||u9l51x_lWt zRtxJTcQ5y3+X;=Guj$p-YbXs{Di^FlRESk+IL^hZ#Fx;G41VwZy7Mzp0v|x>rv^y~ zE*BO_p(QSB&SN2Uk=n~*AgVh>rNn*h@Q_v~iZ)f2U>zGxG){e=s^+O-vvf&qaNMbL zwE|y5_NS1R-GpJQHK8_j&8gsCMO_N;QIsW1JV;rXjG`ntQ7Q2oa+lgi$!;GTScTP< z!~Gl^crR-)nfmGB^O8I=vb5bK{_Aaty@t zN#H+K8|PKBEx9+70jJq(J6T9Fbf=icd-e8prD@HSXT~7{ixqI=qrQ~yHj2VeH*CqM z#LD002s5cO@^*>rv0)ZiF!T?5#No~f0~vcsln7phN^n!IFu3{gsjwSekwPg=@||Ew zO=XGLTD4FjY z?s>L`JCc3AR(a@d{xkGh4@N5Ywo@`w3z>N7Eb+OetcL8uM1b_&?@dh$%+J&Mu)BA8 z3%lFVoS*a+K4TL%dbf!A7e{7%mwcfu*1)*8bsuum)7QsYTCINIr^|co*FpcMp3zFs zf>v&68TItkXHCoO$^yt17X;DotWJF_YUHx+-)BZ${DbKv;G_0{gEr)_t+<=WZ<-@q zX$58vcyk#Pxh}4jUhG>m;pBlqvL;hfZZzL%n?m@+f73SO*zjQ*XHq$^=N!GrX6WiX z$j#G%mu8L@FL;IPH7;y zJ)R){x?M~P%8$8a7KH91p+m5ZArl1|j6B^wy{O@kDw7!ZE-ziTT;WtHTuW*(Qt}-n zk%cUvJ@^xL8;Jp~5IE)Cb{x5Q7}IzhS@o=+q|!LRp@2rHg5`G5^M#LIt7kzywANCy z`!?~Q!{)!TGnO^o_?f5zaLyhBTJ}`486p;j+2_$nFnI!f@(1m8fR+}C*6hS_bY_lI zZL==WXKJ--5hG?>ATv+H|aeg%`vlZ3wWeIPN@y*qKb_r@TB-?~hr9?2a zy7(>rB5($MJss54M|kCG9&}3a=@|{pb;QLkwp=|?a;yfk*8my&i<9v)@pzMuBFgit zj`ESLioO$Kj8?c$zx-TMESTmtII)kJ)|akh{qs0)W`KxVYZe zQ|OQF5j`J=jnAu_y_sGv-`nS#Ts*(egQVIXp3n0}@)5tk*z@yP?(EG4|3#Q070su! zw>L3G=k(}~Q%^6xR@v8^z|Op;IkY!(bD&n`Pe`*!1`}iV`738d%)#znExaGS&z`T( z;VVYlPm?5wCkfPyKUTdn>;>Bv^j>V7j(nKXW%w)%Ysewcle?6HCFcnrU-e|v-Cv%Q ztj6b%@0*e%kLLpqw-9Ir%gg;TC&-z%+Zhz5Y+N@ z^W^yQ`rJmG(3|Qm3#tF#eGw7N5%O*r-NGLt_~sujd!k%|AMowFudz~9qghdFX*qd0 zW2}DlPW8VL<)FLp**@8*d^Zk#o5E?l@gxs9!IF0t(?&zvvtUVK{qJSd&M zVk?rHVOJtzC0-ifD1=N6PkBK_JYyOCPTyVTQtmqWoJ-6%%fp&1iYZ-dosZ#N7WCvI z$Mx1U`zU(3o5Pw@U9bHGMGwKv%l~%L7&TqHTv&WzANetn;^1Ems@b4OH3tQC-|4GK z-cAhO$(2wtUhR6|r#J;iU}4gNks@#9k6Z02`+A#Bwgfdn*}-t*vKrYz6Ctlzx$h+Q zrLk87%eFr00b9CS2gB|3G}{*UcV`4fnD&_F^CWRYtJ_6ZMP!hLQ@ycrbu0{Xfm-pd z5MHsdWCO9}5i(68s0xWfF@bnn9q+`fJYq2aWyAE~BER};4)rJ);MGf5p$E(pxK|^>7c=dwVZ%P7UN3q{$TxRBER7>X_R} zr6ug%^Co_G{21&z|u+qfw%-*-PR)K6#P4I8a=zD z8r!=e@Ox9h0v8q=*Y)HvA8*3av~kwdX~u4Oj0+0SW>P40Niifnd6pz8gNcs@{vMw1 zu#zK}#3&f!jO9&%URBsEnOHJrLXL4sAUwv^r@e@Qw{Bq3j1=!S`a{^vVf1pZ?`JGY zF_sNWmxk+10h1QV#73BmXW3g2nO5w!FGGOu*-?VJ!ii(osfT6Ndx@eq*B1C-gRadw z9+n%iN+>-!^fam93SIEaZlWVGG@_X?4=%%U*WgVCnq$$HxL^1hC1az>W{A;A)3_@Ex(M1vXv-A$OlHF86V_%APuWi=kYdw7P*uzITd69iUX8&`s28~fyZW2&G$_CXr!hz$AswzQL@5Qp|X$A_=dPAx9Hw>(l#{R%-pJQ%fwwuDsZIi3m z6;3FzNKJ!b?rt1K*@x6UAEnFy*Zij<9Xg*#G?qiW#@(0@pq@l9wk#fu{>`?1t$Hdp}M^-M@qSLR_uF_78`F)Q8OdY zL9oq>dkU)NsVPy`gkdYAx?yV*Vbv6|RpWVct>TYnh-Bps*2Sh~wPsmEdzEP@Dx|lu zdHfH$+ug}1yufoa85A7surqx=OMDKKYlKZ2^MGmmok&_mIjzotbs6G1$tdWLNEd@5UkHaap-JRT$No~jo9wC&b z$zq7FOJHJG0dV^@#{qC4TnNVkdB6`qe`Nq%umJciKOWOK#IN;jm{zq=99>I7 zDNQZ^s1#?aqAYWDk^{G4DQ0|y>s&gU3Nj_{_GLyZ7R(2--o&DAl$q&)U9(2QM5z`u z?WxF%h*5NP&5waxTbVzpH1jm210F#5*|me2Tf26qUrRIFRc7eCqI;}*NvvB`yM%_e zi-qbh%MYhN9|y2gYYk$% zdCaf&hc5gAKSwdRsZF3Mvgn^1s$F#M9rW;Ej{Se&WWR6*ak1KjkzK<~14dZL8iT3@ zYHgjZb;~UrP^PXwJIS5ve3uV_x5|F6FWOK!6*jEm)!fEIt0dibahE8J8PFOEcT_`b zBQVx$B@|+sSj>c1XkcW%&0^E(C>92)UL0$N_1A8@-HhLPvR=LRMAaKo19rv;|5ES& zqKHsm&4_GvQPb0eKP%QZ#z>*)pKEwUU}ai5+$MtxT>|fpEz;_nkddgfT{{yxg;W|& z`^6TG`RrZ|L1t37;{ z&6cR?qbJr4Js*A+j(*N1xaBMlo!hYxV$~HkAJ|R}Eb3>hR@F(#;o++^Mm6zB$RXF4 zG&TAmz{i#SNo;w%=I_>;O}5x^{P|aH22EMSlouC>cMm|%(%&Iyb%0%K2#35=FRZ{toP@O zjDSUx4>t=>jpC`{QtEdDX$^4(Swj=)?Mi0eDG$b8cqQ2^zar!63>8=fxmK8Qb=$m~ zVGg~Y+Ld?fuYD5f+b#?dB{2;Xa}NviU{+%6<3kU^atukw3J3X{RxW~`EH+XJBB0Sv zU&~*7WglW36Pm#R)i5ngE91TZTB`sn zxzyGdNVc&qvz4vGdW6y(GC;Rmsd&*G5+vJLwIy}SuTRi+a!<19J}Jq#>EkSuJ%T+t zbaDcC1V2X^$1}p{wXf&%2d|P_Nutt~S6Y}V9|aVf_exP>Tq&yuoIPy{_Mv1isAq@- z1;V?QoC}ps3y}NdO$%2lRc34=4!g@bb+NTKS!~CWB9R)sPGq%DRtgklB&`!YU21E1 z%|vfv_)K|*{16-hk%0$t6HI6RxmD<=R4mm_j2^sd4Ds8uNa=nfs{t+GwboNs*5r-> z48ML}IHghYR~9IA6s@Jf#(5Xou;%9sFm(uGmKo>+1C0PL4EUxe#v$tj=?Jt@w7#{J zX)xNb06#t0`YN$iC$u3NH9fOvRF&jm)4s`)FoRQiUp~2>Qh3tP>kXqG6FXhepB*|U zG_!PCU&g?CN2(PTnP|_pYd_nd9ggc+D$gwS>y>GL^Qmo)4X!KNKk1U|UR0>=@!uRIl@W)rJ!I zpK{j{8^?HhaBSQboe;BOJtv)oWiyw=e1O04o|>=I2Jvz1=q;q??=ys6Kt5&rwBG(7-_6pqWD0g;` za0SlrX|V?)*eQN%LsCHJ&;vZzJE(5+ZjZ3c`h@FI4ZZ0wbN@r>bjLd;6toji5j?W4n5jPdedO(L2$O8lO&BV~=Ux z?VhuDdt+MZHm#d9fkqZV45N*w5i_k^l6lcGInmM_#(3^+8EIIt{W=%L%Bf$1V}2v= z|5@FFXAZ}~1ek7#tzoX))H127!d=Bsap=@|v_onqjQVH61wyR+a39tq(Y2?1A)rHq85OIbz&kpj|+ zK@}(!7=TAm7P|vbK`h`BY)>MOYv3I$j|=#>2txmTl*P^xA^fXEa7S1~%u6ItMhcb+ z6a@JDo$3H+VOpEir4$1EmF)jyzv?B>%6O+24i)4hz`wR=d5QX<@{$NtkV2&WeEw2_ z-64K}O+Qs7~Cs!gC2`&mKC zWZ6{bq}-cQrL^PVK)yjURJlkt>@qcT zTjy{$>z;3#V3F^aOt|>)UJ&F{hxM}}kD*z#5{N?tg5J35R?>}QGR3G~!wD#+IOZQh z$PAobbdr^;`ok#<+>!Xwy(M;t!7Jz%K32X1UHqY?rw^cRBtR*>yf&DbIo#KcH2(fG z!E31%hu1FO0E{30Zo`JvA1>0qJDq!v-7~KE0&8Z!!8Ol-F{rWmmiXYu^dG}FZB)uQ zxV@DAKrJ+~v6kzGNIST7J%ErMwl^%>n(r}F@s_FPX~N2Zm#KmYG-H9tS0j|E*53zM zf_~>gSEv^Eef)go;x@Q?`Qq{WJUlxSf zFr+p@B!mD`5>h%`Qsb<@;^(Q&uJVuw!b8jWS%>xQkhK|vlXx-{3$+U6{eTIGG*PKn zLd7P}o*D=ADUTt2K)ROZA*r%{fKx$udEJ83Kh1_)ic0RInF?--s*&~=T56Wg{mv6G2yO>E~*GO=w;c*nMFV`AI3ZJj*N zd*1VX=SNlT>Z)D!qp$9(_pY^8GciwGtBJ>|{!V$sJiIj-%K1&Gkj?ip)?$F9Ff1J; zKpL?dJN#~zfaPfQXp&#QI6k>PWgF}HXDJSON$TI5u+FTeec#F;_QUk)b~s^`rIGQx zLp_g5kY1Ai19R+RL=7eqTV;)ILJz7|Tq^?nkx3-URc9Z!<)V>z(8?uKYbzD+UhWEO zn(1&te!lpyERCSFJ82B+B&ssCd%?Th?77+I6u{}3m8V)VLv z9$z6ci%QS_s{V&Kx#VUNVK3xMDlGFZ;v}bun--!rd!H^Rz6BL${SU6(A5bfgC2mwT zJrR(SdwNhOEK7H_w^X^7)9hMU&5MJ}JWDspiW=*To3)Fcw22NZcSI_sDTmFIK4;`IiEiO|W~kX1q?$754yg!$=&ZX@7uW3 z%Wr3-*YF>~tL;Zy{u!x;sh2^{orY@hq4&t1t?a%Oz9_7)JykyxA__%l`MLcYos7J) z?>Uok%(+#AUc|cuWBD63ct}p7JX;>6Dw++Kbq3ItzF!T7b1@Z#fC=D52h)c2gC494 z&Us?_K$@Do@&ks}-h9I%%(wnj@LsEH(XSzzuNa=MPL&0&+&9sC4Aw_&b&$|tcqE8j zZ`iflHRYLInAWb(4N+WdAjm`_EO|X%5};yS-pmoS(dyJ+QK67|d6(&zkV(kB_l!af z%tV!#?0j>FDP#_8m4Af4-v?ZWVyHZS^*CclS z8^FZfoIBlkEK#ng(Q8u<(WCyW)dDMEf)OCywTSv4&(su}E`}F$Im#y7o$MZzw{HcZ zOy*BEPTm062*q}U2_D{fjTMV(LYdXpl&4t=yI56KZL;AZ57e0hq@jZ%&rMSvnR$rh^`|;5}1OefZ7ObK%Zq*%2GgdF@bm z)IIxC#*Y#*JiR`@&10;@imUf$Oz|y0%=y+z>h_!27oA>-(~3!|XTojD1f5$=0l2;tlOdNUD;ZC+yB@u=I$LZYYn)mbF_>hQuT+3NLY zCYwRO@b+S~F~WEcRXTAbgyZQRokFz4zSy{-A4R9^`|MY(=7A`u5{XgGzVsDDr>w`! zf8sA_iz+yEYciu6sgZ8!M}#Y|B^P*wL(s9b^+U1K6IAtzP*{i(zPBF;lj9M1dI7H3 z=|2IMPVn7+yTz1on#PkC5@RhjJcmQ`M0TFuL`kc#46>-^d(7d;)sQA_LFmavX_i~p ztZ2qLJp47_?Rc3qG`+uJL6fmtPI_gq+eJ-0e=D

    %$Bfa4i`jjA|@UCXAT<#bCrYQvZ&nkxpQjw?#X!^6$KLL@Crt9$^;roq>lxjA>%6TPbj-V0t5 zgwjVrXY8g{eIh!Jerl0KXpvi@84v7J6TgQztdMMeG1}hJ@)|4G(_=$b6|_9(z}xWj zH%TWVY|g0^Ls3e8P8E42Kh$ODw&ANWLMh%=OdPEY*`-?pf9G^1i&;q>g+;7di*paI zH<``Z-8PE#{&1~0rrYP8={R|)JMSZ}W7NVy(YwI+;346%@^G|qmoGs@dfsMq;N2qk z_FQrt^<$c&{)r$t9@AVFrS^QzkH=AiKcp(GjU*_y)g8~$HW|yCv>_X1GhpOJ!bv_1 zJ26EU@ldotkK~Dk78<2fuH7dlF3~|LQM2jd6#eq!txf>QyXq=x-Tk0d9sGFp_Uti* zf_r!$S{0bCvn{KDGwMEKgC&;Ce@3JKXHsS9t3xb#(U>RfZtoBal?YySFrs`SE7L9(xAV07Xt`!lpIT*nV}UZ|cGFnepgX?|2%LHC5$dwA{i@ ze4$CnkK47!VeM(gRGqK2lgc8Cwj9SYUw$_Y@vHtIti<`&ZJ@21J719p zxreOz597H#iju#~HI7XdZwWyH+4 z@E4bSxErrnb+Rb5RZ+7h?%abP4ekBsZectdg@_HqPOxDJrQWNsNIyK3Hlv-2{BRtNep0o5q#MQ0_Jp!Z7!hsAhO6VS2IUxVgG6V(G^WWs0V&doIV+gZEK6-2gofwA6tJO*&kngdX-br` zrDSk3je}f6cE^<}OnwBN|JCtKA!e|Y0b+?R3h6tfo#2cFfw{dv|W@&jdZ*xrx+gqdAbrSImpV~U#CoF<-5Q8^m3bFGHnX`WoMy_m^0hF}a zi+&9jbPWT{yW1Eu%G+LDn3VO|Se5pwnz zzV_ewel>S?>TiXd-yC}PA5?0z2L4k8WU*wSP7n&)r(tCfI;W9$n2dHr6!euP&Os)h zo_Bnj_s-Sd5qDG`z44@0iQtRaas9)TnC;Z0n)~L}8sF_Tdw^#c+lf@emWXl&(2AwZ zAchY&nmH_o*Yax#5Hh!|A&(@QVbp5N`ukMkc5gh$G5(3o(iT#q;1er8GPkfae?$zy z>j#?kmOuBpsQ`aGUR1iPW-1=4S93%NyZBo|XdA{-=X0|!x@uu$$6IWDI;5eS4l}zd z9FZRxIsg7hK!=OBRxu#Oz;csEJVOd1t;%jL;T5&reOloufrVb9)@mx_;+1lpOh3hd zky#Y$?^P|wTlCz*hL*_!a>wP;DiyZ=7KvkM6e772m5@G(mnLjLh~wDQhr$Olp)40K^t%xLtcl&*j~5X zK5_k>Iv2IWg|>@_axEVO&441xyaTyZf9NB4YPgF57$N3U{or~9`eARU7`JjF{w{ru z$Pd6W4O#frV8XTAK}cVLT~uMrBe&`g>}sVtZN$71F4hd8u7{r^t^%V9_)S1yh)1UL z7ZThK(bvRotWD(j?N%nr(xAO5rR_}PfSmtKwX8(IIhn^ zug2~r(%HAvFR5MGGoyfO&YG-2iEzWMt+kF_ScssXFvF_iTOasV)z|ES*Ontj7hzps z4#==NFgRa!savtbX|5zg(JM3)x#jFyuKKs76b0ddgxKSK$-jIKLqugww&6V0%X@|K z8=57btX15COe%@}6T&L(Xrd&GrAkRnZ3%|3C3&J1QXz{D(>8oFGg&)nP4Dj467XzA|rFqTPj?6p51~{9*!@>4wGsPAA zvRBfPfXN3QGZQJ6Visagb5f!?x(Nh*Z5}_ke;2^;=y%P0xl=UYADgQv=qyuF_=NNs z>wv7(_Wstf0GXNKKI!ZnxV(^jv&~LBY96Ng^9*w~9H$5;jWL}-@xvw)L%sC8rSY0{ z6~~7!KM*C!wwV`XuMKu4kEx!R5JUU z*Pe&6@0?%J=CEXg(C(i(7-B;NU{vpQW?o~n#DzS9y|^N{Ah6+b3($EUPE@I{C~dP< zSMV!vkEGVj&h4~9Q7?a~F;fX|wUZI*R);IfwrnZz0Sd0Q{ZgF@x zct>v8f+$9h%?f?bv?x{s6{1 z^zb$|c6Sw~WlV>w0yH%k*CPE^DQn+nf@2IU+jA*BP+pN4P%mQzMAB};P(s!F2NWNl zdqsR^Ca2xfc>ya=JuBm>?U;5F|@?Jp)u69hVntU{(y3)`}jY-3E1}2OJyh z$z7^@S{e}}CrMj9XN1QNBLYC3# zQqilc3aE}^{|Nuoj{qM2W!@DAgf=vVNxDWTxJ1Z1pSJc=BGcW`k*3X?uWe4so1_fZ zp-_04Dydi1MeG=HAg$@futVzm+zgoO%QB>5P>xTpoRrUPG5UB|RbdBLoK~_X%EsFx zyB@zHPgqxyJ_7GuC`~LVQx|dDo`CmFn>Rj)rnG4FLN0}!r3Ee9BhCe%i}N$-d1J*# zn1Sl4`kh@i4!oUrfsMr8N;u!QI@w>nx>_pt6i-uwtF^0_l)Z>c{XW8M-Zchx#q%*B z<)6x+He~$;cX=joQQnCSf;#qnRczQi#cc3Mg+&`~K(r?z@6o$~-=@1AwOykTpJ34( z+&TXcz2SZ(c0DsDs+G5hkw;A0zqPves`oK~kPf0%(W{|(K22ev>*uyoCn;-Ht5SvPfgbU0WjGcT(dT4*xAG{l*#rl7mO;#?# zz_UV+VncLu&xmQC5%>X`k?YYS{t4=SoDV9L+wH#}#~$1hTZ0%0kT^OP`dQ=Q=+U@^2(ta%`{z?&;m~N15vbKdy z>gxSD%&woWuLAuj*7P2wS3%-4Top4TnD0Zh-{#9KDu;U~81jY*u1k!f%rssGZ#2eu z>Dr1c%IL=7G01atMq&DmU+>f{6p(h(ghDZ`6u~uPtAfEhiV_!AF@gmcrp3)~2x)E? zo@lC2!jKf9Nt+@*$NF$o(o~dR3lM74Xd8`E(`NJEOqMBRrbd!xg!()kzFS6y8mZSR zjwbQ3vies=nJM{n5ou%*Pa=Z`x_tze{waz55Xrag7#8&Pw zDJTffOUh|9k0cSBz2j~9SVj!OWlJbk;rXuE9Qm5@ztD&Opd7|M-X2mf0=WEF$8;g= znvmB2K?0g8_M5zGC}(?7PzqMm{0=9}Sbj_>S^T0T#YV#ogMsNg=Kw+DTOff|iPdME zq{2Fgi9~E5Ryf4fvirq)8XBp%H+|t{-+bF045@AuNXuccOWtNGTK<6kSjs5)F{&gTXC9gY<+EWa{6$#^7@f!EKbSBV z+soDQ@#TT%tSw>zm7rYj(Xt0Fi_)TKBtx%+HipVAFE1UES8oE4K~s_g+jm;Tl?!}2 z#F2{ewn>aLdzhyRKHJe?evbS3cA|Rp){5K45u$+?Lo7<_`&G64Q^oQr3C9;F9fYS= zyUX!)SZI;Oq-1@3iD2Tdo>RGBB{yWMb^;ls(`_WznLRMvThg#<65xeuPhaBL3wW6* z@gCg9jw=1@8Yw`{CSg;ca!&D^GgI#d*z{R;?6V>&8SA``DfLuWjJ=K z7*Z+1Esi_Fv#sAD!i=|GiaMrvI&z@{H@i1jM5BuC%C-voe=e zw?2u;q_Xu)n+3tDv|stGDn~7ckWuw8WR5nfa6{x5Sq)n?po~drqeBAx zDOj12wlY@j+$R`S95VuYVd?t?@k))itpx{ps?-OZ=LWba6WC&w0eH!skqZdc|Ew5X zE{B?h$}?R@$Z^y_w8|!vv$ymXa=CX3S5u=++Os)mkZ`}AKerFGery?s)@g8m_2WtF zm2w=0@e9`;$Sx+W3Oq7gCI^SKA{K@RJtjL*gV) z=8b(a&OqMYTcrBX6c8x+r)5&` zPYqT;lK)HlOCJW#o=_FW40>iA(0hcCf>ueqkW`$ScghfB+rx>E{Rkrlhsp09n%5+_M2#aIB8Ey^oqH&(bnwm3IB88(_xtB({VXr$Jg6-x&`(OeDvjC= z@y;}j_O4gr=$~*)N<2AmUiK)TAr)c5YY4^F@E>+(*wZB5)dL|J*^ zYead*_Zs(z9}C#b`hTq%cKja|J*PP4qf)Z7A83~Ba_b%UlycQc#P>5vh3PnLH(iuz zb$u;{;4B=Er_8b-&mmU8lcxh`8Ee$5yN{GR9YXW9U0Yt%tGL6VjE~MOO@+$L_GA(r z5||9?hcM^Sme)THZzWK8>0CMRvL4utH_p(d=x{d5kQJ|;tOnXC=>f~}pOH{|e)q1H zV~t3&=Um>xsCJ=wbQ)~>%ky33YQUUb&7z(J73eCrK7P}~Obh;-*ON9-fzY+6_8RRe zhWVJONhEd_6`{A?Qo*Fz6A#~)`}Mg@4vTw|i#VQJ!{3ryUBVw}N0Qc@MZ(`InFz2o zKW`}?NTGa}QPcN)y8Vc`3VFvVs#&V6I>^dKLh>jjCzHt=wX}l6zcss6t+6g-?}RG$ zaE@aw5mOKU?@B;G?*Huyfc)3k14%s;`7X`*$Se~ibL*fcX(B~0*Q~LY0wMRkl#0KK zUMPPRahs&sftaMl^z^IP5zloX5zpcOXZHW8I*w9NO4S=5VaBR9Jx9O6mJpq#dgZ~@ z2S}=ZjT;z#no2v!K$6Fle>)P0lJ_6tqeiu>Jq-Myrr8Z z_yr{F_zDKTG=40-edrJ0>4V4N<9pp5Db?BLK07}h!qsare03kSd7V1b;-i( zLc)|h<-pcfhoJsw6a_ZMe=5yVOg6+V8SrqzTH&n#WJu}gtJ4=K&?+4kjQPrn2Hzt( zRGMcYqOiHJoTOM;sHRKW4DrQVd7#K#D z05$L5aP;@EsxC*=dd81Bf3jNlXKfay590sgd_~8lZ`~)8r*rQ+ak^QKG(3p2dzAdyoKNw`00r!qq;{l)#K&hhxt3NxcI1B_x)|au6Rl> zt5dkTg*54)(F2pb@s)PO4SjjqjGlwBc+gy;5jd&rLniT&l9X@^NkZmIi>hNV_qUg9 zg45{!0tG^5ijWLOLHGSJG%tjy9VKIt4o@^yCp31FV@4cVSseEo%@p10Q5;F z4turTIsUJ<*k%+__w z>-a(FWaco(UD}Q;qJ64T1%ZDJdos-4i@`OOGBd;bU<|oAZ=yx&%KFS z=L`}XednqPx}uE;_B>ma6Mg5YOfF>EKE3ZztC1(kk!GZ5&e#VsK8nA*1{O${uSG5o zZzw1G)zM2!C51K7t^N<>p@YYulVT9q`+QNi_A7ivy_#zdiq*|HQw^og>KPDXX#|+u z#i8T&mt~tA?o~!L?2pSMcWLyYu9hq^k2TX=Xoz+LuVWi>45%q{E!hqGsR`SxsG6IY zaL7WObUajVo=+V!(^YiSbIve#Cif@(wBGh?g7$v%^~YfiG2^H2#K__h#)R2n!;gC+|hgA(%*i>h2Ei1Vv7i1WCM* zJyhqj&XhN?f)nK6c$}IN?`0(e64y;|_&D<539+9T)K324Jz~7Eft0%R@TAPBT&~@s z=yYu&6xZ@0(`1$U?Q_@DH>+{N7V}hy#A5xOqbP;VMn|uy3f~U8_?wnTX2j93A(Zxq znU_?W%*;)8{PR6ewXWKI!RKE=nx$>*nZ~tR{0c@uW}&mLJw5a#mGpP+v~Gy59>p1R zAC3Vw_rOh5p0i<8td&!;61tvHXDN>=O!@a!2?$>nT8Tk3J=m|hNX??|TnWQwT*=q~ zMooV-u_RgfL|Ky+7{*|6^tm#q&F}z2!syII${WoI7`+S@Dgfo<5rp{t5nC~b!_(!2 zl>XuecW?B|%Q(%56oqC|cwLuO5C=Qe2IyhAE(pSzE~QC5vXO~-HZq`NpSw)4WEH;c zE3RbDk9@gP_MGY7$TqF)MZ~QneAiJ!G-QJdiCRER9PZbzNIS}vjp+?0SE*oZ?!YP8 z9PZI+ka)lHTbnC8nGCRYCqV%#cV*@#vU-YTVU} z+b3ypAs8u^mh$@ut4_r|jetYX}jA zg0zs<`OY1pQ5yr-`BAo1d(1u79RbEqTaob(i?<1Vl%j52Cxk2x*T~xY+V5p*iLBlY z=+%p}%@P*m+n4PJq420`-8Zn6bo~Fkn;9v* zXO1nY8_{DDU!0~l@12W_)7Dk4qzrwya$a%_k4}lwOZ`u!(phF~e zx^_{JZQ#Xdna|s4xwo(O*rX*=rn2(=$S*Qa{PC>x>ISirqEGSD%9r40iijLGtT7(imwzF>Fly>IRk^o zkT0(;;(ZoIE#J=RZOShqT&l&-6x~t%mSo-osL=u4bP)Z1C#Nmpn7F z@(y^98m7FOhSs~{*V-k|Y*^s!T|D&t~B4gOJG3|3F_ zAUi1&YzNcjt_Me2#=s<8-u4?G78K%ZIoE*;w?7!pb0oFLF>7$J(2)MEk1_IMHDGgi z9Uqs*(J-IfLT(nGY%@V3;*{9Xjw(;_N6enJ!>omrLf=j?oC;mqCOjG!E8uT6Scbt& zEbnTfRzl9okG{nQQuj3wn_?rjt;xbXX*^zdmCvCUf`*WD=tQq&i(wS%7wANM-5D~) z+huKuoc-?>g?h#J@2`=MBk07AJ1>`MzwgU8D1`p;F~3GUr~d1}cYlgPor>=tf8hur zEdWMA^xwglI@14HVe|VcB40P(!y$Ac-~IQrZ_oc-`+GYt2x&q$s+7{iJEfLx^m%j@ z${CEWc+++*NHN5y(DO$!dV>}N4&Zxc$lN1EQP(RB>||oV>e9v;_yQ z$1^H9h4{h%Qd6_8Z`yifGeG}YYnC${w&AEwi_^wqWI)FA{0V}lgYFirFOxPkqn}7D z$7j3Il0p?VI{x7&DCfW!?M!YSL`(N>uy@hczZhd4zbzJ=->dPFed9;op2+LEJQ$-c zv~)>4?$LUgc?rq=(_XYQpGz7QyU?gPtyy!hLF?BW)$GT1FVs{!nLaDcNt!Ms--YoY z9oJg>1c(bwP0oxD3(pv1^{MKk(+29$-hE?x2;G1|AZQ(ZO@GHKgQUQPIUsH+>08Tt z3q_4EC!#9dw0|X{f{R>72zCxW&V;&+sK5jvYAXf9g%C8?kg_DLX|&|DL6Z7@kFG9c zo;u&)>+{6-gG$r(;etn6i7r~2TB}8j{&(Y|Cq$n3mWU5(hpwSt2NF|HJ(F#c_ri|U zq;0m_8wb-r4`#nxL6=mCiFZ&;49KY_DOoGSgkkY@lKTyi&;e^2Yo%sLkLd2%3H)2;O1 zNz!S0QyYm9eoGOaI?1AzXN9(DLQE(o`dIY@1!`jdhg5w`2djjfKWy5s&#S+1hW$Mr zn%!=%Z-U#*BzYvhZm>8-4k2JLCS_eeWwg*&MEo&*@jA}(fN_-l7 zybQyrU=Lz9H8m62+_{d+Q?~rN2=CJ)^|kWPRI*kFb3PhyB#@lqqRbPzxOAr8T(N&E zpcD)Y^#Ym@3f%KVZQKu24WIuAti0%I7La@s4METpSTYP*6F(8wRA%jr>q)ks>YCJnSSWzCU&90&sH zBoHVYFqPPgd#EiGj2T%q=$X*ep++jYq)kM0>R%0we4q zwGG$euN|=9!1JH|wlqxlG(_0;+6RbBK{Cp$N5>b2r{@2M7S;Ky4D;8%#T z2j!VUD?Lh%)+@A050 z$sZ)z)HnVs2T^7bTRa-L@c~6-AB*CYYrR2ExawGZ`6BPn(I({isJfbw4*|nzq%c|! zp{Bm$&l+E+q&eWK6YUOe_#opf*v7Cl@LGbM)LcN3rhip5lnNhDQq>}xo5>P@lFl}g z90`uRVZf8zeF%MOn{fqcvWC6TarUAYyVsWAS`Q(L-Ur&9*(Kh?%646C!h`04XeJ`cpM+^d)-h$C@2?$ai}fb%wO{4njODnzzm?#{CZ|3M3^{3m!M8T@P9VXhC-jmDi~2|jLLkl2P!Az>fWz( z9GqW(@oBSyzCLCqmEGNPY>8IXdkn#=7w{m(1V(sUx;(~PM){>H< z9QFw}LVu%-9*U#VXI##_BKqEqcpvhN0AYvXMK5KhIovsFsIe|`%kSEa6wKd$o zftq$Y?}nT6xK4#`{Sgc(m8T}clr`f+?C@VHxUzJ& z#QF&2LTj<{wj*_h_aTafM=VYuHA+xm^yTRV|UOdt5#f+{m+ zGEBpv;QYEuL`zi(XUgdfi`5*`O(NjdDd+>0V$JsX^;Qtor|LLT__o0nkg7m{MdOdq zQi}&0-w*2)q5fMA*6qn84Ie-!BQ=f0i7*v*>dflohBgOjGcb=R8?DO#&f(VjN`Y!; zW-)qNgheXyXL|tM;&ed9vnu93@Dp%dx)zN`ae7e8&<;qqya?lPH>`-`Wh!NDQ#vPG z3dy!$6GU)NMhzx+K^dYm%2E+K+SK$80HJUU++;64baL|^^fmQ@bv)hN)#3)vD6h4C zznSQT{;B;n-F>QM3o2$&H`hzRa}v0$yO%bZ<3?P)+2Q-tT;7~)w<%HBOMMcSC>i%- zFN2MiVt3Xc@i^tbT`y)viui)wXA6k}(Ka@0icThqBB{U}qBX!gy5&#ln=J+Dno|xb zVJ`suSuXMd2>ULW0=s%VJ4D{4pOi*2kZfM7ju3A>&7N0vIukiJ}q~ZH^H8!$=RGk|65_tP;7Mz&s z=Hq?4JbCZ_{9LQQ!n!8v=Ki{Ur+SOo%F52p^nO}e^7Zk3IK6s*FTN^k>ge_TaC#*0 ze0UNiJv!3=_wAEZoSJb*_Wqp6gi5iI5*2Za z*Ok6Dz~_g0dQDB#Djcd3F9Jrl;KXoU{eY|NIZx|zvV?MbONuL5 zlJAJr`9;Nsp7N{3IzNo3je)iW0TP_R#|#M*HvZnNkz=95%{7&nts388iCV3=*%>5W zRXProkvl1v%LbFx8@n@)6ErNXYJdt%<`MUNI^z352aHH<8a<~$PDQ;<5MB(yZdq}# z3BfT%z%>`M%3BsFl>c`2nf`MrqXMEsPFgm9zVV*P)a1J-s2D|l5%9@(m)m}XQnlW2 z=F+E@?lU2OJvBt0s=#)kCD*N-P~9z3-PLADM>xz*nhT4F_;IeG!<0H*V5M$o#Tkuh zWsPD}1!UYmLcSp)!2@f&KIz->7{=t1{&DrfnE)R`Qgk|-J(24R!r)Yph?vbRLFwWe zih59behL(f!nEI+gVOvHFAXD`M1t3~`1ZR-TOAwdZ$Kc0GfM}w$9T&76Smv_QlBHY zdW-OEOcp^pPgT9m8P#FpOwv?15->K>nD>navo0FUH&fbl1-he6V=2^r*4jn}%wIP` zx@$96kags1bCcOEGgzDLJ3NS&(9ORN4%BDs1VTXT#~rPL#YIqy8a(i*L=;K5B~}Ic zv`Wyva5;X!QnyNXDQUPn!Qm^r%cxeOXsO0Opv10Q zOozmy$p$9kRbna&b$>be-P65cfPfZ>_1+S~3JCMfiNf>BfqN?B-Rp~UG>pP6K4-pp zhUA{HpqQFPJLoW|>qUy~t@DBEeo`TC^I=KhG&{Sg$|vR{k?>VY)%Ep!==GRtM)tB% zVU~9>{$%rdo-z+jbthl!$Hh zUJ*~q4o8l%A(~QU&ryZkR}0Sab{kMIGG8fq(kxk2z$>TVl9qAG*4pWmHdu#60e^2G zn|J(R*`ZG)ev6DFZFx~N0?JT4$EZo-dJ0rtl~1sI&!AP!%cL4X--(@`W z7YXdPW*nn%zkBpfUc=EPDr1_huAF(p+GCkMeLyG&mJe&qo*(|@;7-|9y`f-otN0sX zkm=n|eA9rMf-DLg<8z&qZa|yBMn#wWV@$eFHwt(Cl47fc4LW6Y^mff}BiN3bKK+YR ziStz6U)nBD=JO-SQ`m7M-~LPUpWdvsbAJ^OTP%|NO+f9vFHB;AZ8dva`9i0F+-|30 z7WR~{BAnh!!iXw6rqX2h^RGQ@>zPn9bq~Lqt=oWAIDO=RA*hyWMr`;-bEXE1?35;qf3WJ zC;6JhZK*th1LiFCSiVCubS22W{{id`CTCku+YBk&+ZQK*?Sq2U8~{l+sVUT)@(2M~ zeU7P!JBm4%81ey$2*snm;ut@TCc;IWwQ=lDnVY%UL4Px`mfn_hyNeT z^VA^b?^E2;cQx(k)X{$Acsubu@cC{3cV@e%fd0<6+gAX=)r>~P)2lSywPovr#y0zY z>K~^_H6~8H5e6RowHX+v;93r{m6m#5s8<79SFS(L+n?x-!@N_4M|GleX0ksEpgU5k zZ+G4kOLsYE1YfYbM#6nY$PXx2s8Z(-*92fbhY9qJbyOrD#XNilsTa@%O}2)x^=Dd; z9G1FM@+gDfV@|pTM6rg0w3#N#WKqW`FQ0toezu~BT}M5ni`_*e14>uagwL|Hu|&^# zzWjf}TKDxii7&vz{L)UuqUU6%#`A#Vw_c?;rhn3=84CNf-4Q=lbjuEQfh8&WrS!6a zHKP_?%6e8LpPcLKi_$1oN@V zCgysfE9Po7P-2BAzNpF{rl_FA?yBX$VKro--#TqRjEoEC z$l_8mTMEfnA^d1>-SdiG>Z_-E9zF|jA@DR1`vMEN7zi=typlE89_Kimvl>k+muuHC z$@9v?A^6&=Lms1|07kHDIWjvqW?9m9XfciQ&7SB8ZfX1$S#0FL8AX41w&5;0QODQ< zZ1z7*&uRbj+luEsA96k}`OF|MKnviHK|7Y>AERly+259rX9N;1-+7*pf{>@SR)-1XhF-m_s=MOoRnIe*UgEQwvE^d|HYc0c&~Kz1DdI+3 zQAd8EYjBCwRkdhxQV)SJv{=5D*2vUU70x}T-M?W#BMk06;6{w20{ev*v>_!seP#E{ zc`3#H9S=y)H}t6gSq;ASjawZ)PEGqhlkV7F>Kl4=w%wQ2>xTa%^5i{EyQTR~JfYCe zES0;|g?k`e$xJlyhGodkci835=14~Ze?njV?94|4@#0(Kk$`OV&iM^J$|T)ZgMM$# z#cAg`#;lVF+;0MUyW((*S7VRm$Jdt6`*{R^$Isfci+QI@%Lb=h9jF4xRqLSm`r!Fz zs>y9AZXysgX`f|;W_%^I@oEv%dcDE>)}7>9c|#pP^)N#Ok#jmwlwgWW zk}B-l8Tgftg1&GSK}EJ{9J`FwrYCF~j)|HBiNWF%RwK#*;}i^>m87jL<&#)wc{oDx z$m?!`Et(%SgQ}H~O9JCR^mwnD;^|93+{z|>KR-Xq5Sdaf76Ml!mp>Y0+^6sR-$_<8 z;k(R3_H!IQeLHf|I~RnrhG!m!GFMieAp-Hczf(V7HxC2Z-~XYo;FGV9j+xLvfn|X- z8Td|&2}^0;_%>3uIUQR32;d%h4pEwb!v&EC+X%HwIR_=d$@y_*`P)dRxQAjrPXMV6 zzks#N>+KYGmj3LAG%NEM$R}cVjR^^ki53dOuvp^&(~i<^Jh~3;kL0=={8f;pkvYis zEw`G8L-%|&Sq-kmi~LRxMVN!JKJBmwn&D3D!Z)pbGstZguxy&mb!B$dTJ|Z)ku-KP zcD0p|L?;=R88!fc;==uSNgmuS&_dF3s#QQsYomn_kx1LeMfZ>IgAgn<+?*0mE}Rz&v@d4`%&)b~f@ZTwZjQb)T2V&hUW)$k z!!8$&(35@@*FSGK?$V!<0`YuYca;JMuiE>!uza-m7)E^p zw2Kx8gB5FmzY5QYJhl=3;lGZlnSgr=J^o{eiv5SMc>16pq-r<5bG(P>+O1|Kg_^s} z>#3y~TW0Jf09)*V+g}8wZf)<8~%X92SKz+tjI?%J*sW z@tc|sabFttFw5_9cW0mb2O{+LD#n*Id*<==IiOa+3z$7uStWb8484(g{e*tt%_6JZ z01^I92mjSGuT_RhHIg?a8h#nNY-P0%cjh;^cW%;|qkiLC#Q^1b7;P7M9W{t$Qj z?^ycLe;tT?zgt4m*GRb>F0-^N!T(uq({Y>)>Awd}DV75F->xyKhyVYLh}@{`%U2c! zELRqanM5_4BjQ{ituJrIUV2QWS20V)+bCaA6#1%^rU-b1hQP^ zf9@?5Q0q`w&BDMQpgTJRuxP3j{#N0q`g%Km+xlb^{JK3qZ!f<4dVRB->eBD=eLs93 z`+oqtKt#XupX@(=dwza)_T67!|M~sZtH1N}pT4Ww@&^C))7f|I)!!3|oS%RB>)^2N zq4@QA^J~~G&;PIGMoB-yp{J#2D}|X_z`|j*#j!WUFn$&WC8#>4Mp_6rwQ6rNy=zmq|LV#6K!_ z_YeI`lB^2426Hn=b;%%kU>8EYGlj<%C1wX{c}r+Qe$3N^4qRXkzBpK zzWl$Y{4z028TTzebW$-PnlmS@>pN3Laa`H^r)#^LB|V_{uC4mDy{CG?1QbHtm?L$mh~7=#~A|$OXx>jixMzMSf?5=wEIa z%|#Wd&Zw!{U}U(Y@`mO3f${7*FD~|aHbZ3u+I>xxo#rt*yYFl%gEnk6WTO zfi!TVQYPL|rw+Ge{=4GoWf1^*x&Y9@^`{Xs<5?V-+qLcD! z&=fQFpZ@bws7o3^oN zK$e$u%?MSFLzN(`l5Jx)iYt-}))fO%Nxr#!86^3Xoar&118+gEHzhFDXV=zSS@KbQ z12`;1G$?pxq!L87*~V{9XCtU49W2&4Zs*)3o~!K1W^j>9TkAba{?9YmPG(ab}`HJH}oj%Rs#uA&?PHyqR5iTiIRyd zJHe&Uh+=UAmmpn%m3RRmy_dgPK8ju9Lmx=v_ zUd&BXHGc>XHBo%-ipa1V=K;m1$Y=owXz|S?yBoUu6;W2_jZh^+b!l~C(}n}=PlQKEK` zyC>qC3iwQqd#-Kl{(Dif-dcWA*n|iVC*4Cv4jy8Qe2ZVTigKopmg3b~`t}Ho1l)bM z{E$Inxzd5a?knvPIqxT!hYf}?qaYghJ;ap#6@7ATHFB&}!#c6R}hmp|8 z9+J=N>x=%^!JGlGV!fF9?}X+hmok=Dp`Sap+z4?)&VnesbII58iDCA+oo)=}s$142~lt$whZW`C>RB;Ytiw(E_tKzHDDq7xrn zGgMw54nqVrsf@{K7$Se^zDII~#R?1(iRfMuTFQN3-P+_7Y6%VLQ{MyfzB}2GlS?3- ztKQH#n%P=Rfn2jX%|=whB(!TyKvbGl=bJY1bjLjP#P`-u7GvPGH(%6jWR;geFMi6_jN}LI zW|sB6`6(7$-`gMa=%-K|@}1tpbxI!BeM*L0sGrO;6Hkc7;{6HGyVu?~JR{CUM6W~+ z2?`zfKpntWSHh_zt_W_1*9=#r-eYI+VX~$=8DRCg9MM;A zzjB;1Zuwg{m*Dz_vJzoTZ>#Y@+sGmhm9Ut#&I%#J=oPpS*@jjXv&I)!kU+L%tRkXj zCOk!nyr^(A_^SUi9*|_0 za_0`)_|Yop+8u;hFW0@(y&wk$j;MDOM@a&sRP0vsHp`e+3uymDx*t6F;CXlCYbGUK zv#Wroeav_TXM499z%B(AG}=!_XW8XdoywlBTjF?}zINAer5J25K$Zn|m{+?6$*29S zE%MqD?}+LIy}kQ?Gta`=7m@<22lRTF7v3FKft1*;Mq2bNiA#=*@!!Yz8m1Mcr9fu>b5|y z@mHF&VZ~a>_A8nO&r@hcE$JQUW3uDy5QWJuH{L=m!^TCCwP(MgMX{vW&3o~+Sj)H7 ztELf6f9wU-Qhn1_H$;pB*L1@w#9C{Qb*pUh@z8=|`L(%3f^xV&Y4(wK=BO!9K@-UT z-Ekpo7+5ds7f*lx2RtT3G~^FYA;VP8P#}R!)fZ+-i$YsjQ75Ysu<=8TeK23EG!{-) zDc`^Uo7Nl0rJ|CoSk18J!C7jja0#6n=>|n6`#60E-!HeKXiN5z-SX5nBfW1}FH6BC z075Mbk=>Bo{80rVZ@{`Uk~E6s5J{&P9bqq#qNH`rt2Ji8H}npN8?^6$hURyu@=gGp zNb^M_;m}n+Ft6@QcG+wAZ%nnG1jj|XnhZBbykqnl!3hn!MiQOB@f4BX3@hh*EvYh^ zigglGQnN;KsYUq**x?ril-Ip@!UMfxGb#kgt5Iw#`RE(xy3z&kF*iUx)wE&bw2}@NNbf|HR<^f zn0Uf;by1r}4G;6Wh>R1_fTcpiQ(oN(pj}b0f=E5mw#uMpPxM6uG4h z2P}bxK{ck{h+~ypSgdO1-%4B3@6W>fvBU%uGP1>X4tjq;CfYgaQIxG} zX{ZDjANDaTTFGx8LO!JEQ0_+k8C57#rTCJMa`jHtWVd;H1&-jH?AK(B4}HjGekxu4 zh94X?#xhpyj#9*ICW~jbr*J?G{==Egq5U1*wcvh@?^FTC0DyPx|6Cu$vwM@2P;jpo z;i6!T!j5B&Njf$(iKs{+M7^Zh&Cp&+v5%d-sM=OM8!4R*-C2cLe}yr*$XbcD3En}Ft2ZK-9B0`Hx|8kEV^urQ z$6!uQ_A7|ydR(h=f7|EepK>4xD8PP&SbseX@47cX1ef_F)63bi-9IRE2-gmLd`Fx8 zNzc*KHf1qNUcF;9A5ZihOrEUvBG`J3P|8qJvAbeRGOPn0F{!=gb^7NW77nepnk-ko zi!5uX+(0g+fXLophC@FalipHZ=)1SC+&m>`waA@x9QwilX)1{n0^p0V&Kt&j-lb1e z2u3t9f#O>6P0Q3$vm12e7uuo{J7&I+b1Qo}%eCnSGsk-s)bH<9UGb#HdUNRqGVk`d(RwXE5K1p&F=A*?k-qy_CH$8^C`eZAdy8eq>&@V<|mGxD>dS<{ODh^wDb#nz(H|2&hK zsLh^$Fg^E{m~PQ?GD1@t=?hWDhSh~p(rl~F!lId(3Uir|ww1J(xEBX) ziEb(>xzh{`8id|K*XDOdXrc9l^@d1MGOuxR2^e67xEt-q zVDe6s=xVgJzRW5EFa!>+iqKliwT4q=LJG81qNIvv+5#T4b}Rs7-SCn&yx6)d1gs^4 znTU$;oRzgutjY{i`CW}i4agM~JBez@{F;2GIe<;BIBVpuU{69UF>>TW7qe*8FcjSY z*o0m+ID)S31{8xTCu?!b8c3#g@Dn3ASbjzO9a~OOe(Qz{z+fw=lqrKY?~_b_flw*= zy6Wyst<_BVd2{+>O82!EIVORUq;mgT zEm&=nR%Me^UGv%X;yzoAs*f6EFouL4e|q)89MYT^bZ{ z6I_7*@l+S^QW5H*y87$0AYTo}3D`%fk1z<6R-z?Fe$b~hQw7fUhcVaG;1)`jHEifI z>Z_~x7M7LSd_VVL)gg&HgOCiOLF|P&@PON379PAVaNDIJ`Cl5K3mPAG4m5*;5T{S& z2~*hR}2AhNCPT~&~^{(i6P zS-9znmp>1Fidy6gaxF@iW1zQ38s~!Y+Ut(Gjj|0ClS5X~ohH2DHQG~DIilb_)_?YE zO~(|p?&J-F!Zu8#qEJ+Z@wz)NVuLN+{#n z+*x5QfV&@-<9YD3+M4+#_o3{yl^N5mM7#(?%LnPyb>u@2O5nUrMcQt{qzjA1)@=o7rvhoyHGLkT}x z7KldKr6NlzS>6r(Y==wE7v$0~)`pHYwWx&YdqG4+SiNB-YiQ9Ql&KwhZm!9h@$Hqg z9yL0v94jkw-q@C7#z(CW%*k8mEdH8H&<{`G{gVr*>>Ta9wwFQTzTwgl59rmO(ku7? zH)6aq|2=y0I|zc2IZf-vKhUxv)Fb-9?$E)y_({#Qj7OEKcmnf*owdUKBJVHj;@$NR z?5pp3=Z~z%l#cp)UGv9H>Ub~xK)?LJ6&|54t|wi0%jj#Km8=*!i1*MbJU66`rzY;m5v6BLc^_5q6MSj6KGg!nhU_TWj69HbFtWmaRb;(dmr%PR3`dr`ary4 zAJUlN9uOdLAJV?R?|}t-p-rDd_3(8m=?mB6|GEx!f9>}ze0_b<|02kx4QtT%tp84E zZbDfheT^+QLfnwEfd+G4cAVsE`NZ((T#q_bq0TD+;@SsJvjfZA$YD9*(B)(q?5JtJ z+n+RuE7+=g>B;-s5bl^B*$SKSuXY*(>Nf*Z++7v4Ht#jDP1mSCaW;Nydk?r7tggW-zol(mzOqV501Bjue@POoBp^QzO&hsea6mK7`es8fF=kFhy|Ii=*_n` z_+V@(FxAnhQ7=9OWvsmzQP?XJjvfuU>{GJ3y63R_Khg%_5FOx=+>C)gno^~UEE8>& zQU!Ir_l2P=rT^Bzh>gqAnN~{#}F7tz0v9CqOJ6sVq_x(26Ebu~Z~*C^*}CS5UV z$KF(G`_RRW@#SyceLaw56WT~68oG}1Aim}Lptx#7{p%ZXXUX)`{DB!68@|UAdKGAu z4M7}-_2!&Qyhq<)9|mxWv_0@?Ll`uPJm@+^FJudseZvYyB_x9Y;5sai zA(@Gik+!Os7R|KTF38LM!H@ig=@|<(sigp1@a@jPd1ceg5cn>bXElD8L7v$tN>P6m zUzv9i?CsiAzIV-H%}#eRW@L6AzJ&p~?@UkIbrzdrL+Pze$^^L1_pGdoKqvHx^(Tgh zpBV7UC%>|Jcf+e0H|k23^C5JX;T6TVNyDH-U#qTnV1#aKhK1^Q!tk}gHE12oG`m4o z^suAedwJMgl9LmQ`ow4zQmE|ZVf^f;9*7i<#|x@drky|(?d{BvI47bZuUQEryx*Qh zwv411OOpS{piXLh>mp6iQrn*;Ha_b!iW{C4$B6#km6$>wO`WzJ_|`rKu??VOt%Oy!Q$)TZ?s$?*wR?li>z;XCo+ng&iseOQ!Ca zX@hlb*C}*Z+;s{b6I-7n+fpy28NI0p)&i*KLW@N0YK;LDr10#&W`wJKm?`u@v5GZ3+nL%}=&9eJ>o6JZmVEAR*fkDfzW1j?uiOFc zAT{p&t*5#I{4W%{5UQ89%ICBY6(d(4UWBt)pg9~E3thE?NnsGMxfYaFOWPr~3X1?% zE*XR={lXq7T(EY))rS`|!WV2|{{^pxJ#+{~rDwgy_5;8~E-(kf_l?aosz3Y@x;E01 z{rCKjMn8pr{b2mlKkR&p*I9rljKSeF|8OPp>zZW?Tt4{&##GR6vOpa2SQ<0w+;IPd zCaiX&oLfa(3Kdj6Sx0{*6cP@(3!yk$uhSVMJz|;Vj3^6|&79g=8?SSGLeP?`!qmABLQ@I#Ye!BJ#IdIHh-R0m= za6Y`)@4*(1h#-*B_Z46vXsvaI-dq!wt9TJeq)9~IABEKOpFY?nb*Ul-$P5d`J#aZV zs}X5QEH-7${VHXU2|bi6t~3WBIs@)e49b}{%z5lx$OIrY|A{C(c?$?=ov2JfW{z|P zM#Xv(qP%l$8K^TWn68Ezb;RNF!F|pf7k7kKak#u~g)H1(=my?%bPrxBeE zo{e+t&=_mpKzw~1yTsF~!{YQ{&h$c=l(UqL-}rubh>#&j4VYUXV4d+$vAYC{R=dJF zDZRJ}Mq4C*FPfIj&ciSTb&0ZwMq|?UXD9qGs#=LL$z8 z|LOC}U%3SkTCxtXzhU}!aP+ae)IMLq@!1;PLv47I&ua<^dE1)FnLqMGk`Ci4>HSIa zmO#q-0Xqx}W*w`>_wJy5VaJjuXZQ6nW*6_+W1s!B&Yke_XE(ceUl04tVV4tL3%2;^ z4>GFS;?r|Vmb9Lm8(xW$XUSLmI~Ir{xYtK+c`YIBNyjZUSF%=36Sl>;qITMnI#d_6 zfq~92=c^LcJlx{5dkhHx0g?z}hb^PfX4O&zV-$&NZJL>TgXqK;IjaDLsIjv@CxNz@ z-7}_kBN5ZPtGndVZbvV8X|Hq+t8iz4Yv zn(>*f7oZ*G+lrPv^OYt)H!p>y(3gf0TS$d#s|@icx!T%UbeDX}*1LWQ)BvdP6{ZcL zk!66;4P&I-l4`@YLz!^~woqSmOPbx>(I%IWN=Ox7ay@|*>i*EqJY7rApr_rOr+5WO zSCDRO)o!SL*lE2iqGF-ezP%+VyUy>u{&uK%nB@eMHrhpx7(au$6NHyqi>A=MedRh@ z*|^-Ko!?tAZIFDs;f2pZ1_KC^BvK}RC%P&x8o0eONP29>^@Tv?)SbMPUGu8sY!jq-%6IQ zOTjiWbHnayEP;ZZm6fj&(^(R_n9QXh?M@1sV&t7Iscv+5i`P_Ya>}kJd;QImR3%U>zt+*MzE=da% zuDdF*l5v*CO|og)1o%i_xo#PW60tYsT@wAy1+WB({NonSS0*uYq^DjEH{l$5F5bLM zHDi-=8^{2-5A5^M@YCf6A+)35gC3KBZeNiCRsBNTk{=!=b0 zIe4iJc)aOI4GGh(4eF$Yu}f;wOWYQ6NiHiQ8naH5Xu1ktX@+LKhh9Kp6y9r`q(3&Z zZKCPz+5-*K#Z0(N%Ju3$kZlX0GnbV~_0CYm);uF6Yt~Gw!`VO|`2f`^GT+Sa5Wbv} z<7S_9ZyP>yEfbR8wcDQRyq;#6O`R{O5MKwH9q&H(G3F-k;J&tFT7-k5Q)08^|3_cV z!1(`fvhl_nxgZy|m8M6*daAr1OC+jCqm6`rwfvSAreQCw2tz#|D))Buwbb){w-FL` z2A2STa>KSKvpqR-a#@|EQnm5bp3HEX4+Jnz;BNtx&;B!{E>2MdAIqY+pKsc@8Uw9% zcb}cg*q!eW0A(bs&b3FStC6+vgAeUPg*L${dC3!wG}3-8@~JD>ebIpt_KMJNLq&rA z!$()nA^LsA3nsTxK_Ws5$r6$5pdg|Wits8c+8iMQfH*?3J^Vr*z>3`$Ef3m6AQ!w! zb7BmYKtOhii-Kn<1rZZwopjpc7xx@?WAx97D2bylBo}(_Sg}_hUN}&-(dylPo$esn zxPeG(Zg68#5g;zEE-liGJ$-O%;Abo+GTX3{q9hBR8D~cMUF&UT+ch+hbhT%hsBT%K zAjY(=_>aLT6OO>uaeB%qrloXiiz?eHiMBN;24w4+D_d2NiLz`fuC`Ag zf1fW~ZJa&H*)1!cNWPxaCfjhuGSxQh39b1YCI}5)<)X|}f>sGoBUsg1iVdP9d19W@;xF+6;U_OIEy~a=4=BA-5G0LFRf}?zge=BEa6}zA^>buPs)Z zQ|lahTEhxhgxvKS?NlRX zZ`f9PzF*^ThMKs@&)bSQ(X6Sywtz}T6E1B@xQ6>DILJwvJkf zc~AhG1GjZomnO|G0JLA6^h933y?YQgh-ZZL_X!kZ>ojXC{;!sG;L;94J~rV?&&cSP zNxM6KMrffnqFrw=Lq0V}siY>Sm{5Bk0stFag_wPbd9Iy$MQbgU8y+vE19P;OyS;G9 zaoEa$7h6!Os@w!wab`gn3;pn%m9x1E)~xLU4jWjv`iwK;SMw?0R2gC@ljpcyZj z9WRDWAiq~G`}v;iRD|-R;@1NTU*n0r(~Dly@9pWuu2TzL-7+MwO}xl8wQ3=`jA=^Q zs#lJhK$%2KR`*I#lc?UWTy<)x>Vz`Y^b*x+%Sddauu0{qlWUr$mZhear1r{8lP5e{ zj=EoBnoRRyN>L{-?6eZp( znoRLtrKS_hOb;kAomOr+rL;7)taMUI=>a8nDRY}tMtZ2>GHG|FmX4aD(1c+mmW}Qg zgr-a}S~9v@44ObRQZ71J2%124xK#9B>eNGP(}#OIrxkrpF8J(~bEY{Hf97J(#KV_b z24Z_bBX~v}hCAEUlO`A>;4fg`5wTB@GViMQIacnnj3eL3-_~i-`>l0+dCZl+i1nD?R}pZ6Eu}X7yI;7L$ezrzHBhh z{cMbB1$J~@kV{uzSkPu|ql4ZlwzN|&JXb!#Y&_NIjrza1?5av2(Z`x4?dZuO$cFXHbfh{Et_8eHz`ah=2xXf)}m_$ zN43AUUJMu3H9ky;C9`}s!k?g$n|DmgV>A|y>3>5z4jm!uMy$0J0t;`lD3P-wbnR_w z=ZH2DEr@F)?l{%4Rdh*_me;MkBDAPC)XQ?*>o6W68q6cGnF0^ehSU!Kcp>}(F9h$b z_B2l;Wm6Mcj2+lkIcth7_!+S+`-3o;sE~K2sDa&=> zbKO_AxlHP|&@jn=WZ5*+WWRk`P2V>2la2u)Rq_6KNMFe z#nXl++;fss#pe14AKFQ%XZF6)f?f~l;Kk{a=&evHP^w!OtqmGtww<$9I!xyT@DD=x z{IgiUl1WL|(-+Nr&#u&_Eon7x7|j8U!u&O9-FTG)LP53OIZ1k7ytPbOu8A#Ks0@K9 zjm#XshE8aN7?|~X?ng`C@uB+tM$Je`vkkAzJt7sYa0yN53j2fecpAG|08qhTfKc9V zdGb{5EoUm=DF-kh(I`nydII^xK%y{ed=1_0kVJU=2C;Xw;l+T#vp{46%aINXX^ZycV-KyJ~&W5N%5oOeLA05yI9ueK_;;zK;0d3xPi>y547{ zzN7g)U28_UC|IK~@2=?*4hb){PJo3-Ijak?9Tws*y`)4Jx72ihFrb7#EQp0H)Y}6U zV&JpTzdsm7l6o^=>f;E$l;@x+r3@#V_Tv;>3-k)KMR!v30$nm;|#?~xelEAJgW;D4oPw!vBn z^Gi?yH!L&tek%)!hQ+j?FgLs|82O_MtIybK#WMBZq?I<10Pl~nw+n6m(f-$e5AjdT za>8F76N`PIJ$Th>p>`fHJ*1)+1jf7|uYv`os0bkl7cGh&5ov0oy^T$5u1Xt5-iT|x zHEqFWvb$@^YF%nm@w-(vTiqQ z+cg0&If9W+%#L|%niVEU{Vx*+73GpwI5?xTitblOiQ1HkDu=#*aqnnlJAGrr+@*&uD|*bpzZgg{h&^(r1*Smc>@| zbetefaF;fUXKg_n3z)Y(cpo}8y=K}(8daeB!gKdqyV_sP8ZNLDD`ctf9F#LekFY!| zR#xt~L+z#82Z;J$3+BQ+_awPHu?zAir=86RAJ?7zP>LC&IW}gCIw7zU4ZCFxIm-nJ z;kdF}o~iQ%`M+5sfbA-_ri$OX%q8mqQ~-elI0sM38GLPgoMp*!P8BP*=QeKyb411% z#ClbDrT+8xK_&`0*kQ;?ob$mnruJ40HJ%+-Fjr|%W)9@0;z{)KK0P4Nn89RJk( zM@hrhaK`ZT9C*SK^v@x4Oo>_)V!get8%FaNqLQis32?B&g!G66f>0h|C?2dpjSSI) zj@gJitT%<<3~D>I$R(krFq+Yt-?GY$`VhSb66miZ1t`48>YSorYZLh6xsW3&8Ym1|4n5+e6OSyf{ zMM?Rn(VF6JpHk)WIQ17Aics!h5ok_3tgbtXIYBPr)#w8G@j z%8}%nk?CIkgvC$B42jf+Bvyqa`oU91pI8GjJ*_2iogbs(<8f*|rdN5ya^EM{cer~t zNcJHrI}$i7;g#ui9NqT(m2J~(^w+(wSlvY#4YIqiu^j{4)j>NQ)J`bEMhh6VdM%u+ zrD}5n5uXt@JnYP~3U(pHOiu-|Z@bPxh>W zZsI}B8}YitszXwqvJ|6aEdZE5%LReG0u?G3)v3bYw1?EbI!d!!C^r)0|5%U)EBb4=M z|1ho&a%EL-i!4)XD-wFfceslVb`eUpz=RxkfUNpPuH`yKn>7R*jcI8m@VXF7T3E=B zajE%!?_D=?^Uuz>pVpY&vg8bmzqYT}cWiC8qGc=G9Al z9|1twM_Seeo58P>-_3s%6(d>EO2wM-ZL)0CMQoqp9R@25GRZ-C_RiKQMdMAOl6>*x zizm;XYuy;V`c$&nz|QF3ezqV#jZ)Mzc}9MM|NX-J@9)S@Z7n^(C19^INSjElMpSI6wb*V z`20$GgMP&e$Y#}pJ^S7K**I#xsQP4Xxc~@bGt;wY=ZoId^%TFDrHN%5U3IgDQPXM* zKRGTBtSGpsws&+J6$ti9UGrOqtX4NHW4WuL!*k4Sw;uelaS6uNf?TT8Qwc6nQhPcuJmmJ(NAq@Ira8`w7-~ns}LHgdH+9>?)dI54_|oC&83`pN#p+p^|6jk2f6b< z&bry2L(JL=iE%T*2@kE@c`?g{Q&1#VEeMbqv~$t7qF$Wfc^X)>%=XplF4LLT8)z$; z1(O>WdQxw!HI9CF_B%2sPtP@%S9pwI6MfYLu#Cfvt<^>w1$swh*Scm$c>tDl8FpGP zrxDDHUfGvki*8RYt1FTB18_Y6?9lP-fU(J${f{n5@qBUesCRNDa(1zTCTZI%QSdA+ z*XHES-uJvadL!=09m8b@)*GGw6+}b*K|Im_aJ+>*?(9`(v{rlN#8h$#+x)i1vLo8C z=VVU)A}Y~5N3FMlqI7qajSeH|H1VMub}nYA)zeEb_@5t2-MI^j!i+cmqu-C;iI*$? zqV=yExMMiF_WEfv`meBT0fky~fS0j~39Ivh-Lk@+1dqElNzFo+dLELC3VCSY9F6;H z4AoUZ*Q19d>e|pt`X1u$Q@P*&R*3iEC&mQ;<`D(?e2||*T;uUMvWyn%qE+VCW>9Oq zHaDUEh~@ZV&^cyW$?Q4;wX#ONRFPSaLC4(?{gYuAB7x_}Nq%1)%HQMX1;%m^gOAco zWBdnVaLYu0P0XTv-@R|$LN^Re8fP zAWUdLVZbZX!jNlq#HFQ>Z#xQv(QJe1zW4+GJ$_>Jex5Ab?1m{D(P*~NG^tf&NefzK z`h@Ied;~yG;m468j0%y52(`X3_(?Bs5IX%-84HF=SW%OP4jZd;|5DuL!$k&6d?E`Nen7?dl*CN@AgR&MrH=0Ra$?$R#dT7TU;~MYX{%^(XQ#1HrxhiE*YV9jos;$ zFihSfVKh=YSbjL9)40|>wyj%^Ive`7^^^^cCp2xhZR++2rDcbEL-j)Flt5nyEn}dP z@C1r9%VMb#!rYtb)SXfxvxH`Sc_Qg!BNuXR%>@?T<-_kxhutyG{? z7?R}l`6!4>R3*bnHKL|#;MSzM?S83CJ4c1AG^e2f6cl(Xl)PedbhH+&8Pr>JA|q#4t%O;tf-Q9dCVqG{O-Ke}tUVhJTwXDeDrMnpr}%89D;eoY*_ z5}wvlh=Pr1in3iEd@jua1Saf`rDMU(%rswp ztka56pjKWB{W@jN<2p{a7J)hc$%OeNFL^lUaPjURZ!TWHI@b~ZY_J&-Jpd}bhJ68gih%2jE?RBtd-xEQ^#dHD*U&rps-RNw>`#TrZuHy} z9Wz7U?9H?y7vEl!mt5Y&!C+1AtR7+lwSXep2m)ThI2Y!)*6^w&P#}G|ftHMN94|1`$A}EXaFa zKPRs$*peE-6aRS+`zWbmjwll@N z-PPB0kYXmvx)CM!ut4CDgrL8ZN=LxSAF>L_9*?@OST^-O=qVr3c3T4sdbzS)`(`GC z8dCIF%#Cdfe3WBi6K z({EZ6Y+i{bV-{r(UJ0&_WD0}{uX=8bPb%O^l4U}i7IpbDeVyz#z*Ge+L~{Ak3V09S zFX!?mxKOT@H^-a!?4qPUii*7Y)3q7a`OmayLCF&<^_#mj4Qkkdx+(f&+c2${Bq=)X z?Oo+Ye@lN+i*MdtPMX9w?=I~e^nZ%g-tb|FVL|yY!=I?Ac6#bpJ34B2mA}SR+u-Qw zo{pQST?VPX`ndQMdgXoB{DxP-Qs@3D)^BT5h-HfPlBKUVR0~p4wuRz#AcjpW%WDRM zoO{=vf@Xj)LdZ*oVfXyGt^SR2Ah7EovDXl1$c=O*7S8y&>>m5uhCn|3%cY5 zS6i(goG*GUnk6j5y`|A)mB(p3d)`iRrvLT?yr}2cODU@NY*@3WFxfATDn<9yU*dPM zmAJtdzIk_fue(}HlJvV;zdd+Y!ym3$bIUUpF&IoF?(wec!-J~{p2{Eyp7SG8y65aXW z2eTneAyn9UX(&VpT)(ANTNJp~s$p#AdzzkndM%=Job;v|(J7+l8c=-bYk+(jrkrxy zgmj1fweG&tU+wGO=P~OM7*h$z7zfm)m1s0?Hho@VDuYFM(n{7W@TUN0K$yQjm&J`; zbTHU=EZ9|RXldWE)zHeCk6q+9EO1Z*qpjLtCO&vxjXE&6)CivS^RI0xhF&ORWO#Lb zPls7WHQBo-ZnbjE?w;I<=7v}6`5jlAxru>3(Swpteg*#%8Ji7sT;#b4zqXRC+5!`9 z0`id>*zxc! z<*_r(OJ038(jcy5T9xHs{gGGs_^R)?G~wO80%<-oK^PZf4i$q~?LkyF3f+%E51y^v z>y%~=9IIMxJj~$2(jP>Ev=Ppp4VttVUHalvdX70JRRZp(RtS>9BJ$QB|hf#oL-y*NSF0!Klbl zKlsJ#2loj0TDHLgQ&XANC5qtz`Z-VI&m6bK`Al4y9M6oLwWsrv+)fFZD%MsXtkAvt`(ltVQuvQ&&lcow7q7pD^v7CBvJ`4# zepSEFBOHKe($D|@`t6$z2BrT^uo~TR_ELny!~D#2%_i|5nKi0N_I@|C8GwyglDASH;k<0 zpBvHE9V#xS!v<*@AUYgG$BUAwPnd@k^-uQB61=^m5t=&b4)C?6ydiPXt3AqyxfFJwp}dBHQrjKC%ln411q0`0-c8vtw<*1(JjqmR zwh7WO^_ZEIh=UlUS<>c4+lWRYFQC3eIwUcK_`k*IQc1Ud-Bxt0r;EiYiV)Edsc2)) zy#D+l`&7K9wM0~St;9m#1pyEbUDXIpYQp}B`a_rz8s2=+Rt06`S{@kKX=!fdw+G(J z_zc|5rukIC{L9$Ca1+Qr=#`AmBV5e*Ou3#fS3yuvZ(aLB&K!7Op|Y|LiUJcTPPvie zY0mgO@V$)Bfwwb0sdqF!X*V@Ku?QbN{jR_!%!kuAA{8a$$}L*Lia=|^+7luR^4bVx zJX~}sMbRoGp27^T`!c=L`fL*e5Z3NBkm3nsJM;$>$}y2|rk5({9J2Md69721$&j|@Z9}WvfUjk&0YSn( zD%R+|cyWD6@`m5CMv}8Xg^JV-=-O!HKgm4-zm%S+46j?!TzN;X8#c}orx-I)orh(i z`3=f3K@@I9+nAV_?R5do>M}J}^4dLAl7_9d3DS6=lL!%V?=;=RQlq<&%Xl_&a89WH z=pDV~xz%m37l%U+tf@5KLHNO`u=rlMF-!1JyMwzg7(1l^^a1=ZfY1U17FnweF_g!8 z&$@htcpJ$C)=emofN>tG*5=}_x}!~gaWz^$)>p~8_kE?*eoXY|bP0&cSi=wqc*o?=qU?QJqFc|LA75-Q0)?fYJP9-p1iSu zQ(-UN)VRgVPST-zyIc0mPrDKlgoDN4 zpcjFxMZ~R#4=6j({RBHN%v@K@%2q>xW?9pW1#!2!$6c%m?&b7_BGA*@s;4P$k067Z z^^w_zl@$K-E4^+bSXx65mqaUbiXoT@Yf4GPYSzokJGp%pvEA)6sY|k9m=Mfe|Jet< zbFOmDEM2?Ogi`qK=>o8DtZ`)@VY+D!Y69xjDtQ+Z4H0|}Ge___B#ao*KYSe{>3DbH z!U`l<=XaUnGf% zBn_meWh9XOMTZZ)OY_P^68A%i35($Sw_%+T9Kxvkiw>DrA2kz6OgXDfFMN1%fI z;zwsNv}!f~17TGrf?Z^~=RJ(EkqXqR&K2bcPfI)ANnuGGUz`Od2z-lE^woh%QKt$y z#3M6d2a=eq6Ilw96ujgB9)P{mm{gLS;fIT?Zc}82dAnq#Xtpy)Mt{9W*Ss;_PPTiX z)lAzotWw4H5p^!Na2z}Aw|LWmd2gzvKImfX@t#%D~Xt64d} z0s`Ptcbe|`Pu_gAm} z&d-1Pu4>C0{MS!s-?3MJPb6}F{^hTehS$^hc%|l|nd8zw2dugTB6fieIWOHM<%F2i z(jS4NLb@Yena9$mvu&0qO+*(p&%5Hfqo6H6HF+x@}?#mZVVwAqr}xW={<@Gm?7 zjC+Ir!T56C+^34MWI1ok2|ya~zP}=O08(qfZ*5Kmz7`JR8u)UW-Jq4^mU97mlUHYZ za$@4(C8IH4AiAO<5klllueT+HW9qqo|3CKrwaabX$`^w;zgeGi%{EVJ7-H-PE;0V>B= zLTz$ZnrDJk&53axytAnN0FGix(Kq$?m`rf(X0kQ~K}M-nFGU5)h8cmDe}fd+9Nha( z+!PKGedjQiv`9aV7QosqLRFeTw4LMAeR?WE_+(MheAv19B!!$kMVt7oW<%Z5YR|gI zoA;Ur3GGG0^$usYMYyiiyAI->I zN2>H67exycYm*0)lY&S{T_FyyUx&@C*hcNj|18muLVH;I{W9}s2tRJ5g$h#;nvh&4 z#&~4`c~y#2G<$4g1N#X>u}r@5iZp=vh!Zuz0ARz}3W*Z0#ga*|S>QF!s0`FD4=B@tj>9P$wueZXf6e z!syCjE^9~A^8(WqdL#%zFYT7rf4L6J$=VFbn?YjCRfsAz2$;u>W6pMiysu`Uvy+eA zw(9(I|90RIfMtsckTPdpK*z<^rSR2wQ&3TVAvpMgl?xwet3ILX9w?!*=Z5ALF7yqc zJKsz4`SqnHOBYKyxg+4x)t7nAYiuWJJ2JKZlnJhU8qYAD+QoWVNVgzf}WQ zZ~kubTd&N)@7}8$hpPt?YJ-?J{D5v(pY8n>%dgiYo0>8_@3lsZNGW*)TI-5Iv=z17N)WRxb@ins)*)T%JUVz??YT5ZvK_rmQoU)2h%aH$J~aiYs3R~f7nMl;TCN$GMD)NODR5(1G82hb~H zLsubYgQ0AR%LS=(xK}ubJ@O}~0mRH0b~W%^N!V1vvdok~;#^(-SvoLN?OqO_RWpUO z=)HK*00+g2NStGd2Zdz6#;# zRYzijs0@YFG&q_2nr4&cI}otdi~q&FD>?@t@VhV>hQfZjn|zdqJeAqcZhvnv9_|FQS$Fl_bu=mTq$;)rRJ* zC-%Rru*8Or> zRu|<~Pvr}5%V={?63vAglu~#~JS%iAq00_~apU3+Zv~v{c(pDxa$tSCG$un zd%q0}T!3&<2zp9;x^?m{t}fwas{<|IKkUtx>Xe7`MyS(l89(qW0Zb<32R;WL`Rt(m;ewJGyvi2KN!su_dA;y z!EjiVurDAa1-(97MlVY@4*N7?5Xa6G?SC@uWKPp)X%rA+mruSIKGh`jIL6^D+q^3HcYO z@7OqX0s$>^bt}N_%3^@Mwuvgs2yuoozvyp$5t!*&VVNzH;-nE048TQ?k;r8e7u zfsAFmC%)*F#>j74q68=F0vn^uyecyt)ko!aNIcq$wrC7`ywG+jI@h-ITAxm1tac-T zEo!UpQY-OjX~qEOeb3T@_hvY|y~8{gkjsxH)y2t98{K)=m2R3JKlaAT$wRBR5JL`dE-L{~~O$56y{o|MsV zoAr%(yI^Nll&bURWf~?_V?Sv-nl5Coy6HS#))iyR!KVpoO@Y_YRkD?u7m&|1xl^@$ z??@50BHD&UkWq=L9X%SjyXd0S`P)=j8hWoFWQl4U5rl7ufN&3+ z)}fZpmf?pwZ5n?u`Ic#SwZU+w8ZhTE`cb3`I3DYEIju)ivb6Er=vkxYGipAa!y7f9 zt~P2u&GwmVJ}#4BxYkoA=y*Y7eng{tAldNOWxl?gqqihu8^Sf?fB9jJOC|e$k7<$0 z_RSUHnb)A!Pt(sc(3V7Ri7SXQE$te$~m=!mWYxAymOyXt7xR4z0egQxL!dTJ5cGrb7ymturI z)gaeqJ3=-O+TlH~bV{K9ab{#g8=8&71%=y<%|SD)B=K?%4Nn++BpR?)XSyo(#F3rU z(bd87m3|v|9Uzy>IG&GR?)i9Y^JH5mMBmKmi|Z0Ic27tHWzc?ah(OL(R7zsQ#oWmN zKVO~*P~rI9uMME} zru`9; zMIP9Xx-TxQ$1Oe;=(0qNo7vst2$)Xl?pa%D?V1@#v7>)x!5MF?;L*D3JV=q4=-sC2 zfE5-Tk1l=|${k@z774bhW?K=y0Q{URsX$2G2HsJ+?YXaq#VOi0#QvyTyhKRzX}f9C zMc|=Y=ss9QKei=(FGE;TolBo zVeNL_=aR{)9w|lW?-kwPg6PB#T)L__Y`!CZrgIBj22Y6uF2>^FE!jRiZS$pMiVrW# zhc!Z4g9kYo4qU?z;ctlmkI00yw?*J#qV0X^O@~_XjUo1VfrUtgiE1G;v?z3x5=ok; zYmZKi(O^=HcC5@vBoDMiLvC_fwE7^1urYPHQxGOQ@!A2k(B`){AXF0a18t+dv-}hI z5!f7kpH?2kqexduLN^)mC6kDjSyAP>Y73ADdrHd5*<(ug+Q9XzcIVpQF6S&vwH|aO z&Oj8A4!fYj%eRLQpPEZheBG8fviB>pU($Ybnlny2>Qb%UZdYONUvOTcm(21!zWVUBr zv_~KNx)jmtSQa0^Z08^tcVW^7(&xSW`e-R{fxpJmdTFwdQdU|KXybz?@ ztnxcGMNpRP4&@Bp5%7;HX#`@9L<$nJH+qOb&Dd(q*@_CAa%=X)!N5vI6@ty~KSvVt1(AvlDy!9u1Tkxliyc3>B+H4b$m<)KlISDoxT}=W~vw&z$N~sPv56k+GfZAj9l4`^SpbfCQs~Ih8_>Ou3 zVzw)FuB>@N5UF#*ABi&S7gU_ccD+>LLjjipDtD$D*wOA^hn z#Dd6}Ij$+6)$b_o;hz!nCtaH|M>xVV473oM{LYAZWrNC`aW^p*Mmo%YZBP(=Tg@>i zSp?iFY4@8R{rKbMi=oq`-m!up{JS|-37%Gf)1`l}d=(539Npv?T8E7?dWseqsZFLY zsJKmoSKoQ9d9?oU{3SBKCTcF-cEXsUu%9fe0Raz$`d(zDc249}+#UkizQ5_QtuI($ z(Hi+TxZk#{LU#r~&gdyObo~syEJejh4ffpnOl%J)YAxuch7uIdtbP(CJkW&MHd6*^ zAA2_hKbB-KZQDd+TE_j@OV^l7P`A=Ir+dY+Lhpz2(A{}Cw{ga&23^acnA?ZKO16Vd zP`8l31HX5$yXY(@n=_H)C#TSblq90&II%zvpFEtRYr3KZ=7r9FcMt&fxv)u`an})W z#h*V!j~Is@Ww=BIv03zn!)&!y5YgE_7TZHdH-E45L88lve%`xMee=@Qi`rQgQA_^t z{N)t=XjB<+cl3czN~q<@1Z1moJ{7KbS92 zUwEFP8&;c(0}78zR4{n4u5YKB2_szA^)Ucg5|`+*q|#L{3(J04!AcW~t*9Dn^@7S% zE$Qixs6RKp5+F*@l`hufy54JONYbWkX`6H2$8tnUTgkWx?N;r!EfCwYmz!)evPx1n0CiT%*0aX>E&g1hNM##Sp0HoSDG z6KU{>j&mF~hB#-9E`6N4YLMr7?~!fsSNnKNLv=r!Ou2CA0;n~>uLe~FGYIO!LV%hp z?aX!R?Cr~o7q4GVSNS0yrql??B38_csq9>n*E0oWZ#d^tt37raqW>pm1y3T{Kj^Kp`{^iC~vgUL{ z3$h3q0++O0c;FsBWQZw3(=_D2R{vFJ=k^kY^YP|~8>q_!{^Elb9o3Ow!Oh8J0w$$L ze^WAZUOYyBCg%GL3QcjSg1%&~3J#1dzM^ncHJrbB`%F<#@~6HY8+0fEK;qhae2{#- z2Zy%Sb|)2jJ93(Wgl+jeMq;}vXn9*>RL_~(Av$xCmbZ5Qa)QIzyzSfJpdD~9_m_jL z%wSMf=j@M`O_*R0UzvaS5Jne~*nj4cC#d!Z@LVb~LDLv6Jlf#4CAz$Retp^Wi|iRc zoe)4BRFM1pB7K)BjOIA9YPQ7di%KqEPyyIb=>p2mn_dovE(B$~5s0dZ4sO(%#`8a3 zTp?Vk*^{OU$AOYTJ;rODnKgKX=Dh>PXDQsr=#Cpc-|%BaEbmzE;l~QoG(8*a#|rze zK0+w|fj14D=zkU86)mZxSTa7yZ_s$lnNTVr*-D=9qWgKHm*bVofeCf#$WB1m)#(;3^SoOhzyDY8H}r}M2+fI=8-$0c zFjap*a=TJWCtbh8dCs5dJ9us+DIS#HkOT9EFiPviek8YR0~dbt{0iw3yZ`acIY@F} zKK7#lV4;E*J#}+qYh}D=t%DUF!wThRTLk^oS!VvQM)<~Dr#@N`; zJT}&*w`s~fF{qOM68pPi`9u}l}9R79`MW6n?ud5Sv+4cU-pX7ZF z1)+o2JaB%fimk!?+N8x{^y1C+yH^+Ay?h1prlVT|7D>6J<=%0R>8G~ud_|Y7Om*6o zEq7gs-h6-E@YL;iC@cY3gx4JB1g3Y3K4)BCi|1qO_MCU=w? zxt@)f=V-ViDlS+Y99)+)qjAB756Qqh(a&3%MV%kbe7Iu&it!U6QV?U>yY6S*SqB+> zCuSV`viWsa5t%d&*ZHbnUBAKq0>{d$+TB-v%f``Q6AUZXI1B9W<6+6yKj~ipe&rzM z2^5A!hq_nV$;Um@6;nudr_xsETr-<&?Pn?92WLKbuF<{mZO69%9L*99PD-cVcO0EB z&NJMe*_$&gnPlZ2nKfN6Hl6lY@WgXhsJCCxa71+cvU4RVS<>E#th7$0rc3F7Gx|$P zt3Rem|Inkl&Ics2sfX>HU%oIBrp-hXW|N_WU50jd+P&ZC`Q?kf4<_UCj`1NN9nDu- z5j5s+?IhBEZRFEBd)ILGT4jLx&R|#~c1r8OBN+CBIKNx#2!2{eZ~t2|g392o&ERPE zLS;Bu|Ik~00MD^67(<3yxSHc&Fj{$M>Aty$?akhtgfsQG{!cWSh<#cc?{)2rqEM^C zO20aX0aoY_x7URwR(NfSKj!SNyu&;X)ay%1bI=+0F@?Q7Lrv z9>m#6U)K)1O3Kuxb8UM(|MuH&b*bWmT0KwJ^q3-FB0Qh#9SCfg3dExfa;IoToQ{S%!yDUe}+rdEB&E*cL@>%Av&%zGNII7(?etqvn|13YJ%86 za?J}Ohathq*ZAg@qpFv|0Lavg2Nw)VW(NXHbz^?8QC-j)N_d9lz+^0M&BR{gggUFu z2r2uXkF|sFj8)P`FzCNW$!h!ufa1|J6n$e@9ghKE_04btxE`AF$tfZ@TcTUCoun6~ z*3jhEV(~%^woy5-uEnXu{N!0fHukx(w;=J)OZ&pY2!g2;0g=4)tTd)B3xr}T#lrlo zi#8=vz1+!4Qzwz+in!PvMIu!<);hdL@nCZuSEcujA@d2?hXff(90H24t>fYa5-n*K zJHM`@xnk{2H%?u8*PQZm7J&Ac0pd&BdKa6(O8} zb?CQ!Y7GsnG}KpT$c)pRrXiI*_A1e_O2lvP?Dh%B5e;BCCs_=jiTY$*?W8oQEyIeO zgoRuu$PS{JB=ioV=mg}55Il%zP+<0i;z2|`0s3;@s+c~U0qW-KJ@yt+JE0{>52EQw z-4V%p5ZRCv-jcKj(G5uDsr3xe05N@MSbNnXjk(_2t*OC#xuNcoGU-VBEs11M=A~wy zQ}o(gcmp>NRx+?)MI~LD>fSe}ZZI%gY(}FESd#T^I(XRTN=(0m&8x#Nh@q93hSXsS z`jp#kaQ&rW)tAz%`VR~AOB~%t4hPh2Xi#7x*_6;h-3IPQD-iFoPWDv`P32O78j^hC z02xP$M}zWX#cfjzcfhb?X5r@3i5vfrDoWM>dU16*(3;@A*<(-ep7?An3YXfV;1{QT zQ4i~vQ6M!&!Ew9~ck)m2%0sFJS@e?cWPX65?wtpS{qDcTqhU^MrrkOBQjKzh6F zRuPBW4_!m?R&9+*@3n5sh5+Z!bn!M^mP{AX7N@$d?0}+k$lO*Sb$YQg-186)k{$P} ztBbt*VQ4MM#v~gWc9ul8Br7~v&Qk4rwK7*myu9O7N}>@I3111s=0nSXqJ~%T;pWMJ za(lY%T{LrUhjP%=UU^CW+GYtVb}>}>L0-g`v!E$vztL;=k)~yMRZt~kx~Uv2C=V+3 zi4Lw|Xxm{HnGQ>gwXO_WrI01&)QUYrm8*~?##&c~(L93taffD-J8}@om?Z|*Y7oV^ zCC0a|3?dz{#3cJPcWEa1I0sv#!j>46CCJx)`(;^TJn_cfnhmSWa6^Zmg=0-Kb$)b0 z>|whZXO1(Ng+QNhPG$HUQtE1#HsU zdWN4EJ-#6W-0Nz!Q8V8kEHKw%vO{JWP&?N)2ODMN#a8#mPq?rtYVcATR%bDi-03`Z`qE&MO22v%4mI6vYb+(<i_!Z zqrZ@se@!Iv`0?+5J+Pw zvL)$0M>_Ha`zj;hO3+^HGYphW?%IIc*~sR?hlDu%y1K_Mmt3ph?9)2 zsk%P^5(s|RyUtdCOjTl~AjRBN$t(S`&b3l(l*0qMti>T_tFqY=+MN8&_@h%9z8?RL z*PBMv5KOTf!my6wH8uEP_ra&ghc)G}*{?~)%6uQg)eenXWCC|svWX?HG#SY5#Rbi& z+`5P`Q`Q~p^EGT?E=0ZN2(Qar2r@^DY4ih&Loz} zJ7{-^{;sm0R3AyCs6{r(w*aBVOn1C_px!}9H!ZOuxA4pVajx{=%$#<0#{eLtWT;4w z8&J2?-0xN66HmWFOIC3)eKE|ZBpG@FZ)%g0)vqNg2o@4OIY%p6R!Ve9BIv0s-+VRT z7^W*L>1s-^g0j{Xsf?DQxlXDf4)&X_Sl0^NFjv>`tx`&M4;(99#!r;klLeN<2jilA zl+fQRHrvWHI3@FJ6G!R)m>+i_UVHn!Ta2+rc1a(~eYiHZ1}EU1u!%DUU_wIEc5Fn7Ji=?+Erx?^vUMY1r*GreWs$$n}AgQkP=m>LOlaj@|TUY_-DTv39~B zoJDRSgI}rcMUIPl7QEP(LpP@+B`$-$(A|k+Rw}twD4wu9+6A4xOn=3;^|256AeA(58P*HyKHhMHS6M z2eApl>+KA~E~j@YfWJGxFJOSqYHWjmL^P`8MA5la4>Nxz# zZ5~vzyx&nU)^KQQMBc8~(DH{~+6yen9ZtWq&GUX`rdB;ByGw#YytukVf6$~3uO-2m z)Z*t0w!$>MjBNYGcF~>G!CW~y$Pn_16wm3RQhKY=Cx1rx{vDC%^MUU_r0%I_m$NAb zb@kJf<%LVfMKT~kN%dt+I_?Apu{{0$4iGKymXLa-OW%2i)uIb36hj}C;L^nqzPs^b>6t8bpbVbo}cxA4e%4$Tr;!SR_rtdSL@}DN0NIr|4jXdd-paO!Lj%6 z?!xYWBIKQjD?h=6*e-KR%dxJ1GwvneEo-)}ick>RPHDvh{15v3hq$5kshV@*C+2i# zcO@ZTb!2m;)XoXgOaZ#;OVA1uZUI{z1FR|uoXXbDNenzW|K_W&qLp^q)pX))Azo>x zBbzGjhfJunGbDRz&uppVmfDGW7dkS~FM}EAHn%)7pGkS=19P!^v(4jqlu z#|Pw$``O?@nkkv$yE)*|lYvy+E_X$EtWck_%BD0XBbN3)9)$T~y7pq%1lMwgab z5YHM;$UHb&GC^9V>A?KbI&(BE-3%{(FvOTX$dGYTztd@+vA9uD@xsJo){yFwDIR#w z+A&Jor*RvC{wnJgHM;C{KU}w*sP5-I>u`_0(?I)7>fiq&i*O&}#S;x|O>2w;<^f_F zo26990PgF~-^JA>$~b|{VO)qQ>cF!x(;cL0q(v*!VJXRKEyKHHbmD#XPQ0Ty1eVnb zmlJh@!HwtLs=*m(Um{av&Z@pz!SV+wZMuAY3P>8RZ{;KUN`T%T^P{DM)~V0fI%#&y z#cJ#bZm#SB_rwq1xD(?XTF}0N?9G}&6Rk=sYPcxaorreRzsS|PtMQq1@)+=P6fLhg`r%Vpv6?^$Bph_$+Fe+s6U zHIIP*MpJU!T*IICl^YGpfzIHx$s|9A>=>3CV;0(9wA|=?{{_vqmoJ9L4Cwdl?IT>g zfchPMhed=9zQ!e9Xd}a;>#OfxKUO;htVfqG+8$_M+R`6*b9h=~YYGP4zB_N;!!#Tr zbJN~mvC>o>H|JFwutEn1SZo|&SxCz5xmo46hYchi6j1kiKL#86!UWc$;}1ca{WhUH zGTZ%3vm788X!Pvgk(thHFLPATlAw8k7ZB1^YJ;3&Ocw>=as_*-BabZ1gOwDoi)waH zZ)eZ`cyn`w)|lfJk%Vglx9VvK0x5nKED6p@D1EHamsh5)qj^hjQDZj7d9Il&bd(dZ zW~B(h$fT)J#!Y!mvWio=)yI5*vkD{}UBpz;nOH&-!!Yzv;y#zwsL+q4AW~fsvjUg5 z;R`BJ<53S}@`mG5s5{a2bdSMa1Eq;062wLG=b%#f|l9$e` zbbjztY5WK!O9Cnc6?P@Grvz*p5u!8B?gVT@h6;vrbj8_<$R(+SbAD*O6|a>TfaBo~ zIuTvVTyVRnn?^^}0ZCEK^nVEutznw3V}^FCwC7|)3bp5_EA|gs6nM(`;*6A&AFs6v z|7S8gQxwnssL$6~?>Zi1yL~W6LWy?T7dgZUY`5C~o!FD5=u8ZuV;w7W0nS2_*FYF0 z6D6I`?{Ct(M-==VDXUc!C->YllkVORoyQ`VZ+;t1DrZ#ot zq|^Z^>Z~zN7qNV>!`8Mju=`)s7o?22#IDs+!MKmI^#Awzx=w1@YwiE<_4UNdi>VhM z7sdW-f5@UWdvy3TCD;~Kcb_JTMLM`2&c@kOtI|%<8zv1%hxrQ?VAuHy1DNVS=~F{KK-OALOvXX3iIRz)cX(Vx%?J;mO*Oh2#d%jzWITppANKYsBw_;p~G6o{k58tazGah|YcS#8XO&r^6(gK`@SG0Kn4}rPkwxhzNjXwv=v3FgIGak?s6zvz?tP0}12##q*p5@)=z( z3BRtW45v)_hrwDdYXMtRK?2ACXHqiRMsL8-Zq`V=>ftBTrx008R%^LEMSoz<#o8pU z)ZAFK@Wlqx0@R4zcXO6iD^hxSgbU_=#4$b39`gk&aJc|&Ykj*o%lLLJ*_kZF*{{~O zi%G^xA#o`Kb%vvf0t1wP*Tcw8Qw>-etxkdSZjLz&mpy@3CrYg2S&NvZ=+m3 zUXU@Oqg+19<)d6avYq}m%H^L8&qwpdiqC`y5_mBt(jwXp|0q_O&b1d#7OOT(RH%8vS?-TZWR`CtFs)4Ukr}1S` zsDEdgfC-5eWKSf{TiRtg z{Gx(mPri(~SA*MRMazK$pyiPV@bsVo=&MOf56_2N1fR1~a#pmE>a;APIiM+41Xue> zXgDh-r0EWC85U1bgTtSQ*25uNkw->DRPu_{xh#%geMO%l422sx5#bsczH#+bt!>zF89&D1BRm1CT+g4}W!w4pgjnE?Klc1VNGz+?-f7+~uhqhFqKi7t zVES|On!R%Tm|&C@m5{ra`deGE4B|r&EwiG^0iA}Fwk#-=so-pd8dXkDbyiqM?Gear zs$(C2fb0pzAxTn~i|xDw7gR2**)(G-?>;;;NR)DWHY?cdY=zY*)a@uaomG-?p+k7i z1YJxp&z4k@Ojewn;WeFPtlTJ}ot0v`%6}!+B%9zepXiBdUT*zT4Yoq&TLOWb9f!Ip z%P@%xR935*-q7xSv}U=`S+-i*a)mBev{Xy6HHBFpcWg0j4Y8>yR3xo$3<)EIe6hyr zutmoDwoWUutP7W0gfm&=zz|fGIpGCVIMp5Q;s-{i0TF&i)U#{CRrg6gMJ8jpAd)!Z z4&%u^)-%3thQRk*&61T9N%)GEke>oe+FqNE(RHmBdL);Ga#Y|MDFj+!nHk&+VEK$c zKIVE><)ItdqtHF>#wa@`;U0FWj`qs;9MLfs6U2`t-q`d;q3#(coG9LNym5D5W0PH0 z)%v?W)2$zQ_p=@QrTs*6?&CMxZnq~eF%Np*(l^^P5I6J>I>XBUrJ8Y~u}8%-^xyyU z;ES>S94%X7F2z~KbNHXhn&qx|(-yCaV|?TM{QUgeZ@*Um|K!`R&;9>DKYw!m)%laB zUp)Eho2TD?bN=O*PtU(Ne|rA))2Cmc^J9EG)+!+}|Kj}oj??nCAidAU{(i2T`~T0R z{mO}Q9N$f~I9Cb6P0kcZc55dWPMDDkJAd1pPrv@n^ocF4()G+FMQTsJMY0{GI>xNIG!mfmVA(;)CGEN*_I~{ZfTi6L(eNA+3Ky~DG;wqA<}zAB+jvf zKmc5pOq&eNos(%5t{{A}Amvoau(OI5IpILuetL60{moRdddM56th}KsA|zg|pP{mX zs_|fFLjE8+hxtHpPW5eFy%}{sq{-PnVo4T&@T&iNf;uJ^{-S5%isi30Rrs->LjI}s zhga0tVAe&&anb0vX3tB;0q?2)=E_9P3tBF!0(1YR2S}qcJgd|7)qQ0nVHAytnW4=S zO(k2Bl?%>bYf@fZUHruj98mY)n1_TneOfdJOhYyYnV`P7yR`&wZv}KV)h#0p z zBn@6#)WTB-`^gEEmaj^Wp3a%+)8y=#s@1nWRW^6CpL9qUI~Ez(wSg5-o3~MKSLYhI ziMAH;w=b`6ko|O*5UA(f168P(Kus8A_ZtLEt2vW0*M<9QJ5GzJsAelFh1I(SApAWC zMgwjWEp1KFWr?2S6)B$M9=x+-E`XY8F;Nqnwgf(pY(Knf=P(?sXT7u)*v3Awxf80d zSZ;JA^#|}N$v>mK*$U`ZOUHc*v23s`A2BS zjFP`k{oa668uT9z3AV>nCmB-9^`;+nE^NvRSlj-_X0Mb!9n%DX3C>7iqfR74oJdZI zQQ-c@W*XE6Tu(hc=(Gs~*;jN$JKS>3TZr(9$T(gTRH(-hD?Q!RSOprcQ*2Tedo~S5 z0_3H%J@fS2#-0f&r{8=vNGXS|tNZ&EDHn3N?_3n{wl^0VWXPM4j(M~D)KJiI%a8B38AQqV*`c^~^|aaxe#8?`a~ zt6YDmYjtA5AGnfB#_2zTi2%)M^xr_j7wXi^+41y{N(n~spgEIakZxQQ%83p((C9LL zY`;e~Ko!*?ARyKU9!-s9O^9T@w>Z|lqF9q7D@Tyh!{#qFF7@8P?kYdlWjHaVD=o#McWYBT&J8sr6*=H7VCD#> z%vk@5$LvZ%a$#_`sRyK4Tc&UrVSmkX$0Bi=gYi@dIxC2U3GWDZJ*1XSgyG!_vm4Cl zk`C#h3+j_%ZbxTqN%J+hNZ65>+0lxE#gV%EnO6ei8qTx`pdtP-grC@Uaq5X==2bxC z>rD=c&&ixAwf8N_p}cZh!LRR|@6_;$b(FR71KC)C-euFD*GQ>fe)xd6CA$BJmaJqP ze5`4s`D6V&%9e}?y<`ZE(SFA)CD+(HhywL>#w7Xxwc;HA}0M>xf)bH51uX<&? z%YYNH?r|@fE-nd@HLbc1uvy<6fD!f|AVlc4mB^pUqEU$`oE@`_pZzuf*#Q5P`r;w?AAyWb3ao-* zSSwNvf!=yszhyew94IzyK7}13H6WLLiKx}wcji)0gqW&%CLba+GrX1+CpxV`2R*q& zwB}U)`pHoyQjj$Uo6R5uGxH9DX;QMWydwmnuQDu&&LZ8sGnlR83DEAM_%u zuyqXa#vOF4pc#{rDVG6}?NA&sj+bQ=HARzhTSZBYu`yk`h9iX5xOxH+!fx$MvH~19 zx22n;?9Pmk^!BYq#4(5A6fSn17oqMgE474IDGCOJc4Fsj_^i%V&{jE5=FvN*rzlyo z89fefb-KPS^nY>v{PGeNL}^yy)C3)iQ-tFd|2t9?2n(IlEi%UA#W!jV?k{-qkBiCw z|J`5BKh7t=dH2_U>r=mMOZEq()Zw>A;+`OJzwIuHx)kdRffG>`vd$x+0x2y?k}$>^ zL}_BXsmvx)8j!v-%3SSd4G_?LCOXl;Uc(IZaE6j#ri-lu!;Hk_~YAG z31P~;eUAsd>y{K%2su2Ldxd>4ZuruwiKMHBs7D-JCxc1XnLILN2q+x|_V#W83=j`@i?!>r|bYp6RvM z-fO!CRPC}g?YCu7hE45Yv97#V`$jG{b3Lx+Z%~qo#JJOKWLnEh<>5 z9ARLJjjn&bJT2uXf)g>Eo_d7BpS7}yw4t8o$=+VHgYoMAvnx(Z$kPP|a;zxb`&#bjIAdC?GN_f(5-AfiA8f9&6xrY*OgJ@( z+|I==xaQ5H`E&*^@tk^#TILuz+}XEH8H}33GxqDY-OXA_u5v;ybCeS>6DL0jeIYj2 zjDF7W|3|uaG04$}nxTh9;QtPj01idO^Y36#WAI$S_w9hwun?2tOOerpC_4Jok+l7w zN{249vEkY4rLV0d@gSZ8b9{>t!$TNqU#0;)g-sQ<^5yrFB2w% z!Bn)N*0JDG@!D&^qi-L4-u#)eL3s3ebdKij2xzd+EQ!T5n~Lc0yt6!!6rLR)&v5p3 z30kZccG?bXtj?T1}{zt^BP-dOO*L{mkVbvayz3ZaKQ7E1e?Qs=2 zWNV?(cD&MVbno*6B(e%V0#y#`0=p2O)IQn+Z!n^ZLxMMUE|&MF2Z!Kprg-_&f4gMe zm$#maEi%NA{0VA;wg=01RDj(zGInN+>5L&UEQ%r78B^`%i2PMJ*18~(z{s_jz~8(< zb@%$;Lh0PRVj}I~@ke_7Z!xay(%^)iJdlE2IKEtzSGP}rf*DUV6dwC4$8is%uPKIr zM>5sn$tt5aX8Q!w*y++q%Tc^+Lyg$j*baYIu$mXid{Z^fK2rC;sHpS6dz3PEVX0=& zU+p{19Tu`ND}NNrBHN8-8XyjYqG3>PaK_^JZyQ%5L}(UvVzp|QVt z#I@?V@DW2~&0uTJ`3Imw!UyM|m_;64xTNk@k?zmd^9GXOHShmT>{&Ap$^AjtKs2ld z;KNZnTGF?_J7k>OZ21r?xIICMP0*dUTj?< zoI9xI(P(f1;kGhPR}JVCHdL&93l((zDjiVWJ_)@7>Cjf$5Z3oJ9JM+^mjhFa8U@gi zc;uq*!3Hfy&is*hWWHBJ4BXFcv))mpi+Dt22vJHpjpv&=iN4$xmxu@U?-{UdzW>&( zw_Fwb4qi@kVHU@fnsrfrxD5D)SJfM$tT|B$T%@sc5|$h9L`eW)NF*q*>v7f1Dd;3e z9XhEf8beu0vp{fY`iG+ANTjNf;$_YxF3A za585=i)_n(7HJBtjCD(u$d*~=VQHVE167}%2J zOKc~?5{o(XaW}l`ANcI`e3-ku)lJ6t0Kd;su|LU*r@m9m!_wNAzuY->bN<;qF zf8&}ve4kH0q2ay1e7=TX6CHcs&dwQUU*~*~tdir?0-XK*N3Fee`P1Ym5N&L?SVLBleqerxuDj1Wp^?NUpSP3j;JQX}_|)u(0IJF)z@hvIlmH1DZ(5Kjr}R{`aoj32?5uJv(1MS!n( z%%zjQ8Xjg#;Twb&3&YMLp*msuY40q`yw%@*L^zJ<5!c@!c4z+0v(;yp5$Rzup}c`s z{+J$rmy3Q_%#YK_^O2d(6PW36>Mm&(EMSR5R2yto&s19QLOrU4NzIzhC2L1Tc8Apx zZC!UY!NWI@`P8qN0}H$LyJ~gB>VknUt8r*M1P#!((9#nex|A9Y#gPg4Jrrj^{r^CI zSlG)#)>&OPV%xkSirkAuGjV%l$}`$B8h#%Q;)}Wo#eD^t>qOqcwmGU_3IBwW|d6#HSopL*uo!joYt!=$QbTu*J>@eDR5$16Q%SnsWUu zXuGGYNBC)>4(oC)%=Jg<3y6_Os?rM>&S? zA-tJkO@SB3^*`G{SGC1-i$RZ^D%CY7U{8htyY#zns}5RTHvgLr#ee1(SW4jXQgotT zHRRPov~L}_qL_V#E!xQ=6v?t?yDWn=k~yt|1{@Chv%7hL_Sh5KP(-?sIy5hXpnB?I zLUuCrJ({P|)oBJ?t;b(#Y){Ehc$3egy0U89vJK=fJi*Nsu^ob`iuIe8N&7I|FW$1` z4b1a!(PX2YrhcGiFb)Tl&;FIn5n>9Fy9nE1 zY)?yM4CTO`ReEYl6bDe;<4AQhd!@hnw= z0f#G4lyIA+YH@w)Sc0_KeJ+$NHwkaVU)i*niy4uiXp+BVZ`fj-u3Q*fAHM7Ijx+x`^*~_13D1v9pd2dR6 zp(UNlMZ*`F>CUiV+>ru!!ujxrvJeX7N1aMcp%#?wPN^!~@DzDs$?!+xV+$omoeTHF z=bWofxMbWGwUC5Ew-Hf!`q&eqEU$UdsJUjH#rlYz?)~xvuH&Awl>QwWL@cg6pOE$l44WneX~cv$sI^U~&&{pgEJ#icEbASBW-j*b8vi_~7z;Fe9h`-i zKE$8sz2_3UCZVu6x1#tq%VIliVZ+nBW>xvsZuXNUK&=$#s`$Dfl}Bu4zFAUDnR)q0 zJFsyB`#o=(W*^|@qQ@9-uRW#HKVX?M#4G#HPNQyZ4F>kn5tW>8t0r>moQP=$PSfR; zjm5fzC4cJV>-Mq|OufVOw{xN#8P#94Biti%v(2ipiU_k7RiBo)SNAApXG{~2vvmV6 z+uNi!wwD;Hml5I<>xfrh=d@_Wxk{rAej&3T?gNK!#HgzgEINnSO2m2<%h@cNi9rH| zW<($jiy5u)y^pbTrqGwmW||<4o0Rkg53ApRin|w#0f`uw@lU`7$&2a9eCY(#$ZWC1 zj9~C=anr!VGvq*B0QTFMScuFFVeQGUSvs^@H`J!Ykf=#QJq-aJC4tpXB_nyq*B7b1 z7oqW&K9lphR4pxi(!#$rwf@bMFeuO!toa-fxh}*AHGjDKQypqb8Q8QY=nz=CvbDDX%7Iy}Tzj`$v^x6b`Jc0dmDg0^? zL(f9tD{z*xK;6r)I6>9tcMW+ni`Fe#{&jDAjq=wn)qy_OA{1lR+%9WxRU^lr=Vp4u zS2gQ-R2%U+T<@wl=IUH*_tNmiy(`U^g>V*7fB^4v)IeblFCVgog;`wOi(bQnI>+~g z{Y`{F0m%%rUj$u&4Ve^3*P^B5_*;n|e-N{&5|Cln=8ae6& zNXSakj{7FB5$dRWIX5txnjU{M$o9T0b|DKXRcN8OQX<7-2*c{{SvU)SMLxi*gxo8* z;lX3}F>(MVgl-c6G6sUpZZ7ZV<5tEaYV3olL<7~v6T<51C+L!zFEhk;n&Vjj{#shT z4F*fBxTOH)9YuZz*l|BvvpC0>KLrUrn89C{YpWXJHPf$vGs=MRjWe0W;E!_YE0GUk z3!tn|c_W00;`^NoiQ`7@sb24NCssaouH(NaUaE+)}JT1QoWvo`U6~E7+rXVjn_dGvr4ewkuJ76?!rv;%pE5 z=U^QJcB(oTHtb}12HZscJPc&7D9(axcBL@=!;Frt&7TE>#7?)N4HX)@#lClT?__SdRf;QQwIxZ( zWaA(4a0|F-lznF$lIg}1BD(_uhjBHkNSv=k7FtkN-+Jt)j!u@<`izR0xUyG>J^% zkfI1R;l@>Gk*!WdE?C7_3UFl7TfMyz_ng;h?(|-93XlvAQCbINGKdU`?b$jUHS~cG zn;FVJd5|5h9u1Xs7CyOO5P2{l{3-<fCOFM7{B=c$~^4 zuDF#4nxG7wD;hB#Lr@Y;2OJW!pvP*wT`{5J=iCI4w~|Ur0{iOCSg|G4ROn zaRIPT**aFB69vVZ=xQXRjTghyz zDYrmwqOb15TdQR6#YrmuXTz+-e;4-rTyN%*y>kW@n1l~J zwnF!U2}_O0=dU$03_*4`E_mG5)7pKe@(mVv+)stwPvvz%?KP_sU-j?cEu|)aBx_oA zLiYy=?}EVP1)=b<>0!@V(khK{a-_DJWx`FjPuBBUONAZcD`MdY^GL=|Pq+WHo*6{H zM7=x@WfyaL3E|tuA==_RW?6~#y>Cv|;2Es1S9!kcc!N=exO`{dG2qw(Ldj|${=tlM z86e$HVw%1u>9!2%MK(qs-v}o1WWhFQ5g&N|x|MbGW4J2URC-{0v?hV8!5`<>-A`zp`_ zF@rto^X4YOPyaW4S$*-p=*zXpWHpxIsuJw|5a`EbPu~AcVD31~EXqp?NU4b2^*=lb z?hb76dFMHFTb^(WiFT9ogzWw4kR|$h>yMPGLgOBu2ij7%jI~C9)Ux_fLdm6*;#Sx5 z|Nj7+-%2CUUH+n)Vp3f?f}-Z&?E1oOLNUs22oHfKrDyxQ0Kd=4_`U*1 zW?pRhpWc<3MkUYGZ=UUpJ9BTSotZo2gh^jvg~TpI!U6!Z*dHkUW_0YDkEIS8N~C zz>P(D2itQhoJ0vZzFlJhOYvr!EliIhse2zwxZ52t+S)oj0Z~=^!$Eulo)|8$+e0t>w zdEZ@gSaZK)&o|rcKvg`f850BW!?}a zbr;7WTrhd?JZvK550=X#c-Bc27B!XBeID$bG{Xw&^5$8pqpLO4r4OIrUJ(CtE^tms zgs1-hx)%to$l|}%PXr9ZXWD|-JVOQbp@Y!or3H^SKB{`N8I1+e&BIdxPvufM&b>8G4<8zlVW z#@I?19n}M3Uoa`4;IN1OkS+VXiM4ZZ4DWfgDJaAxUt+1j*8T@?zWKGC>6`H5S`qRr zpG<$EK`XFb_4;S?DNC4r44UCS;z#_WTKl3^Vo`HUT)6F5jBJql!`;gacP|j+{ zZN_N>sAHE#%_aYqSD&4|h|^i}%KgV^`@IPN3Cnedz-7R@d%XR2*l5q^I(IGqv*F6W z7)+xB5IebbFVHvpF5pqz8!t|xI0J9Yn6NE$$s#*_tu#iO7zEbq5K>; zw6O>j=qh7k=x)jUsz?64VGe8^6HgHr*FPH_iYmIdAFsSd$V_y3iV;kKWRhHYSq^F~ z58hI6@2;IOkz07w6Qh;-D9a-E;>M+{JDe1kx`8>bYQ?4B``dG;YU+cFuGXhmQURTs zNE@>4W$F2F^YzWR_fFi7ns<7QA7a0QxoYWl^W*Ur8fXz1e>KUZ zBt~t=Ubr3o`LX$eb)AIndWLF(|D^k=vmC6}M%*^%TlIt=4N3d}-`zCmL^NcOIr8^^ z4K!27dbGnd5lbYIz*;5*`7-{>>!9zDm;Q3gV95uOrIA}K%B1|)pTB}>S*l`^*~7D_ z%qQL`Q{fBYmP@XR3AGFhY`CdZg50kyJy179J8T5EaobUZ-kd>%ZZ#ZIb<8I;zw1}2 zNc&XiPNT3pAW@)GKC^|~p(W!Sd6Fd8>=D^$ZpP!TsSzR-zl7)e?2(07_BIv@3^s-n z_F2yRe%Za)ZbVFkLehA;uUf<~B~ziUOqye#ygpP9t-4#KgJo7a;Kl)hXN_q<>&Q*z zkN8L?&{UB37T+%zw8j!HDco_!aIDh=oMHd5`F%LXj}o`#0wKk4YGS?eBe}55<~a>~ zf+^oz#ugb@cySFT{X%055x*@}azqy$SF;R0RmSFvmR%@WCtc2m>@-GU_sNwN;|2`u zc^gapI^mBifKp^Lxl-~#jE>}DFBU@wz-U&52lhy#4b6jakE}zYD>OotS*7k^Z_UlQ zGsU3C(r7MlEyEZp<~GY+BAW$!lvU@>wmPytduGU?|O_$t&$>Jqq1`r z+Jo5&y#U)niPAc0i8^*Qf`{D-Rd1NJCvp?O(OME|T7?#=pDkV!?^E_n;9CISJ38C9jRSE9jHv(J=9oDeRM5gPx&Dr(# z{EdH9JgS@Kw3UP6=wW<8OSt#rkK6Q+9?b3zK7I)WhA`$*Oxbk_iA*45dU1^Jn);N8 zE?@5d7I^lCK)7SU-agNQg&+NMjc_Jo{E&GIlSb8zo&Yr&+MZq$evfYBQIUuO%2u69 zD_OZbNSAPW>M{@6Q-u&Z_9Dq zq$!ygX>w6V+fG366O%Ex@m`h-$ACN%vtZGhD7}pXc7G_NbFHDgDA1p*DX+|rTI@Zz zTxAqh-HXtN+lOyP%SNI+6%CjBZ^LUrXLYXDc%y6~?59Akmo3Ec_*L!{DhQrE6e{e@ zeJ#1a&DAnb2-W;N=e(ct@3#c@Skr$``{$9v@V{D*2@ww0Cd_A}Iep@rS`M*%*XKX2 z(CyGdTTI;vAw+X1SjOww3N!I`luXx?n02v()HWS{qJ0J@AE3Fw63qds!=-0+8b1=e zHc?>(j%M9vZqr0rk7kPwXX%`!OoH(3gsI#lMjd2gMwQV9-wlyhgn3tvNG%r2E^Ro_ z54u)11~KZEYit09?4xL1+)WZytd;j!2UhYgVvgdWM@@fuCCn^|*`CX>8K!=+ed)Yz zH$wzF9(RgeyS8RLXJ3QpRCOCYGl&1i_;`-Iq5lalvb~(%do4-YlXycY6ag(M3kHx# zAS}H;-v;GB} zr5xaNeZ;MWK(SBj@PO9>eJ-|vk6IbkD$3`eTQndsrBMzG7%gCTx@rETwOU%=hq_pI z#Yw3SMB{y1G2PjVOvJ{>HC=+o2HDijVlCgCH+R?AQ4Pfa?qfyYL|IUB=cFT=0Ox`w_;&s6eQ_btM%5J? z({Wcu8`t5|71P1eq^wEce)wNyz9FISy_hVi-@YcH|LPS-D6L6AP&SFQIf3wt~Q`v+Rn z8wlqDHJD1l`=Dt3CQVy50Opix`wL=RCC!bm`yoythgnRVsmmkzL4(2emyr z7NcnZV-$ZK(^YKjpSj5I*S~p8fc+{7KDe!)vXJ40b(3)fFgA(?8pdHEMaKgDZ140+ zwnwRfY}Tr*h7nZjq6BJ*jjSL~$t2d!? zZZN5UAkFr8J)b6Mm&Pg0`&WD4r$HGrwwL0HK`kLQ^h2ow3JAV zuh#dre>3>|aQheIxIgBlqBI8=<(nR0>iHek5!B;RQhJF1FUm{MIXu_|DJB+&JOd{{ z>si557B++$n~`Q;NP%JfQydE^IBZP*r^2^i9MbvU zIi(F5UuOuw20QqgAXmq?2^vk8B^IIS(|F zGLZ%;R=1`2u--S;%$Cb}_TU>eOu1AHHnBPAbI zj7g};*ccCl1S_xnDjQjh34~;|)rJ&c^eq!36+{ZyB@h+==@o!miAgfqi%DY5ABH8~>v+M5kKeH<9mVvQMB+#+{tu|7@{cm$RwEUl4y7)hP)N}p6 z{rj_tIXB8^39oOFrRxg+=Wclpq<}p8pSy;WagjiKH2+MXO4lLNufkGd;S_l)Bm72% zQ(6=XcIN8eu7R|=HX=jlLxYRA&x1!8K!Zc7o`ZlihJmnLE`UcE{^{F8tiu0{E4f<* zla}m;Q*)WXNUZ-fB-)GiVhvacbP{Ay$m1qr4N$Gf<0N-4;b!#<|1(t38gu z&l?|WgqVaHVV&>R4t|7GXR@=Cy;9c!e_ibqff@45lv0vB;QM8sE^3q@q?uJ6t_P4= zH5Dk$vij#&g~YfBzf-sm{;DSu=E6uNVQ=g@`mmuC{h+QoTWLm)%sG4{dvYgZh1USY zI6H`l$To!+wDLeXAkM^4!_Ii*?F;y{7Wq+W%@ABX$KZ=i{_RTut#V|crs_tm94b3J zL+Fshr*`qWcVvA7BFnSzzr1XnxM1qKVLw{qnP>$d6O#ebU2~93r zhsP^AxgTEQI-w#Y;i2GD^pr$i+rM%B&=pNCVbzqO3+sE9yfXK3?8R z_h4#sOYX@b?U9M-s$#Am^-|8H&Fn2^f7!`lv8`9ssmvklaB|Dupl{iX+Dt-&j~h7s zgZBR5YOr+8KY=<~K6XcFTk|Z63%fKK-(Lr-uG7mp#e{se zA}Nk=t;QteZGa?~X%&(3hL2B(Uso@oIQk#RFs26Ef@bmIIrP>EHdhU|5}EjZbz;)^ zo(?}rCdv;qzcBDAHX)}_TH(Hues8rf*-bC$My}wV%{pBbf5oWX5H^ba&LiK*W9K0G zw)-rMp+$>0xiFl^EkII?^wRCb7kt1CJ)Ws|Oue>3D=FU`$`?e&f^r8~62znI@tM2Z z)$UhSy`F(HODQ3Mw}_C_1X?(ttGykBMM32NteUnynklpHn!5Tv8WPpcsr}QKK)C{i zRfD4ZxRzTzdeWgk8AeQ?D*bnKc(2s(H&>F@X0B-eb<))1%=*n%H z+7m}~mNtWp9Z&;R8Faps)bMb>PS;KGi-g$_X)Q}el&x*fh|Wf|1l}U1n5^qp=-2t& zs~{o&+vMv3XTaz4=*DQT=XcNjSGMoBfzP*r_czAZ^}pL|!tb};cDf-~jcMz!f-TO> z+L{FaVTO91UU+jRQ}c55*xiRB8U(LZc3STxHMOOBZ~s7+f#Q`zGTyXpYRQ#H|*XXsQG7h{ZappC3>?2Cof(s%w8bz_k_cnq562YNBcru_X>^~ z2&nTrN?oP;$mD?eCOyBf%_Tj@`}1w}&NU#ocSm&FOOJMa229;hwTyZt5B&df1w({w z_7eurL2uDz(%>n@P7!mJ!@F0?6XyLnu*KDr@nr9!>L=(nbSreMk+%A*cm+u0{>AD< z$2-I7TyikINZAqST%{L&3BBQjw#=_55QK@TtPAXfN--S|QS*Ju;c^(T8=?pRu@rb2 zRhnM_Gcon~!n!n|;?L{dPqe`Ln%)Wa1_keYHx~5@T(Q+*=%Ll}*l-4IGCArr5Ghjo z7pyc%ZygFtRTS$^<<5x^>Z@YvcS5JD>F5o%RXaKKs&fJ|j{Qx29(5%JzcKfW6L^#xZuKXg)GJ zO}kjd^A^!MfH=!sEW86TK1M&jebL*Z^y{WvNGsf?7?HTP!y+mr=;*3uB2q@GASzWj z%S=HW0*dZG#cEinCE1pNIwx^VUlm8qpjEXBRO6UqjSMx!6QLi}O+Uno4rUxN&5@RS zTfw)$*Y_7gP3Q0*u0{M>ia$7S~2=UF%h;9 zWsiS6$X6;bLQ`7r?mfx8Cx6}`h%A-M(RzNvsU6;|;l)8JVRhcDxh)AJWd=aj=x4@q z?G=W8GD5~VFjCJ?e^!)Tpl)8@Bv?lK+d+iGR0`M6;2D@di0HwVj}kt1Ef$kAl$H(L)9PZLx64O@IOI1st4B&OeTfcVvGDtd!sd1YOBXhSjdsE~d zysB`rUr85)lycb51{$!5sa-8NqjiE86bZF!S;Pb>SxJm>c@Qlv?CD36Cu)jNGoniDs5STBtCb+YZeNjakw_ zFfwh9F_z?r%H#^c*^Jj6SN!ix=y6ea+MHO@yR`5c=@PN=KD;y+w?&2Y#B@YS>ZYv8 z_~&0&5u9;vEh?#d$6|IEJLPytjsXSE~HLnX-d4 z$KKu^AkVESK%-54mp}fL%}Kxl7CDVV$VS*xY0pbL_A*i-lI>cw?6}h@$m^vfdw8vn zfxH*uRV45$dvwv{-;f`J?iPH_wN?ky`Ly)fVB_F1X&sA=2!p2ATs?E<-#cD9sDiWc z(EBAqM}=kZjhf$4Jw&z&d1^A6VN2(fltPyWV$gJe-CwDx;2d?VQo|has!O`y;-5S6 zc>o^0d^r-jb=s=Zhs8xLWLZ75$iiwZ^)ntx2`^|FYE3%MN9k?><82aG2{Vrt54Y(D zu&NuKtk307--3?s&Mo^TU2oDclDGSu&FR>wuW{GsBKd~=;tbZCl};Q-f_?|CMD$5{ zlUQ1*7NX)e%<=A2c@qG&u`)IV4B?f|jGm@8^@j35bGa=~ zD%jK*iX+uNlCCnuJFocm!-;pMyFrL7$cFHga5P0hUs#I`r^`PM4@l)UIj3~n0gDTt z+R>#lkV*%RE3`i_JN(;r=r5{R9QV1 zW@I)>?4#?_lQ8(btRsq~;b}s2DSBecVT^hi5eW5GB6^kSBpMTVUv6Vy)PeFBrzq-VQ(iyCLU|G6ewXpJeb1 z=jZ(O)pr9WL-321tevenzfNXS)K3u3tsbu2ShM+smWtafvcC)Bkw`a2sR3A`}(3*5e_bkQo z3`3JFv>rzhg650u-s_)=2Z^x9Kd7WdpT9Otc^)II{HoXX!g zRX>rdR^vYwR|Hu~SLa>yrE%+vpUfDghfigu0C&_r{WR?BhTk|6ebNTzp|I=ScecKU z)}foSnp@XmW?uHASMs99el)5N zDt?&<+XIg%e^_+^@Ow%w%XAy%)Wzyl=?JmD)m37ck!^rB@C1i;*t*qD>56sS_^7Q@)Bffajq-KAktwKr$g(PEerJY71x7X% zig{!ir?Tt$0jJY#C0sx5!D0uX<23G`+*hEcXDgN+YoI=_M@N_6oo{=0Q}Ri;4}e&w z2sDS;-b9~vu`JvNl*@e->dm|W!w1(0GKbln4FZOQX?P#$YnKS)pJ5h02Ig?w2bGUQ zXJ{1%0M`(0VUoAx@KiUo%JDxgo6~kfrd92nPm8>{(bU5u%1yc+Lk|zEbVv{g`dyEi zeNgGf1B52uH;b8?FGYGP!y&h@xqHu4MuVRA_K&Z-!;L~Mr+kAmB-AUU9(-(tsil=H zsx^HyUCsSwUsDeeeYVi8wt67Bf=!-DRm!=vRL@;-69xH@6X#lF)+c_(M5wj2-8JW8 zF)E6v-0fkMRvScQk*P-%VjD(6LEpE%L4>V2M^S|B=d9|`7a*9oI=Xn8g^lgcwZ5AI{+j8sv6_nn zOP=tvZlSU-I-rVYkmVGF6%MbMP~g&S$bS#SUJHXtste4j3rMPvd&LAnDKMRuT!cL3 z*vLwCmNrv&mdifr4YitLCdH0&OTt|CUz9Jx3Uj{Cq-tfD0o>fLm$?RCy*t@iACG<$ z1hqSm47F2#;YonH*y*(u(0rxxp&7Bb&qPQS`zxiS=xCYsS%?{HHv?+4{k@aLR(uV_ zYBbo=@=Mh}YIMtn=RE zAXtjqFE0pGfo_E?K73YIfT^TiYV2`NVgPtViyS@{*T!w4ZS#EI=z=(oDm;Gxw>^Ss zlG-u|3)3{>)SbVRhnJO+%JOJo);%4J=fQl602UEnO2sF}Kx50*nAGbH>;E{e?aa+~$@DL^U4EN?bxiDtkbj z65PlrIJFlxgV4k0b^Uew+TZ{66}Qv#_I7u1VCjGJC8>O<`BMA*F4&NmOB#rW_v_S_ zY9pGr6~!TRBd>Ea)pZmX=bx5qBjWEfQ>}!|+0wxXCY6VLwkL-`lCn^o^1HW4IL>|3 zidRJS_D_Q54a$Atp_go2#H)E%^9H=lXI5t5xtxJBWy6RY;5WF*Zhs-lGh1mGZ8hvA z&~1Qm+D6{1%+{FqFCsMOMde*cVxeWRJ+JI0wZPB(%~lBb>V<;CVel($+ENLw z8~qK1gB$+OmS##4s>KVZuMv!m!xzK@()0#V$dS@4{PkV2a6ogQfSbk%k?J6E^;@vgN2@acud25_#$;! zkN+j2YVR-$zA$LeNOUzAJzYV^n~1Fl5ks=I5Olz0$gCn>BoMkuc+Q8T zOtc)iLKsexwrCzxV(u>alPI(vY zbw4AL&6HeQB}e&ZKsl#nP}LyNDgt*i8G7L6n7<1KcfiUXt_!6dOV-~_PDU2~%=<-tHpmZd!V{JedQk5ZBd>Owl46MC7IC@VKm zBiNFFj!d^KViVgk<~88n0?Z>}R?JKuXOkHws^HMxg3PR}bO_*skQ2#}C?mn{wUD%x zVC|N$Y>PK}f~7UT*wz^6!QT&uS`A(|En?>*GjI^ZCWgVad43!t7~0V@$nl$cz|dxn z*GmE}Yp&9pUHr2zdSH-5U7^`WX^aRl2j#^G*I zX_E1y9OjD_JKhRA-|-SUhsF|C|J4`VO6}lh%4b}Yl)|mOLG7VxbfG=^dIUG!*7goS z_hP5tgzQn{ZHug|RozHVh+c<{as#to3kikQix&=*+S%Z8q{BYUCG&g2)XQ8t?Yc_7 zWajxIr7@@QZveU&oZ{GCk{GBdzbd?CB|OZ<6@4Y%C8w#XZyigo<~OyS0s_2uXg^G` z+)=@tYI<*mBY5HvUa;{jmQw~-H`z;O%vxS_2jXH=)?au*8O6 z-6zB82r>zbKHEP(0A2fY263GfwZ)GwNxfE=N3v0-&P*c(A*UPmw%eKT2EU8Kllq+| zZ97(t_l2)$R?WIgEC*3&ayH4;AGRLB=G-a~p!f5=1B7?W+8e(w{aNkioJJ6&nvs~0n4-5DZ)C3L?PdMv5WzEo ztT^On^nqi>cvfFvhNMV%9IA7wHVrfK7W*+8>20i|GPt9zN!yRJ4 z^{;$Lz{T-I2$ z3v5`$uuPx~#x!1?JSMb$a!2HTAhnDqc#Ko+QLIy}wQ!EyY{m1tOfF-kpcU9B(DRxi z*+kb zW|^*?r!_XvZ&-`Iq^iw8Gs>Ifo;3L6C0CpRQ<7UfBU!?w-pw@yi%PHR$lC|W!bA}I zFR|@Ey;wCnn2Jz!(&zw9q~ zq*-i;hk36?>ez)(RLjJy1K6nN;StUQd{v+nK9#0mB~+c8T$!6xzMEE(u+U;P36~IT z2E3b4%(0s@5T1=^?RjT0l6#eFL?q$DR=8-VDBvA6W<#3C1E4_hZ*7Td~h z)GMt$eo}rc&X{I_&qBoWuojpr?tb)Xk9eq)CZO{Z4nau~=xq&;{jCqXjp-tGDgO~6 z(QCWsvWh5)f$xti!YH1~R3zXP>a76=lEaui`2@b8!C3u-qs16&nCKWmXKNeM)yQEx z9LVYWWphzVLe5d~%l5vTN${#)tlgl~MD5ZBe$9#@_jULt{`DxhTo>74Oy?e^abKBX zEWq@ZVqoco7tK*1hiG;J>7}Cxnzs`b(?v75NL~6ex#kaO*hx@5n-q4Y9Q>H3mWFl< zv3g+lOC}yJ+>e#$1G7NE6KGI#<^pG~n2Ov5icxICuvwjzKn#62grX_YDgR}{E+YSg zm{yJ!rjQ@`cUO}60Ond~m9@soMe)JJitYy7$48bP(?%qLwim@P4DZ1H=&F4`3Ml6| z7P>lePU)T|u6D9R&i(r4>ciRonz5O*A0r3{Jyo)Ct)t#0TR@+rEkn&_w#|~DhkV|+ zi7Fv+Q^Fm{AD&TmdL)&hCa7c?*N~DT_bv z!Q=81p$Av=^P;5SrBFM;a#GC%(?RY<&+KIB38k4Ui`Y;|VE66p!NMLD3b&xoW#L7^ z_k?Ih&;p!`v6;+mx!n+IB;$^6dW|j7tLzB6>f$B}tBa^K2okU73>2&kXLzj?4Y2`G;pj5o?&>eE0~lXqUP{jnyzNJN zbABr{Y5X_`btoz`9?2&MiWCmB@egt!=il!eb%*N^&{&oZQYaTnO?Be;vT&EW{}*F- z85KwHE({*G;O-7VgIjQSclW_jryh*R@TgGyi*$e^&{@9&+*3&V`q|YZ1R-+U(^1tD_nXF^)DFM{cbrv>9k^JVVD?c^{E1M z$&j@~A_6-7t>q{O6Kj})zxR=H)q6A7Yx$?W4isAm9QC^#!B81FMD*zvQIk-p#A1Av zMo5BP$V}O#f4+SGE#xoE{{-~^H#}V5{qODhL)rCdr#_ekZL{6q_os7h_rJa7nyxMZ z{^ty`j(@5e&p02a?;mkz`$sb5@bJbTGyGLP2tN(ieGy9+K>vu^66W+Hb=DS?Pxe^Iv0}?vf1E8$uZ?^1P_4$w@h%AvXu5cIOaPGuIWPd&MOU zA_NgH<1^RPa%4_f2p~Z5n)2cr0){t54KQlma2WRV2c%9VD+a5p`i=EhxpM%{N%!aT^_)mK7F{i}aVKWTia@&jEq}ADPdp2A zHW;~+hO<4NXaOHe?{Cxzy%!LoMG@!E&MOsbH!rze`AGd>DaYb=+uoc z&#M?WtA@*KYSmzaV4loTn)$IugZ;1TIDHTU3|4jBs(56D4xRtq6P9Z^b`VtsW%IC(-B<99Ql|6bc}G z07}r`Qv6@_&Vj#q(sx@O$sC}?m#QiU>R#HCHGR=MUCFwH{2Z?aWb*RR5e*&5!<|ZX z!pYKd^4@YMoPC-Nda+ETR=gC2JS}_rkYFq4THKgK^8_dxq)Z%q(i^2_aFc zd0d@lk=QiyI-?Sp$&hx*S6_nnW)4hAXaclqidpeE2TTU__`* z-tEuRHfh?Z#v*rY{PdZ10PgWt8Fs5fL8!X9b2wBQOZeq9red1Kf^^lg47y3kGC0o{ zcddB)<{+m-Ply)S&t-)HmER7APJesgPlw*9?gIeSeP zhWi^gLF-Kq(JlJxDmWzM^<8@peaJ~GkZOb~{o)o6OJ9p4JNFCP2Jw?}sZc)#3*0x~$=1Vl>Z8PqZ^6=o6oD%Gp)HYp_7%6JeV$R-hz>+Epp zL_mi1;~l1eomfJV_~>ssVP7PZqfwKiB~~zKL)8F#AoxkkA=cO-=FxzsqD2_ASluL5 ztadvnniq9Fb)4-^emnN@wszX>^xu=21<@LjG{459>eUD}U9r(>Yq(K4$2|xzCd49& za|}fq>|v)=VFW^DMB}l!1AZk(;pnuX%7WRW8ya|SkbblYiU3|`$PL0L9Op#LcGQ+L0ylDiCz?H?Cat5ajmz`sZEFEE z3=^ut_@yg?S8en358Q&jTyAh+8H`d&<4nJDa`o3JX@W2 zwmaCIAm`N7+~%g;&#h6LJ_g9AEv2;MlZsB+*IR~&s+Uy!94Ed{f_YtvEs)-%gK9y^ z#WDFnfb$b(EgZr$Qc&`t&QzY5+nDs$d~7JMWblf8tHrMbg<$eEgpf0nZu+m*;vf{z z+G+z|cT9#Qku85_PwL9)zuVsI_&L@cH^{|sxUsc(gN96Q{dI;w;!>uDstKuVmWq)z z63~27Q{zug0_ciPznQ!QEFlNsI} z(=bGwAB~L6IaQHGzO5GDfS$cFqVkzUwAB$CyRiB$5$Gf^hV>zWk|)$H;uahRN!c}v zbOTsO^i2a^F&wht9*~Z;OkfmdS1Q?Q3lv8?1{u8+l_qtLOhiPShr3PKAV2K!bgAGS+lkm5JP0wfS3b|Aa{MnrI z8O%kZc@+GPBS)*AZ^84~@qdka4nF(u={h(*qq)~>SWcg%pE*Oaz<&lQhbI2_B!5(Z zQyiNz#mQ)?uTg(8&E}{x_l-eisq$#C_(2H98;HWzNCgf)7U@DEeN6pUaXEvpy{l&y zbQMbj5xE^+s^+7$hnJ(Ro{vj-DG09KRAD3z&{^wgeP)l}!9_RPH9avhiy8=r$t=|r z0rtJ=WO_!H znRxfZzs%4l;{-sPyb;VcP?BiZEL~LfV@c=UhIzE4V!xI;rir3|)N6{$7t1EP93q|! z5SZzTK7Hh4X@(fqf#h-NWp+cg2K0aBXF&YSEHcnNmm$F{C%Fy|_IO2=7ABSribk>u zj%yO0W1wCMSzkzVc(S#0(=-n0SE}@(@)1QwHket|B!)qZpj>Y3bx38U!kxZ|sTB4+ zidx!Ds7Y6I8=MqKeV{nNi9Vk&g=n)#pFaM{(K1R~ZL9plVuScdeaw!BW+}55QXI@a zh3HzB6W_5Evr1V*d<9pwgF$&OA~`ovux~G$N(@y^9`YBR5v;-~OgAW(-mS^F{#`(# z|C>r$icFbfk&MjUo;)aMnvT)r-}Cj>KRH~YenWT)!hBVE#d>cn4qC0l=zw{Ap?TCG z4}N5t%Xgxy{iq5_Z8u)Im3!6GbwGx}girY!8ryAN=2mb4`yQAUN!D^KhYUVw7g#W6QG^wvL7Wh17=_(UlH?dnV3_<-z1Pr2exoJAz1bd++Y)qwIU+*Ku zL(a9{aK3YcjF;E5G8GT1x@473lfXcBxSskNG;M>`A0q0Ev&Fyd9R97Tx+(QXXPLuI z1b;hrDwznhRnTn{!26L!4{@vdH*gL7^0L7x#85lf!5n8WYWodzEjp$!zOqivZAR~? zCiuSLOqUTsST0T0FlXimf~U4dz%`?uDSTVOa$-CiAo~dyWgMFi49~n1;V;gQFmZ|R zjGu{FuF_RGhDjL(*$%FkcO6#7i&y`LpA@c%0E7(6HX`cG0I-CFFQ5t3MTuCVf^-` zvj?>#6z|^~2%+4CwRd{uHe7>ZrCi}~Syz189?2gG@bpK6s)Lwun-)9XZTX*<5&W+_ zSpDH~4Rr9C9OP(3u2J2_TVopfqL7;$$oIn>5tuD$X8H6GY+WZRs${!UgZRcQM)2mr z-csUBc>CZt@SbsSL!W(<%;{-n!)~WnyjLq|i0beG38PuFF}=O5^op?%22>)cXtMPv zO3kij$(VJQSCj5J0PO$MR<5^x`>@9)ZQ=OMKk5Vy*M94g0_L1X*jwV^8Z4;r8!043 znzY&Q1MGlN=FpV#p`}tdRtU4V7L|uTcc9$wbB&`vX9tI9Bv5F{$}cAm478()lEb2{ z{yh9>GK6-iVP8c)&mum?#$rSG!r$K)RcKd*7LT@K^A(th)fgvL5^Qt!U-uXr?`?)7M^%=%kJ8 z=$${08RL%{Io&hvv3iyS9PaQGW}HbCR!vmSBB&qze`aMXA}!s@-=*tmbehgihqQ*!+lJEa7@wZY#zO2yp(JJ-5_?6?di z2Z0FmbwBAm(;{qEx}fobS2)|qeslKyc)7b@QU?E$R}FSEgP;HI<6{`>Ra{qwfJHwa ztP+06Pd}CpFnJb+83r;q!SGdB#W)5Vq;^v0bD>o074dTQVJ*jEiu+}UXtgq={pA{V zdn8TgRMhvwM69~YM4YI*AbC?yeZzZO;wsfCId`AK=D!R?OoVG=)CR z>(^*t(j4te){uPlh+SrHz9z3at2vyuYqq zo(Xz<((mVefqK6aWKc+>a4NKRUB1X&5c22xYV1HPc|v6=wirj*?0U{?JH8BwxG0nF z1@>M^sC3IwuEzYeq3W28NLAWl$Eb}npa>Dk%uSv0J|HJMo-L)UU4fJ_kys&@IESK} z9Bo7}5gi$Wvet_v+$p~lul_8SiWY$4n6=sF@O|}pKM(bsX1-qnXX+&TK&_rEdfhL! zx{6e2S$<#7b8b`|q$rE7MiR3our?+;j*T$-AxYn;V9})i1CUPq(9>bxx~EUkrGegL za05e}E=7^(3w*7fen%iCWoPi@2qzQ#&4xs?*b~Gcj1p^DkC}bGANBnEgVR5H;Zx)@`n>7> zG;94#n$Ff(w6$(`vmXB;lzPrh`_3$47;Zl34@q4WlE5oh^76h#qaKPKnGmYLBNKOm zSt3$6ZMIqIuk_8P)l6?id+w*0)gC;gEx@QO7_ybJmBQ}$S702?Jl}7KK{#N3TC#XP zLJP4nIB7V}*bg?SdV)}>;b&m>8rO8e2zQfGiG01#N%PpDnlrP*Jqt3I?O+39gfSDJ zTj>NbJl}f3x}XVGG{3I6Of6F#0FAq_-s~DQyPZxX)=Zkm4D+_@1PN1__4V@Kl{ApNjXC0f!2 zt|my30zn3wMB3jP&IA@JooyK`}A7k&r`w>0n8!I+)(JzQs zc%Tr|`V0a>I{@&4u?Gc8QG9 z^UE*58ppiPr{e6O5#<<_zWUt95WPg$x8u~sW$9CH>m@bo<-q%X-G)aIdElT9q$^k1 zCIYE@sB~n?yP-Nm!fMMwGQFdlxO`zI`+@JuGk*L$QBj#+R)CDAx{UYWfOAUnt2&zo z{E&(AX$+yyG3tj1^$>a~E5DtKOyPI-uYSj~9~#2IJIF1`G)UjSgRzj`7jq-~@$#Jb zdCj%w^Q6!u$UqI<`|$_p|5mr9g!L~?#wbMl>`mBFF|4?HUqVxErl+Oc5U3lL)B-6g znlO#=SlE_EYgr=9LWBzfV)IDqFX|b}2%kx@^CyUVPMjGy8Eb4<Cc*ch)G7uGjQ!WoL0vfLc7|VjJjg5ule`JCUB#0HI_! zfHVGD&1Z|Ill*%>aU`*(B|XX9{)eQ2uSoa$ANH=V@S!XeYj7#-(flF3S80q|>R^*PIFPpo}$aa~vMz3_XZ9hkJ^xU|qN7_0Zz>gQ?o z*F&sJY5|hO@`}%H!woQQ>dB5GyjQXA?qv0bexI|jCUC4vcwTMQ%*4XJsttB(=j0X& zpO}p{+ihbuo$60qXbqtmqgMP&dHO@*_X%w@DRo~PWJJ~%RlU@B4AZ#`<}u&|t9$;- zA6F$OmM0eImX#Q<`rJUN>B>iV+R!$~NuWN34+L~2TaJ17c^F}1C0&YHr1Qo=yiW0> zCxpehDm0&JDm8T4RVLC~%nvz+EI!27CI8=jHY*=z#%Kj~`cxq^$mu_)R5|OQtTv%< zH}i7J112fIQIPh&{k}!mBS_zX$_hJQO&k%Zfsvp`ezjazt2|Xqc&Cwf`0*xw1lRcL zD-(3k)dPe(m77ca(@Q)mav~zZWJU)S?_+W1 z4rMN=`PE#6>AS^p@5Rq}k?&i-rVuVGj)BtEotkrWdva4iz47Yh603hc-JM?`gwy8E zt)AsZ5*MIOVaLdW*xewN;f**r3yRTpX?mREb4p)P(&;=P1lcOxbzWV)uL`vMUr|4R z6Wmo{fOsbViBa0OYLYLV=FX0Oo$&6?LTngUUHFIyI4C5r4Q{(tj)E#R`G1{YaJfNw zk-dfC5|7+c!I_?Rw!FuEfSPi?R){TUhJpl|R?HhI zQ;m#ZrnJC#LP^|lg;K;il0gd-`KfZIb`x3f3#jpXyAP9nnhNX0gRV1np9{6&J^VU% z`T|3=o(93dVPlYbKC*=TcF*&J0}RlhX%~>8u;3#4PDta~riG)tf+6-6T(0a6*Ugqx zXO|(%R#@L9lfWy5wBgt6@SPh%4};dCQO~3A4i1UgQbD{SZ{o<|Pm=5&jO~4AV3YT~ z(pG4O)Y{jF7Zhx?noY8IQ=LiHy_<y=11-AP=yCNkxiFMSz&jkElfOYL!y9j;mR`xOL)YC6x3N}G98lN9z=e2Vc+U%;eS z92;yhS#&F3SJ%JFtc6-!s%7v#xEL&@GDMEk5OWeC-?H7hPH>Et3+vZT3VgGu`O7na zY~32EUj7g}rkNtH7TkHE*sdJKxN-JdP&sU;UVCaSv%~<%jjJ2t)Hn3sMMEvs#oTmKK zU2-xy?cM(jpkHuXY^#vQpJ%NX`~b{v#)dHyzxU#L=FGg%J6b3mhw)J7_*4V3-ZGvLIS5td7rrfPv}~{jr2-0E zAgbzrs(Aexl#ztUQ|cV)6w~qM6aLi2UKxMS!)2r(~4V z%3(Gvf1EuFp#XcN3UHA7xYzFxBJ9;8`qb2O5Gpj%+seAgf`ex}b47%B>jHnJ`zrPw zCwOR^F5cx}m#oKPox~hvB*DZFaCm|5B{fOn8tw|3tVC@rzs{CuI3Db6gfeh*U|`}W zsatlAVI($x(KM=aetgDopc~G-ZHKP45+GT%L%>LNsnEwMR?fL+hpTZd`7^_Fct}a? z{tvM$U;L)}gC-3By`}jv&Pn;>`L~acaCcPi^`T$hjkWRxL~kmQSjqO+xIjb}9ljG@ zSqmD@4qJJ$sU@|)bM-I3Llu;#3bvbawR?qh%n`lCDK4C8x}Da$tOtzhiHYhvSP{0X zWEV&bxQfSM=S{d1Wt)|63s1k27PkvZVHOu>osl#_Le&;m(tnegNRKZ5_;X3WMsLZUo z)sK)$2W>-Gw+_&%3JdrlQTQstwEf~mM7)-}|LrZlTD+31?_~-UQ+Sq}_%wkpopplc zgOMY#8!WLAQt8(n%;orG}JLH6RM@Tbn5*paQFo*0d(7TH~*jqYt-)g>HBh zeBEMfa3OlPXWTSDj_CAASetVEa%``*)ylkiWx?as;?w||`T6qpUJ0gN`)3miEA{_d za<^s`F7jL~To6dpZ58sjJszxIhlTMjuAf2Yz2Ix@gKLhndZo7|@Fj0l2>HcMLvW?b zJH+|uH?G+he7~aZCK7sC>yEB#t;KfSe}PLKj4nl8#rdLRxSk#j-+Rr^Lu!IdpFp#t z>P_Dp#iomTai5VMIYic`$A+~e)DNs7oBs83jw{{y{}Xw)3#5J;@7fE{nhh!ZOt?X% zJtfP{x6_cr0>@JxgvQjhO!j0A`V42iN11Yd-j##?Gp^1XPEa}Y!O`;1rj;?+5Q^Lg zJMBr9s%gK84v&%#0Q^HZaEp+$K`AN~53j|lDw}Yp$HUd#tu5hyyBXPEAxFLFQy+vFty9|Tjv5A|-!$9uW>7g09uX!&?+a})E`y#=TExvw1!4VZ*|GKLvVIK(G#y3rE=>jld{&OdW-k}cjCQ- z7*6HgaAB%urD{n)zum~^MHR44EG~TW|0=ubaS^@}zH&B?B7;HDp6HGnL4s*BmSu_%>?hCX#$CEbax{3FAmB^IMs(e(pc ze2I-mq-*BVf+!veTeV{WFz5${hh-{XJrQs?T;llE16gkA|M1mWPsiTCS zWrO?rjJ)M4sLzal?18MtR$?uoyP>x-c>h2Jb;+4?<4))p_mG`e+LfGnc1dzha)$3& zpzu^QH~bEQ%@MzCq+ILCe5?(~F4cGQ!H?9?>N^jZFz2I|!!(?2V?lqzJ()|!!JMQ! ze)YNhd{v$UL$)TEDfhcif@x@nm&n#eA`x)yD7d8cEd+MHuR)Mdf%^2Y!;;NOQTbXA zR;30cL{sE%@LB)RH|AX1xi7L@ta_emZr=b^!kw91|en;oI6;?-VF1HaO*z*!@mBC0Y?sf`YiKAh*0=e^`n(7nGDLPppVca zm@uX(?)=7d1VK!rwxVP90C)`kZ9QorVt^4gSi)(gCummih;7`A|huv4Jcr$F*^2b<}Ao3l$ zlj%l^&#jDJC%l_OUlW}0f>PsGkPiLAoOlC7vL3mu(g7^j8? zF5V_1+G$Rjgr+6dUx#Z%9|}kr<=pcROaj$FcItz8a7^W2nv%l^2eUMYus*+R2}_Bm znCkyC0&+M(4mnECFSj_4CJ3Vc0W&mLExEgGc} zg(SwP)BbjI!X6*e{|9+5?ak3# zB<;NO*n6T0?N%;Ld(eMt`arG>6Q~PwY&{zQP_=le54$@+3LXlMk0 zGijcs<&;F&xds~hCBMvk+4?{m@!Tj3oaJ%ilYTHE9=b(I?-GqnlKi_c!Y!o!sGrx{>=GF5D6ra!MI%4R z@4kwff?vln^apFAUYgf+h#$D!(SI|(kXdNY(qYmB7xQnxvf|T?X@DtRDIPqg1j>80=PGs~u5S#!3$~dX zuxvKMeX*NMOMf7+NFo){q z!Z`GGst5X6m!Z!tXtbmEt2l+W+IAySuFvBvVd3LEWC|OR&f<%i+UX>5K+zAPC?1UJ zp{;z++iRFWt+1XY8pY+6oR;`@Lh2uaFGK(hsx}iQ6ty8*b?6(MRLxWe8BLj)iMrlt z7taBIiW4_@R^R*aY7E0jHOlZSv4(d|eXc!G4OIneF@8k6$eQs~{76zzUqb=wDi-93 zye_S8g2PBA;KziKT(xg!<3!KY6d1O?>Np|PO0BY%O5kVo1}+nub5gXbD$b4<;m^hd z&A*3yH4`z4HA1;0QmONglqPdQY6K52Lkmi`@V*U)yQ3R_wBF5^U^1Dw?=i%P=vh?C zTn*-+C^Y#!421N*oc2Uq33nW8ez2+<&s)BW-MQGhy?QF#~q zS7!OW0nv-~>ivnLq6(qwE3QS2^M_Ln9c*jn6ei#DM~kX{R;&s7-s9W-;9A6TOQzU(ZEU8B zf8CdnCz%?e&h;_6HqR~iDY6K27narAegZmnB3!z<7O&0~SUR?2xTUrjER`^SeXJE4 zAXegPaS1T95l`N>M~%f<6bJ|rAY150kGAAL^s?4(QeQP;Wo+k)BX?Di$w!>GH!igc63*YFZ*HKylL_A;t7RgQ`!cDs9zR%HXZAy*+{4<**! zg)q8oNx4}Jr~`Z#U+96-%YU5VWJ3H;H`|@B z!$XUvy_r8)zrWo&ZZ(jZzQTiDR19ZMRAnTlK3*TNEoJe!lV}9+< zC*tc2l~;DvMT|+%3+Ve+3|(CK}nw9=3yJ)s`v6 z=+1hG(+LoE*mGqO0n1+??e)pXLe4ed=Mz6PukA7kXler8;Gk@Ymi^=@?aHe&=PWs> zfD3j4Kz-`f%uqwTfDD*= zD%Qj3+lTx@FAq^7F>`lx%$9*z?owG{tWtd9CnYf7%#bF3uDmao7eT%0{4Di+6mW45zv&2;Q0xmzB0EK63h~4exp6}> zqG$?Lt;aXyBJnH-et`5%E0Df3 zy30~&`3Wk`YOMGN$ENRHY4y>4rl(KA6hcL5uF(ceZYA1L<0Lq+) zZvmKpQw`AusF7-|4mr2go||hHzCC&c)90Q(%y6^1ZH&qhSJcSs4n79(Jno#%9ldi5h$f$xsiWR6u32LPfyIFZj4M`VOp{(!qkX+dcF;3^bM%HiE zV6c20wDzu3vdOn&poMhkwF>w_mH4@f%?m-60C=y|))EJ%KYL z!MmA7iCo|Sjf6|3-#(&3w6B0z_yGOsQ$=~?`<2D}R8-&EE=U7hN_lF5a0!`$a*GW| zr6YkdZepf+KF8Bd$xUXK5eK`eh<2Q|hqXM9i^YDGu}f@5X^gQ;(sZ3B$?w5?h9|)z znH-Ahr~HmAn`Ov#!u#WKE&dWmj8A>Y5Ok2?WTpnfE+cVTQH_Hb}qX3W&JPr>#! zJ=_$gDk5w}+z6JtD)b$Zc3)S9}}~1fM&(7+BK_>>t~t7T{8`_Kjf?}N)x9lqV_IM^Cm^=%<3}^ zFGEWBv>C9qO@~nQD~mfQUpE;#I0m&(Mkrs$E6ELlqoOKf^`JDhtPjrkS;0j{u*5K_ z70BvQiYWQ*C^`kR#P`82nHz=PB&x#{&IUDAaBZ4OUt6v=1;uyncWfi_s8d+Es>4Qj zCVZ$Z(<^((<0CF$@&~V=trls0;^nPHFB_T3(A?cCJ!xqTIji>1mtZe>z*)isFq1vA+BSRSi++g`++rIV3*mdl9qbwXPR_qMw?rcb2Gr{aS-5kN5_ zBd8Bqz#wkwHS+c<#jM-4bls->rb0yfjLr{XvT;N{6f7k?6F=+P4ivCtg-!in73PS` zsMy-7{lGIQvSWinUeHGWR62>Rc6AL8g^+Kjcqx+1l@t0pa4o^o(C%k6+*0F1IMWLo z`m25EBWq3~nw=1hAMJb`AOjIz!u;kVVzwQxibqe*;2c9Ri5`ZIa}ZwOnzE@D{@>uQ z`j{3bYPiZ~M*C&VYI?TJSyt1Au)>a!=-T-V*`)Us>f>^89+x zGp*IseYz;PF<7L4bQM+Y_SxB$a+%RI`9sQZylCu(qE^%0$Yo}uTAXl%8PRDj`DOJY zr67I@?tU)tvu%u+Kz_WLeD)G=lA9;qHr!a^`ZPOFB8=kVc2h}EktU)gKUd*jKndr9 z)QTzojd1iOIT!V(t!|$PQ}~aZ?j^}VqCpen`d^rO?Ne5-lYI85x(27fWq-C!IseC6 zZxZ~6WCOPfKZW+E^LGE=3AXS^?0-+k{~0|a=oY*Nd`|PfJMK>q{@-BxQD84dL+;t6 zwL86>o?Ed%_JbzMw)Jj1f+FH-A?GT%gnYG0I_$4F-8j4N!?JD%S>=RM7|2^O`WT#r z$5n!C_fef4<>BwA`ir%c$833qyB6$6&*WZ{q`=^RkF*|Pp%`kS^t?+5HG&(BNcjb@ zp`J|lO^bj^!!ENjTy_vGUM_%WecHHzkV~@f zs*=k%jO`W)LDWXpk{v1dnTXq^vZE;rvGWx9s~Ez(p3Rrecl`;}BW2FqPu8y-Rbq*X zBZht}VKrJ(+V_&YX9I0KAiRN|9vlb5LDBdCx*$WMbpRrV8RYrq$V;$CmnSpnb0oc_ z_Sv|(4Vjuz@A4)wT%LRwV$20kaE1MD`Gbb^;)IJ(V~4vsu!?|~HJRx)n-y;j5$v;3 z#(Xmh=@dXnN}-0P!MVXCH<@Yyrr1UjOGm9sM3%e_R;vto)^F^q>KErcgZiQSE968f zGDJeSi<%K31_br+o9y3vjYS>X`VaI148q2F&I%qjb2H0H1eur}@pE#|=6;o%&2C8L zt`t$AY;LF*UOv^8s28N?)z~aq_ zBv7)Y#{~hGhM(atM#}InOPq6~`=c^(lEsEsE?Z${FO8OzM&H*BIol6nzT0AHsKDC# zd!!@i!*H*?4DJd4L#aVbpm|<1k6_gdmx&K3Aq>lyW8%=$njR^qA@(IR+ajp>L5w6F zm7JFLj}y4ZsyQENIVL{tRmRmQXd>p5+9sL^cFgJGzfH5t(B$)(toE|cN|+>?98GkA zh0Vv!P^gx%{a4mW<0R5vpUkNPra)I>msE-hio%iVNxOq9Jzcnj2Jie~tgk+#X+DR2i_Tvx9Cb;c{aR>5b%S3o?JWLYk|49^#4Ax$FQ&R?QE;j9sZ=K%n*^n0-|reIrmiK4jtX?;wP^p?;CTi}VK0Rl@258QyXmHsIusi^q#X z$pcMGgN{p+w9My&B96A#{M-HGL}r`E>RUOOHRvw-no+G4K1ki{a^!Y;^Dqy7WD=BAnoS^C zw3o@H!_1bq?JNUhh z8@LM4|LX+H)g~av+SQ2=$!7S$*@0(V$JGgl-yY=9gR3uWpvG(A29azp=O137;moT@lUxb*lyDt9~9BWTWQ%qZUAmIB%tIEruHKmX5#`CPoJfp$ z;Sd$LUUrcJq8P@dObD`2!~MKX?sJk8+f(ZC42nXk>F~;Be+Zu=V&2!KC{!EetufUS zw}L*nuEZM`Ri$fMZ^Ir;`KQy%ya`dpM+!Hve`jPYux|;b5j>MEU={E6#(fTYSTf$w zCDHPKMu?JMDOercBk)xB`ROML5_Aw_Kj168nL&t=agJ}nI zQ}=RiK*lsL=ui4~ebW>l911vSbLf?mD_=xEq3gX~Bnp^DNiAARwhH5Nw~QOM=#IBh z?07LchE&8+RTKL`q0k@3N5(mW0%Zpq6YbP8tx<28V#xYl?t>g?px7vF3(JffjC8&s_GvoAvAtSP=2 z@ORCh+&UuZX-az8q@KrDZVotzyt|eaoGY9KZyP_F=lBhF38YaKK&_x-!(9N+NqMMc z`-f=_-4JohCA+{CHBG=uZmy$cP4_A#zWOw_TJ)KFT);C<@f5PPF&3@a1(qvOeOzef zfwZ`CWFIcXznTf1PF&o91l~QXwm=zyV=WyNeeY50?4_I{qnqK-6ASz3cm}d~8BJA1 zsywgs0+|*^k|XtlNr+_QcR@@}e;dIKXRO~ET32etN(AMv^yjT_cYfykC=~AJeQr?z z+0bZh!JNW82bnm8v$h;jk&P>1!)H*kG-TU`+eF83*ib#vzKs5v>7uumYDmsN8H%U5 z(n`H)bUg_#q zmu+`%UwqZ5Phpx6e$=G=;zLW0K-|!j?l?`y1EH=9h9q95VPwyBYnua9tK`xoGf+~p zo-kcAUART}FM_Mm!R`%ypDE4q)b}+nqhVY9VQ4zIDNn6cv|MHyMJ6%jI{RW47E8({Sk7y$=SB2RRt-jd zj3*Vd?NP6*K~|Rizcm|yXkW(oGeB2wxjS7Pds5uJ3-fXq*iVtb2JhCzT8i&Rv z`HfV=l@s;XV1n#*!jtl_72PF4&^=O^6My&jQBW*;ONGgPLN8sRy#C)FL3rZbr~6xq zx!5IwrSs_13{<7|k0^@l;8gkwo7EOSS9O$~!FUM<`(M0YV7?sb)89uv2Wz2kP7e*V z+*hYB`OmDN{vQC2Kykk#8@MnC*cbDQ+Pz+HMNF#>zt=s=!0L7_diwP?*KKkY9>kuq z-3p{oYhHx%52$M?ZFj$}wU6Kt|4zjrc`dd1zmr?80!D3fmd_t73^$+mE#-uxUJ?J?VW3X z%D?5EY2Xyl;DUbvD-2XVN?lz?YIej(Iy~`$&RMyFkQHBmc2wb%Q8iF~wJSp2h?$4f z%<66TN}wJugnxd(5Qa=FMkJLAg=vk8R}rz<1kkSc^>? zxPi-6w?#Ar7G}UqhGgvJ4!1lcH*2YYGc+#uM^|j~=&*A`1|XV;PsO7;8oqleYd!_y z!6W$FqxlYl9e^ERyAi&Rfj@R~O>_Z<$=U_ZIwCRFn9Ql5 ztKO*QNphPcw@Gqqk4bVH9FycmCdqB={3gk5lH49paw~Oz3y_{7zcjb@>AKz#czi`l zRz?VeA{KSo-+}%E?Mk9IY%#F1UW6=4E}{2DMwN=0w0)i=<^@8P;w#a(lg?yMs za>2%$5Vr;0`u$!x7dtrI&z3leq?1TGxE+&73S$yU2gf9ml1U^TJHJUJokY@y6G@E_ zbXPkfR(c&YA+}_zNlu#Nq`~c&zH9=cdjvL%7oPIRJNY=a-lR82?i(b_U*M*Q4;z1w<(0*9zO+zS zTALwXTB)yTF->04D@IgRGAKDUFv%JK=0(}x*t*4-H^^wR29jcMqIS(2)arrBcJ~ZL zPuuZnS6UzBDO)z+_R-jrc?J%B+p!(0EnwI|ov_e*tLyXV!o*W>REC$$sknp044D4F z;+r+kSnF`Nj?|{fuu|g}a;8h3^d59b1a@Z>r4F2Tj(ApA$*K3iK)Wkn>fq0vr-MH~ z%$)pb#csadKf-i+p15Q3G2~eU=+1wcXD$ks*j9R|ufUt##=6Ulu56=!C}`Qg2@P70 zjYrO;yjsxY>N(5$5`=apehGDuXx6h|+Zm0d?yG^}Yi%V4ansY-Tr<~HYdq#0r2r9<(aJ*xFI)?BSxDIWuhyemoiw{eXAL>~ua^!Dd!}n-l}xTyb$MTe z0IY6})m%}uET~=|5;kAW$sWvfp2^i-U+IQMwwu-ZJK#T-JgX-OnC##`=jCuIE~?49 zqib3oWpZ`Y5ui_(gpy^IWwuP8s=E-_x%RwvR8PNe_8VE*5>7B%>Z;wiTxb^&+3Kwh zUCp(#E%Zt8CD)j+@6zfQETOLaXe*<$Q$k`v4Y#?Jo=0Q* zc?BE>y=yT`sOt)tWnpvRj3pY*-d0ZxfQ95~O0L1$ZsA<9fZ=#VQ?HoTx;=@s^$vr~ zu6Hs$L)|~*H7l<m;oBxpUXGT-#jJ9X^M6Gj1Ir7GPDz_ zIiUsr9_ivCx#A){Bo$A;e6Xfy(v`j+vxgVJKQ~4XtJ_yx;Mqa}9K`=T5<6`7^r#hu zs|?0<2lM&pqTBwCF>yAdMFjeV#Ab9h{e_hYff7@9m&{E(tn1S?73jRX2_ng}W*%f8 zi1`IwvJPeS73hn?FnB0-f!K=+lET+!VpdQwFuJzbuNW;^>2GuO6;(Vzi4}!3SB3q4 zOA2Df!hF|kViX_y$~|4jp?-J%@;;v7f+}?*%lK}f4cDrLV_QVVRgV!KlTd48c9@-; z3L5@8XZjE=DkqjUv9$g%v9!T&eIS-rDWjPBvGyZT9XFMU>fo5D&O~+kR`i>vI$=o+ zq{K^k#o}xVC{+?|Uf4Dz+_^PQYZ5+Te^}68e27`C8BJL^Nb1If?nK>?ZUyK{X_=F6 zE-%mAMCvG4p$W~TSfPf(sHduV+hS%7C3Er>BlL!r)^AfoH~(<{{KYW7wo5%ihar+s zu?L}MhD_NGYjg5OWE)F}xd_?L%%}~UU(CPw>-;g;ngzQVZmAJOZXdPYVL+losGDv3 zSH&uJ*s^-tx?+Rt;LFBzK5szR4QPe)al#5Sxmwv~(6UT@ZDeKbJ=IVftj`Qe?zQbt z{u00K9#ob_qWB_e- za&`G}#a`}X>%?&%kBSw=a2~H{F^c0pb8oe0?`q;!in#*|@>^TiH6(w#8plnbcWy+{!Ik_`lt|TL5 zOA;wxk_Z`-#LJoF&KfR(>Y|1Oj+5_^YWhjdr$$O1LRTes)mOjRnfT};mzEvY~0ZJfo3#2@=s zJHV3WOk0ByEea;;5;$le`|kYZeK!lr&&NmoSTcxjbxR82l)^zgA2X$p;Hy2&RxxNMJaDL zp3Uu|jwNX14Bi@c{D;#}7){Sv!o(nB@9o2GUQ_K3kTgZ5#09(qCUZ~!z>+EFyXpPA_J6UIU#$=uKE_EVL+helMw#RRbHpN+I>8NA?=uB@l7$@l2NXoOGOhmub zH=5o>p`)48Jw;e+T!z&_w&~IvzfWT4KMApcF$u8~muQVih@FJkoiPcq-lb0DX?sjU zYz5uh?c}tjaDY>|5VQwu4XcET1k%oZ?x2U-2x8 zmDjBFRgNUv`~97*9(xURAGUifm98^PACBS$S6)Si6}Wf}d(oUvwDFn(3GvS2j@2cr z-oS}FmW@O=>JKH?p*m`wM=$UtLxLOsWiNp zsi;!yS|-*$vG##IoLD=Isi@LaRB3xmMU|$aO8sMEX%kE9ThYG(OPh)+`4&1+or&rU z?BPUpV0_X=mB5zp=~Y#_-?aXX$||A7?6*``X+(h^4}RxN)0ApDA`9eI=ar`OlCsQ6 zq_)+S8fxFr{q$C1l1-+%kN#rj5G_Tx8&c*uzEm?7$E zmT@XHE3f?wp|hNCaC+|x z6YFU~6RzX4ke=BW+%k4sh-+yl=Uq&}N}|@ZOkD%-8-`u@Nm|OHuwvX`Y#9#@vaZ55t zJL<^rjo}l6YoPr4GM}?Kq1b_3E{(F2RAtFT5`l-oMR76uhSaaPWpTkZr&;HQkgr+5 zoR`XKky)99U9)n7@8AD!aWVFs+#B^daq)?r6Ku^JaZWXR$#WjBC7;8YR0ZC1#;*-^ zrR2|F{N0dT4aHqj77~|8i}lWs^*aA$-I<-;7Odns6PjjZ$*8VMKTo3~B{^TMHA&V~ ztm>4}Wo2Hyub($45PgvHpSU-_<@%q7Sf1RnyvXQq2uf#LPK03{svV-h8~_N+h>D6K zjm24QN`n{^2w;ma+TVJQ8j{M)vneKTwNu}WK*mLo zEXJ)+X6(2eu|$0SBcB73%rSiG!o9K0rw~-@M1pelFsr&ad3Jts+2U9eyPDY5#ICki z^qXf_AG4u=R|}gu3Jk1d3eKFOpMGc2Pg8JadrZNZLzA6?Gbi#i1!uzejnSs5qd<5) zQ%8Zm)qGlCLy*xCLwlcvXc-yk*7(Bh4CAsO;bmKsiQ#8m^unf-oMzT zeR3Re^8Wpvynj@Z!7W0f@V z^**RKj8>U`N=gR7!GVW6@vho7c_>)9lx0p9)Ld-{*k7?*l2e_mxmbnRqbb|0Z9||MXY?Y?!EW|Ax(?c4|h>qn^5A ztOz{vZTOM2l%YqShaXu}wa%rVb)qo ztp|7!`(I;c2wvoKHSPzV`!4FaNbxC`X_Zw0*}I#5gl~Y_L$7K%)ZkGqY|E+)Tmr`j zu4z`WfH}>O8LA=c=&wW3{q04EvXcmssu62GXR9isC1JNk$&`^~30+YxlqNT8ZZo;r ztuASzgJK{o+0Z>8l`NfiXyN|+vje|q9C@;qVC75%?4VmKt>3Z}?JksBG9N<@iCK_k z6D%d%DJ(mCi5sEjW}n$@0A@l9NCsJzvMLl5Vx`xVXLOOV?pFTs$%Aa9Vg)U!mgR_T zym0Se@=Zz>Oy4jjAkB_yCa7lV9qvgT{%qFw|LpniP3-z$7C`mJ(1FYPqT>{m)2wl- z7_rT6xzb@XB?5rbmmKOMYA9}~y4zLfPc=D0p}SU}9j5Q$g{x7z3(8?;0Aveqo#J9I z!yz#CL->^8TTW{6pHf88imjc8qk3&CggV1SOXfk0F{)Kicg0Jc(+!a#+c=XCIlg#y zdP*{;H7nJjO)OQ1gobSaH!RBtRiqLrD^()P5Or_o7yd|RzZ}p0-?u;8A0E&C{`Tk3 zd{tk`7)wD+RS##hf=wQA9pHIrepWMtlBp^KrKl$Pw&-(+Yzm;WfK(QVj0S`}L!<_u zH#SDL{Y-G9fxP+&==}*wgyq414j;AKUXk&Hp4=Mzqgj2ic0b)YYiECJ-8p7K*vsd# zK2j#6X6`#Z6F0ZC+mXm%4&sYkl6Q?t`PKq?Hzy~ZkM%%>`WtVB)NjFdnWiiyH)|#u zL{w-DWqF}D_C(44yGM`S9qgij=h*F>dgxbo!|Kd*_X4Ed(qwHbYrs|&nMAavejOd3 z%)#i{135RB4owP+jZKAjSua{@+P7g%>in5Z6-i2FIL_`?v_#QA?H1_uiJv7yW*OS5 z=U~fd9&b{&K_lil*9tbtFvE<&e0S&RHx(e1e&RN!02LLRH-nWJW>M1kEQYAG63~D8 z{Qay+9^e0I4nKT$@a4gK_m9sH4)*td{Pxw?muDwG^Mm(4iYmXtf4$%Tk)8Y;3FP45 z%OTFwF`IIsXR@5(3Z4>O4%GPRn$uq5N#ZXTtXpxTU6vs)D&4||p`*x*I+~950(4DM zITdupX7$HrwOdLZ^~=%$YVVMD!nI!$*UlXY*}^tkH)IFf24>QwNrpv3+F%WxD{_N?eXo`F4d1zHm>E0i1>x(m07VvXdun|&$O1pFa}xQnVn!9}H0bUW`yjdN!5auB zYDEjEbz|guh-g~K%1ay_V>O!Pm%m#bn{Du4zv91u*dU}v(>XAfRtFiqG8f2Vj)_FQ{ zVI9*n9I01RzqYNq-E*LwpS^wAmq8yj9TL5{V#_3_Nw>3Ln4~_CWMk3w!z~< z1u+Xu9T0FwYb$NU;uXxxaZf;s?{gqTsI?q7O(g_e7vE(a&#hew^i9XDUj6=ml4>(? ztCl=`zswag$cLz6=5VZV(G51PcsW#Ek%A#WF1&sDdoU(C@Q8osVnW_ZZT|P_Jy!!! z>1HoqN~YdQ4No7uI5@QArLpE`gVF+xK2Q-4O&o5T=R;8n&7K%Y=Yq(}t${btC=1@j zw+bnyU{Hx$(eT?X__(a*8{x`@pNcPeYZy2LM31O&6Wd!^lMfv znGq$jRcuxtv-0wA3FCfJGxf_JHBH7!8$`zk3?b%98Kd$JuGlUw!cQYTU(?cH{{J!L z8Ib7zl9KXD&B&QG9`$@Te8WbuhQJM6qniC(^IKZjHgZ%DhT2vSP`TLQmJ@Qfl8V`o zQdU6i)T0}=IhwRiNT2ek$kDktY9zJRq4azT#EK*M?P#{cVEe2Makjg!Ha9Nd&d6tp zV%b^1y?ux)dLp)EGA#*e6Wuw0g``jxkroCy#qvqTGFic`3f{-S+3$qr1vz2wv!XUX zp2khK-#TJ<`*il@?`BU!Y0m!!R_iri8oETN*pPxPX|{pycEI$xQZT3XG)abiK^v2W zms95(@-A4|t_{ARH;kydVql<<4E0{rbj`TX7+?;9?FPD$HOVDBbw@jo01C@?a*v|v zqiEW{9FL+YT#uq@@AW8}l1I^W?D!r<(?`+tUPaR(Pxh1P6ds#>@q}b@y{00cLR%49 zE$bdpBhE`fCQXoi{FmW{{_l4}8fNDUS$$p6j9s#d%Se{acGZ3`<$E8EiwJ}<0-{t* zX<4zH94}-Q3td#2>Y$YUnsW3$<%m(>d^1oL+K7{)<=~sx;$;cZ4`;WTx@h*?j#F|XdXKwU$ zb3UQXf(^`z8aW^o3)r|Dkm1!W7*Cka!Za~E=_h~0Kc;5Xf7;!vUKuuen)~HJ$iwG) z^Fj*E-VY(;Rr_hOl0~jaDf4N;Zb5Iy-wD;a;&bq&Qqje13>%grNOo6DYc7_`GP8Uu z4$?)tU&?%I5MxD$h5EXZbx-}N)@+Z?Fw}$=MAfAWAt^9i7WHtedV_FT^tHHaLP9W} zx0LyeT*);fx3Z`Wx!;s#p^8G@!DPRw=d6HZ-3Jl-&Vc-RX8;>5TH0V42Ndi~=4Y^U znP8RbVoo*~A{J4p2Z`uruD;pX_A*H^G?eFhQPoM0%#);fzo{rk|mE=yTyw^T0e)eZe* z3rkKRU>`oXqPKlpD3YT3supL-6yH!5L->q5RSTQ!ud5BF`&%Szcve}-jHzC8@OO$S0|!KUpRQ%8(TrTs z4WWgSZth%c^VgYK?Fez@g3P5}?XYFDjk}C8u+;UzbbHz{7D(|_q^wxRawalYtbyug zLHZVA9}*aD`~%TCK#jK&&OickoQoXl45`Kh-=G56H4V;CBaR9wrj_hP_FAzD&ASja zw!?sIAtPkGd~I}EsvRcvaWP4l08bT0%m(f%JFM@{Uy`Tf#Ek0{P-Hg<=y!#DGRMg_wGvI;QVaLX%H}`K_1K?|B*;yzhrRbQ)qNxc)jX zfPvpB*5gyzEkbf{v?;!uD zK#c5x=$T?~xTxP_59}1rFes?6;3Z78eXp(B5L99@3UH&rM5KaN2xl7^9TS9eZ}c3f z*G5T2Mc&)Lb9-da{g7s>aDN_|=wVEB*u)KreWEq<4HyBq@>_>i0rXvO>B}+iT8KM` z8DwRwgx^m4F0>`N!JC&q;n?Akw6R}r%B``Rmp|dy>t2?&Gj_xM+G7WH^r^-^;96BI z%jCMejP>Z-t&M;$+46Lb$LYk{gL7>g`S8+;IfX()431|o#3CApK;%?#>9 zVjV~`sX@O?N%d-C!(DEfGiTnQxPPp|suK=R>~JGMfq`NuRMZ%TB8-}d{7R`eS#a_hlJoS;(>tQ^vW{NM?q6?;y86vR)EOdHgRfOVOFX{hrK zrI!8#Id@Ha#2u&eJN(=FKm_7<0`S6&b!ybAT?lSvx($IOWoP3tDwrd?7m*B%|Bqq8Z%JW_GNlH@8!A z!z#fFQqqd98H8^*^{7v>NQhe8|_7;i{?(`+@4$Rb~k+mQMc_Xq(Qwju4> z`|g>r7e@8jR}NpU6@B5mU|*FiOBC0>zPLJTzX)_`#VWKto8Jk|tt%@gud(?`${TXx zDkv*l(NhjWiiaeGxz+dODuR#!8USK@5`+Ppz>!-Lka zm3$E%e=E&K<}zoEPvzLaBo^p#C}IVEU9Jc%$`!?Uln7))BpN61oRO6j5OF!*Kus=Z#LojP=FA5q*Y6Zalb0I4?84G=|@`&JttvHgcSaw6?ooMvX$Mnp{^vvGncudcP z>oGmE_j*jvB#-HtW5@THp81%bd2i{NZBZ*x!fH!Uo%P*p;!#w%`r@TwEE&}-fBuD0 z0|eu?#TqDs6)9!DH8NPW#(>xXwN-x;LXHKdx;@s?cof$9m*Y`bgX>XP>%AU@HS#E| zjUC^ku=Xgd-Jh_wq63oI@zr0?w)%|(*%@GWMPUs)V+1Y5I#ZvwdyL#-bKak31y-+Zb(ye?1F|v3EelPb)0EzOFSmPqs!mBeoAP?Bz9G!Upgq_+UC<@&$Ys z+m)C(dBGCwlE=pY_4t@@p){lU3?(aEk(@7CXZPNW3J`3)(U&Sx(#mI3;OpPLd()A6 z7Bg=`qmiMVeD(iczIu0l@#^jM*_&3HKej2+(g@c!i`q5B?U;iUTJG2U#I`!=g1OcF zB#3)a87&Mu$>p7JKz-6|#o$gCPEbu@sH9?R+e+MQ6&0BgDY^E{Y)wT?i(*68@)n%^ z!i++7Q^cMD;ODHeVeppQ#43Y@T-5H6_-02PprL7*cYFD<&d9YSKi90UdA&eZ?RBjd*EES3}UD~5l^A*cT<9Z`I4t(sU5$v z&wdZzy~4JDc-&l@g2D0`=DN~UvqfE8F^%GT5LE<#xWJBz;MMl|_Ad&`*Vk-a8a=c( zQ*B(w4~vvf@C)k6Z(K@s_i7Gnu4|U_SfAncisI9u;&%sX>!xI==-`pT$0CBZiyfxf z4WeSsD^PkjC?y3AC6A6=KR`yKy_TI{(1uVSAN$H3c0576ylC|pivT^bAWjX1ykKh> z;bD6=uw^7oT~YiO2Ayg8w=UY8D78INVuPY>MDfFGv-zgB6eM0ago$i`Q@Ct)n$cJ7 zsrFSZAg_i0!mxuF#@(ox=>ZHY8OVYL>KeAyBgu8p@(M|p6!w`2OnWl?;O<;-eAv?n zrCV1&3M5#!PO-z{WkLvsa~g-L*Pz(o=xq-+s2x_M1+6p%V788&;6-@m8q-uUFj9_P zL?BsU=a_V#l3KB9x~zFV=6rm1?XdQ2h&7<_``Q~x%+5W0zueB1^P>iBsZ*5&x%q=4 z&t_jfo4^r*uH(e2Y^pE56_WdnxDB!K2;td(fXHo+xxxT97cKTR6RhId*3^2SyI8d) zlp)-cM*Ah7dJen9K}>gk+Lz^aNC&AYO+N&|$7XL)D*}TW75S7FM)yvFv^rOdBN zmd$Ya)KAtz1R-&dDKcE4f;179hETXUyVm8cZqzP@+i0KwI0Dl)HU0htIaxHlohC#} zNeU^Jr!(@SAZNn&%eHIUs|64jZw{ky-SW&(wo8BEe7+D;f8o%1V4&_rqm$lB?)q+1 zL27}Xp4^vY+K@F@n%+bk2VXg2zjznE(VF`Y&95|00h2sQ%c*0H~2f z%8E^0y)m*N9z^E#N9Fnpl}eVut%(7(7h{qNUgEW%-Rq@{}0$` zH6w40+Fd~7E3TZd-d|^e;92sT%KEBimc3 zeiYDs(nzfnv?bY$3&D#>j%13Lk@!&75lz4^8jG|GQbdQTkl2ONN_Yd*lHe|9vgEDe zlZGakorU>>F-;o`!hV>zM?($Tq1(3&#`e+hb7l!%<2Y(aWlBr%Xt!@-1?v&(2zrLi zOG1yoBTSb@@KKrx{e8pRhx(phV`Mb18ao+RZh32gDBND?0f?}ROQxIphL#T*(&p?C zI1Qg=GPpIiP-N#6?Zzo-+x;7k8J~6*@nxMHw!rvhCP9~gv6 z$Qob-<^C)!T+!S8Wz5nqrIRFog9Dnv=cdnMRkra;aCzdD==cmDj=4YGWhfAef@All zd9z7K3nf}>flZhi+SH$uFIpms6;lN*)yfDSKm*8$vNvXS6J|GmfE{W-9!mjp*B3hy z(MjjoI8E{M>)GWwj(&)jy6ZI@pN*A0^om#cw4{)0y(yWRgdAQ+2 zJwJ-x5~xP9#}31SIm4>8y}NzI9ZQ~^+t-Jf-B6x>bM0pAZ)FC3cC#A_1Zu7+>~g|m zkwpFNK@N(xc#R48*uL_R$nDSN4TW?k9kPETNm9zPE?gl}U$9%RV4xFB z=k-Bw>Ff;ZyMt6{ZJkzUg-(Cqqext9>!uF|(U324BA~#09rE$l=dFfa>hssew60)kZj2Y#luL4bYg;c0b*zHzB>=jc)Nf*9Q6iH{&j?Zqr z1nDT>2)gE3C^Us>*C{B4wlwtELY8waGK`DkdgEr%F!_|N2N?sjYFitdFEAq%l`K${ zJYuA9Nxfp5uB2yhAz?H2In8eFAm}8M>ym0d=Vk&a(8Hmf1hN*y&xU9BG{q`Fnp&e) zS8aw$XszX%YMz1Bhc5`kha}?F8tuvEFgdpuvUU(WT*RwUX*>IjY+q&<^qnMpZKP-_ zv~PBwL2-EW%Z=LkofR_*$qy@DgcxMYyF-vf%Ea$vQve0)t;>ZyOlqK^1# z_h?^_x(E67>K?v9_E^26>8u@usD%+Y{((ffw#)2$z98=yq#7cTe)Ybj!bqa8DKF|I zLBtXB##jYaO;pPhGRhk2k+hY;heeAkW&<>!cVbzwTP}OUKf=q#E7dk6T50Q-S@Gq{ zfy-z~Gb1-+OZFsucP-I0)wN@huKo+SPVUjP?sxpYxg4p9S87mS z_gl%-m3*E-SnX@CkgV;iO<&$%lpWrjTZ4uR7R2Dj#KFl0Xl=D_q$1@`{%G`Xw@*W>jqD+(!Arl$aqZWVg=e|u+LZ{P zts2a_)zT5qbRtl`@NrE|;#p`?1L^;cLxHMfq$Ygrs#KfBox0lK*4WcfgG7j*J z_|siBqO(9mBE(2O>-Lo#+dxK^nMedlw#!k(%1)%i94JqbP=;dr8i!zb4V06t@I>mdzvy_C05wlJ@ z?Zb?F3cJz!b7DQN-UpHsJ-4*j%kN+KiX^?+yX`#PPO!rYA}zTsew11P;_UL=l}mD` z54Z;2V>waTimfS9vgn!dY82+uK=S$1JXI(GAV9ZrNPIv6#SQ{mkCGU^SD6G z)?fvOka}QZSh#CVH!i8N7D=$3LeW86mt4E7f=sU0wcvX51mgGkyf(_&lbqeM;)&wR zDXp><*DTX@#h%cTPho-r)WX?1PjOl$oRzqTzrC-{txsFk3Y$%GnUe2bU0p{v)w&_E zzzoL!H41aHv1aMHSl9}z5r{sOmWdquizA`tMv7AP92_#v13|Q$kr%O^*aTzMZ0_XD zsDUr&niVgo>dxr7&uxW3kg3_0!!~v}39z-fuWUm0g=>T1#X8>@qhbZS#iZ28!cpO2 zy|WwLPt?=QQ$ZGDoPF)1X(aQLEgBbOMxqU(wwbEX8`VhOvA${UGAe)(s(gxWcsi)h zj#W$>>h4m0hNdJXHkkm(K*cJiE6#2iq1Bx0ib5?D+*h#K4k>;Zo2t<~4pmVM)TX_s zi?p+1&06Msu^E<2d|cqzIv@c`#3jx|nZHQTXFno#U7((7$?Of=sKEAX4bG4gH}H9v zF(<1fHEau@WK^@Itjs@8R3=Nimtb!_^;Imup^hIy0XATT7=4Lxu8nv_OGA|_KAcJi>S!x=d**}VuuCm3HXu}q+&WotX3;Vh z=HfY9mr{dz0Vvxlq7FhO$3lZIT(82Rli-;7SnT4^oo>NdrG5iDnu2Hp2R;EC<9X*> zMOKX-5K7uEhdEo(ie{QsTxp)Ect|~Ufov8#P@#b8PiB)>vbs-mL30c5fc6(z+eAqr zH6i>VlEFX6MDQe|`AC&|$WWY``27$`-=84idu+0In3Xgn9;GCDw}aIOAPy#ZccpL- zz>Ft&C!}u=z;9(WB&Kc;VZ6&fIy`NAM74_Kl!=E)m$?5_iHAs&7?UC~BKcv2u{15| zVPwL?m|*`AV@*q87@5BCAY)B)ktQS~BqSmvJ9_qY@{CA)eu$*!2T6Dyne3c0&h$j* zzBJ|}`@v3|+a?KxpEwmaou8zp;O-lxnw)MpKGkq^n&D@iVt8WPra9_gv0bDGAcrS8>8QIYrpYkxClz({!j4Np9o@w} zdHMsXr$NqDI$HLpoo;_C8Rl>o_~5h2P?I~tCqq0a)pTT<=^iPj=Wp1O=NcUA-M(xg?-RcvDk(f5Rn-H2Z#le)(?PSmdpabcmor$0cU=w@Zj}fQt zTbkbA+Bq)ib9BOIE1fgVnD~j4JrfUKYLe$337*5uZB%lnOWhod&Nv{kbJU`ZM6>A2fu^1)-!l7!Kbo`n+?!=Kl2} zrfH^YT3E0c5}od+Y0$_wG5l8woTfz~p`rh~G{@OKQc;pAcbpk_lnM8QC11{&=RZ4yv=TeI&d9mXFDz)abl$<>6g$`{49};J zFy~LTa-;c+t0t=iBE1G6x`q0=mYPnitpFOxDVY`XTXK4aC2&9?ck$f zx_}U-Cta|y+p_%vm`PzuLv~eKWHp**a8$c%8~AWxU5AGWzGRk9dhioOa{G)8c^veG zWB6a!jzdStvXV=qg}}nwC`!bv0l1Evx;dgwL<{2Ehs zcRGv@0FA~GxJZF}X+!FU|Hu%2g$yA$t1ZdXQrXmi7OMx=B4<^x0XrkMWxp2$lLB#P zj2PIwp<;^v!oXgEB%>Ky5DaYC;_*9p z6uN%1{5Za@NdjVPwK^kJo57$Molrb1XYd@Mb6IOpfk0I)Z->fKjq$J)RO!n~&e=6j zYm(8LYU^zu@@M|OgZ#I7YRe1kABroKVrfGa?m5Y+W>fRQ`!*7qnO#?g(3>Icy*OTNP`xbYv*i)4%;~a?1SJw|18#T&6J|c@r!2Pb1Su~*Hla^Msp}eVSi0p zH!gChP#~6DC&}zfu$C#yHL@iORk0#UJu}DG&?^`}(pV|{FJUkYU;_T;oQzjlCgNNA4{nlAY?v1NjWOQB^L@5U{IPE*?#1FXey4*#jwxRi&E&;Ebnub2uj_|dS2rvyPXJsKbET55= zHUP0>q&B6`)2*LuNy49I#1$+wn*;e_V6iY?M>H^S%Zh>BqznCP#mMn7jOh3nay%K$ zXIS9Yj*~83O^YTQh$>RjN_SUg(f-}<-o5Eau!?M+UQr=C&S`6LkFMdVt?D{9E6?4; zc$NxM52zV<=gNXVB-kX#gsTYDL9$`*o@sKzX3JUlf*rJ>=7rHkHjWvEQ4>8-xmK)LK<+!R zD@gbTXDk(Nrp<>?&d#LBc)?>8w7{gXpuBKloS0GSEFFbhF8k!%s70fVTW(kl`ND+K zHqI51*u4~6#8BGaZo-}&hg-G@fgL+iSw%}W%eexExtUh1kTjn{z1Le>M9Y=0xp>3G zQm>wqXOYK}I4N+RJGHQOVnv}t+E*Qc3p>L&T&&dnaMLKz-xc~#f{(APB7i>aauu9`!hE#pSHs&8; zWeHp0s$Va^prtS2e0)5enuVL<^rpH{(=U#X$%{AV2Jcl*g8DEW%2G?{UP1X+$I@IJ z!;3SLu?o}(gCUli#*P=Qmc8Y=(}qke*W$|yI*>@;n!yYAPB;lD;*~u5q#2~of}{re zx8`we6A-kt@%;Ci`MyUs`EkgdY_OFc9ok}e*DYsvPwr%O!^LuHM9!%#b@2p>Q$E2O zl?m3C>u+Cv3sRybceP@Ry0FJ@7F2zoK(cvCEPu`OKi_Nr2U(&tF>A)gPCa3!5^ z)ShGjf1D9=k(SFaXWlGoA>FXeBiY%x{>8|SJqt*4603B1F;OtPq$$vM#av6f?O{EUPPab;HZ=S;ZF-TGpD^ zORgY3322H>FFEpk$e-`U}!!}m~+-!i8b9gTyV|iHur%GJ6G!8YMJEar{N#smvTq&i+Ym6HuaiV>=5IU;jIoz%F z{m!<+xdr--anMp}C=zggG;CnLJkhm!?!2i@u+xevQdTTuIi#9HtugPd?GwW^UD{rQ zlw@)}=K>SVW)9sP&Ae?QxUlM}U>Q@2R()HmC1FC<72CE1ee3F7+z5GR3kx{ta$)_V zeSFr16noEWkN0w59}0-#d?T~I6WChpH_g*zO>eBX*E(<&&sjz*3zA4d3MtDu&2IYE zTuEE7Pm^3S46~ebkrTS4T-e5Omog{km$Aw9pUgroUt^|0!ra`mfa6+a7@$MaZ`$CS_ru4HUUl8k*lizK$U8UcdP2tFQjU zX3Q7{2fyStjIswTR$RM{D;5PnlttLcG z$a+Fn6OvEJ-GuZ=Q3DH(w6tamn4hZ$7whGjdiy<1@Ej9nhQb z5ljn~Ro-jAFDou9u6x#FSUir3#`}uJ@$8*3B5{IP7Z-&;S26gWMc{;W8Ww%`6no>l zJtXoDi@R}Aw_nU16miFjw;kl8U&9|FArKbUZk1J1y|UQZw})+7(++VrI<;*Y1$7x{ zA;r?L31A6*9AS54liuNK5W*T^cP}CgA;pH9$vQ<)dKWzI@+4hBbl^=}{P2z!eT5kA zC`WNCkcy$7IIZ-IyjqfHPrG9WgKYk%A6ydt>844@KfMe7&V#>K!QboPZx!BGr++@} z&f6bc%4kKMt#x!|F~Xj@zWGP1QeE6I(_V1MXg8uXhI+GnLWJ+dte#+_0vD;>t21^V z&|&s9)Bu@4X1@nvZ0(4_x)JF$Ba-Vye3rE$_O25#x}QN%$2~Hh&!dk)g4JZWZ^0u@ zep;MN>OU|RD~I|H>~s}~KppBgFw{*TW>Amx8Tf=<1P;r^Bt)%x!r8Yzkj%9!j=6jN1Ij1wDc6WPiR_&HRIIzt<6MGpPz|q;Y{Z`cR z3-2;)dtg4~s#i4QVwr3wxelLi<>=rKgbQy3xnA`avBfO%R?ezT`rgi#waKS-naMTQ zyMQdzKDdHVGOrkgVw`vM(PWY`g-4Ugqsb(=#+po8G$hej(xxN5CK69pS^`MVqQz|^ z?N3hH|1_iS_2S)wJ;d^pgoSbZrY*Pb`0#AN;C_R#707GRr@^#G6g<9NVu89sc|lfw z_$2A5N>go4`4!W6ENs=K)`vXP@6c0&z{@0%w0z5WFfsuzbOsKci(sa_?gbVKlxt?O%x z+gXu|x@rPVJ?scj(p3M#Q5Z2C*`ts?oQNDhk5(~EQgLyTl`QEJ5H)$O+b@!wX;!Vl zqPCKE_&fNmq7yAG1XtV6g*|`^ty#WSU-i3zb`-Rdwa6`*^Ywc#wg&t*sNEYC#PGsG zJ`ez)sW~IhX3w4;y!@kzAV3`xV4Khd1Xjd}CGKn-Jfe!x{992(1I5}tSwv8H5epF* z4ZW(_1aI9{T(cyloRE_REfh0SLM=QsJDs((5O`#GnnQs@s-7Qowr!C)K-%VLdz3wn zV^h0zad^zbsU97X6MXVmkxEK^dhqtzX}@}3MjMW`rVhoY48JIc)*w3Pt@Vkl!C%;l z-ZG=Qg8iV^Kw#lYa)q-3^FmOG2ud4))xfvFP~?A{e1G=tA8*etUd75_VEJq2W?m~F zycx`(RrF}UOybAdv8itSW>m}sNx^7 zQe(1krO5}XVx2ESQz3}bGCx)!gxw|V6=Qa^+*0OPzVq6{14UkInypacvR^XNjp)%n zD2Jjz)Dn6t;OdH(1tWj-U8W{%v0$10Q-GcDeBbu^PEUVyfBmToU4~>M{A!X^_X-|$ zMOocEV0s9}EC`HwMqaJUNG}4c3yOS?7DWq)vaCMXSl6n3ps$`eUvK4=*_ygw6LJZ3 zqPYWy@vVIIo@KS}JzIn4Dq<;VE!Q_q^Gcjwt9fOEqOL7vzh*4$diJc_4cjQ+EYP;h zgmOdu=6H`hF1rN#1^!Rl6{v~bbR}G6^VMFUuEUtfVQ?_4eHVGCk7hd%A+c}fP{$a;&HfzUQy>RWzt#)zdwL4zq8LiNbVhcyPcus1lPcOub3Qu|(Oi;Gh z5v?@O>Vj6poYSSOdJSvC5yl=(uUIBUo~Hj@Z@;x&47FqxmpRxxVuh=>4$6rwJZEuM zE*#zQhdRYy1Sswm0Bhqu4Jf%ku`}`)U#xUOcz;NF*jdDm(HF7&_uE{a4kKW(&@THA5}9*Gv&L?^`-Dp*z;rI{VATJanfKWv&QaODh>+sYo~KedHeKCw(>0kG&n=9lqiF1Q{Kci)QpN(iypw zc?3DVFo?uxt6#X`H<{JLwQNi=QQjr$^_*4ET*53B>U6v!nXD?NN{P(_{XXY%O?huF zV3I*?MAP?&Y4?5~jouSlgFH-g_t9%)u}OMjp@NjrC$>Z%-{mcFogbnB`opw7AK&zRs1@|+cITeYM#0`k zqw@p|OL$~_hH|q#->^-Zj(*X4#mO$(N{|i1R!wM6R8I%76Y_*n14gMHqLQqp>RM%@ zhYmJ8Zp?E6b|K|WUAisxo<%Nfh?E~MXm&E9``@t56!kuQzS&g_nP_PRK}=Ij@?KW5 zhU3Nzt#>Lw>QeNAby?!!EN2Av3Ph+}ieru6bcfVE+6%K;C_fTw|Co_8BI*L#c4Db= z5XjMtVNOPz68dc8)&lmOlmVubKmvzJbn6V6xuhJT_g6RUF6s0fU*N-AVJkUXRt%ys zapwUO))783XQUSFmKoZUb2I084F?2l;6L51nD|a8s<~Q#@9Y10Dc?%%EZRTyi-UY8 zuc$OyBt&iJeBpY?&zZhsw%H=8ko^XbMO(@Bp(R*t#;cZ&0xt_Wr-h3^@28sAJJ0%& z+kZB?ZEFDia~H^R_>#{HW_4YI*}t$#y6(0O(5(j}a1|+-Qdn;Of{T~^)tV z1)E?<=2z2Sq+lc~YNcaw@eYle21d`2hrtR1O#)qsu37tR^+L4P6spLVPriKe?75MR z(W*}sn{@1q`S!CJ`Opi|%;XvQ0RR8e{{JiTfqWq3l3bGT|K@+>1DTpdwo{9(=Cetg zh-XIWHAi92h_W~i8~{HFLC%?s7>v!7zxT?o(BL<$Sx*$GVHl-z;MZ%uE|u6PE` zUCm%mzM4MkcO*;E26M}$q2X?(C(ll2t*M(SemO}a%P!d5&l*Ne#RfjvPY)a^I1`&Y zx)~%0j-@X7Eh8(wT(K&w^8p@BIL8>9NHxKa^;0meXXIQTA1g44qCAgsocEDfQ>)o< z!0o5jVBXLCZ8lV2oNHi}{_4xUt`9ed>$9~E(2Cq>>|_aUl4@r~;%*{D+!a;OI~|=(QUvs6M$QoN z75IqEJfc!^WS@KY%+Z8?ntunfpnHz2WyLHQONcHFHv#Xb(X>4JMcRD&BP)0@aj`Pa zgg>_-JrokS!gFDlJc6c%*L>gfq*S(D6dY64-lnJafWU)8&Nmq658L$78(La&6EqTg z9buwTX5PWHznlcC5F4p|{@;>#e8bD*4DY#R%85)G{~wjd8Z_<1&VM^;W_t=gYeyvd z)dWvGjBpqDEN5OoQC!zRKqf@6JK&0u9DmZcXc-ky1Sw#q6C*c3t1_GY*iZD7*;uC> zeRc8`nUbfc2FnE=BiO`HG%+kXTHe46)F=gJM-*iH0QfE8q1D@tVq7Y4(S9Zww4sEx zX!hh>T*|!dfOF|aWJx2*pO{7lT`O5mPTU_&&(711Zg_az4^I{{XJ-otiQinxf@f*5 zHb-xF-V1p2O5Tw>hRY79*E|19@P>wicw+wHaSQmAXLVL;bu4E{Bv-J_Sgi=i-g7b~ zf0aU3&r#|%pHdAg>m5eaXu?HS)H%zYdKNHdWyI5SQ23wk3*DOwio%X}_@h7e^F*++ zdqx|w8{jbmh>9@?qjh(d@F|g@QHo`fN?}oJnzKk7}f-Mjws{$Z{pV`n7Cv) zp}EC!d@yPpYvQz9I;0WSXb>tgX(4F88KQkM+(IPad_Tw^#lHC6e_miLdm(r)%rwNm zCj@s$^i5(E<#qGCGYi?|Q`jc7X3k&^sIc}<(}=8uTEn)Ly14Sra-K7xc}9y+E;TYQ z&sl%=xnr4@?twVs!&1l(4Ftkyw!&~<{J@|3PmI~mqj{a(Fzq}V6|}D?p=WJDMP^RO zcEm?-6IX&CM~E;gOvWVC*~Z|LLEIpR3VIofsd6-!=vkrU#LyWBc-Q;}pTNKS^%zi3 zEE{CZ^->MIFm~U!SUnZ}`?F0^XV{zbe6fvMN9|l+SFZJ_UqysfHJFD-(AgPnNsC~0 zL_UEe`sRZ%9pT8o*&dM*Ade>a)n9DBQ}ES@_}LoGx$d&n?0cSfEK+Ia^Ib))&9Wfw ziQ02Yqn0ziaK%r>@v*m8;c?uKW-|g%OB3vceBuONqFGTO;qi(%D4be08aD^XZChOb zesll~fmv^ws{~NU%rdfh-A~eIjNP(oLrSR>kPaa6R4Kj`k*3eiS4K7Jj4(T=UL9He>Im#M zvPGWn(*h8f?ceu*H8leRfKEli#->+LnYad=|NZI# z3<_nbbpZCP0TAT*_p{^?`TwmP|A-);ocrI-?~la^#Pa@!jWv;c|9i{z9~{L{j{k$? z_aB_wKb+S;kkkJ_`TX~t%Rjh%2|4_G=I=+1K6~cw$0WQx^Y)`vMfSXWL%zf?-B3oM2ZKE}_U2tn^0K zS#vJ*S`KQ7YkNZh%IwwUx>owi%<}G5TuBHAImrzF*%py&Pd4Y z*6==PqvI8^V0R6hjzY}j+URXOs0IRuB-Sx5DR~6U{G+Oa9s5@tng%yo!bj8rW3CPa zJzuzT(i0m)jX{H4`G=w)=NtERLxBS%gs@!u)03RtK2bS+Hi01^rVLafYR!`>a`f!T zS3xc|lW^YtxhTYito%U0(Es#^oX84Blu#<`RgBS?t+6n9eI%VXRx)N-hkpq$sLp(u)r0}oNTgnSM z=LOdrBOjd3T0KM*EW^RnXtc=VC7vB?Cpj^{Jpn7~=?rM*pxm=!)sDdAUhEZ$4wPSp zd9kCo(FeYJcYYABj1F z^NQn#puRyt5Zh5AtdJ9(G-81@?~yZ0ivYIS<5;*=YN1_2X~xaZz2T;Z1QBkN$iYe+ z((nOp7P$WrH^zMramLEkA}xE)Jo@^oZ83ep$`yHi7hKCKy@u7Xd&BJRJGU3)BDhOQCX0e)gH{2#z=iN2J5)CtjihEbe*opps%cMf z1D?Elb@uY&)oh*rW+m^YT25=lrd-E3V?yDV#8M;91T2#BdA#Ee7TrG=~vZdPqvVmgCGcv34?&G6Ho%i<0(IvDIb z7H!SgwY2ZpqHAUCZSVPw6C9MlXsuTmiw~ApD-LunHI~o%^lO`1MQ^A>#O~^TKAo!Y zy61jXc#YyTgY~<{mrpX6IjiYNZ=Qxmm?4X*oUB*TTtYlj1v?(+QUN;Ce9gsYLk&{e z2cXEZxBSRO-oNTwCQW#@EkN3P)(PW%%&~M3Cp}2vywGhQbm!UHUjFq){aj4(>(erC z*8lTy9Jh^faRU*1Fr%FF#Emk|4Gk87rMP?#-Z%0Qq`%Kn?@!{J2P|irjJLlW($M8n zLL#TFv^pq?A+>UL2gr7y69Sg5YIlGpI zk8{!nFSSY5+T59Q?FmlY9mq7+_Os>Gr68HDuL4XxsNUf{bg}$AbIDA*L}*_k@afa1 zPqBR0?|=V$0*&8umhtuWgHgZ;;oqJ;o00#0cJT(nA4{#sTtCp z+qX?2l=hpTHTvajg$T!o`R~8`-Hg2C6(r|zTe84KWV+yBAvK{;6bi(oxNf_`TtN** zqc9)^iSo6?u*m-{dY9I86XtCVj`espJGLF4E21>5?3p+B@3T|I1uYfU zg*QShbY759fGu|n?09Xt^0#|(WxP6gvu!_Bu#g#t z3^xJny|QGy9)L09HHJN(FQR2ZgLQ2aIeXwkfy&W3ND54VIEEt+hdJYQ&wLrLJ-IVp zsXQ95G)|4zkcSVi)){kdKAgTKsbMetMSqj})-WZsBxs1pj9gfvjE9TPl`LwFz*CUn zqD|8qsn4}B0AlSQ11g>XwyFPsM0tqh+v%key8G*MLe(^wZGTeg?^g9Mo&j~?T=_w{crdawOq#H)2c$y_aadPs) zC)~LJqI02HwV)Y04J{0;T(LE)A|*~;Fr>?QTTzkQ@@pBZfRM2FnpI{mUR<4%yyCa4 zQsm?>u_Sd{cCFX*pJbkZFQo@6!{d%Li{QvL!^T5oQ^&I!dIv!k~>cd`w3+UVc) zG6r9Hm3^q9BJtja4%@$CLUE8w40kcI6E`Bfgl!mFpX~!9A*ZikwjZqd>1{Lq?}t-7 zv43^Wz7u8m4#{XysO`!Zj_k)}bORDpSl5dTN*znLA`rTPQibn`Hrk1zf`a5yH~`Sr z+kL003=kh}#q0ryq1~gi4FJ!|3>h8syH zpl(8f1dMZEu{M`?;*M7N*=2A3SX(4(o)3jmcbk~|bPhF@F^3`I@Y?l`eM&D_mw`=7 zL&|nMSy7be^m4__cSH?vu|1q>Xeo(*0CtLx&57-3E`EM~&0wIXscX~pFo-98)8-Q6{+PqIarSed)_wOjPgo5~HcG!07= zQsJAcalpc{#+7}5>82T|35ZvViKH{yW&5y}`z$GdSKPP`?y?LKUY z7AdE2Soqi$FjAls3BaUTZqMG~1vl0aHgMgrJh zwfit&nwQp-xa&$xScK5N4eN}>A*{GRYbx`aqh>vcDQC56xet$zXR~7r{Ad)0S}&%5 zAWUR3+DE3ju3?Oo(jZnfrl`<)x_09o<(93{tFw&sVv=cgb3ChGMK z{_DfZ-`K0aCjvP={qz4w8eU7{<3dkmHN~ZW4z=nMfVc(P=e#sb$`dg!r9S|wj}fs4 zi?_B%@Zb`5YW&K_i%uB?1TM3eJTZmQJ>5=R&}?1GidGwvThrKPME#4P0LH;Z|6qJM zuMWv#T(g|l>k)-C;Q4Sx?x0AmEq?1_D)6zm6W5k6r`Zi^Nq#wJAU6ehwns_;bT_kt$k(o@vX;ETD5%kx(v=w51q8o>%UyI!B;EA@fC){=h6+sJOwKmSw>+ zD18r_Dr~byn`@})`yP77*UBKd6!w`YRC{9l0PkGYekhJ&o6)!R_aT|!%CBT&3WAJM zr(UWWlnpxrlz*cTytTKV5Ca_aWSw!v9@_IIV+fk)D`CI^<&t~THYG0{85na6na?u z{W6Q^gr2t2LX~X^%~+0!F+pR%supTO=BF+;upcnA%dFNxlLjy!Rm?0f6tJn}nkYd_ zwUQca7PO=p*OBgp70qtUtKh7I;Y~<(YU7&2ZENTD=v5`_WxwD?{o7eVm0R6TxK=$@ zB;N=+oU@Aob&5;t_JMH_N-llPWpp&XC@5c(6U7*Lg9AAWuD7b^hFs4iyAoxuyb40tSBEr=AWj24B8z0=_kW+D)T@>ZxH)N7|`wev;A}_^Q)3&GuwveC)9`u z6S^8Gt#v6O+KT6RRA9^%`=<+=u>D;v70jq+VlYzKrNIiR5j1v_g*7<=GokCts-*`#C3mf> z3<(fmsUKJZyUX(pS#zc74V#fGBXEQ<3%HYlKyiMSlUisV4?8k%`tQRsghI;sf~Yz( zOtGxk)QLkw$lpa!ZMC3M+VFMrr4_}Ht}-8-JobAQ_DF6Aal_oD$Qn@yHg=$r&0COw zij55yLNkGN=m?fErD(Om_O4vJU}y}bjgQsrz?uqd5LPj4kX&c5Q|@=@t-x}wXhal zvgWYY(g0*a^NP8G2G~Cw7P=OQ*zF}Bwn!VAy<}Q7QJKik3}Lj=i#6E*vx99Igi5WI zqFXV*O;pnICBuD(CO^hEhmGJaWS-{8+v?fYyxe46U{;R;yB0o>M3RYoncGUjU(4bYm1X``2&uik%KVDT4oo&*0FDIW-g=HjNKmwS(TsWfp zW}3&a8JEwpIujdQ%9jDkpt&ar=F&{cC_IAB3(O_-*WKmNkB~fTx~t> zW`OV5VxX6Oc#R8OfN)U=dP;kG>*SqXp2N#d2U@~E9A``Rgt}|a4C*&!fneLhZ;XyZ z!>f(2-#~K;>s|64$17&OlnfGPI)h$#RBUV{nqKiLpO&=J8?;AE{OFSbhq-}?FO3d- z80wc3UakFxrE_i}>r3wixMHl z)Mli6M}?I!U6?HD&s3yIF=*DcVhW5o{K(nmxh2&nLJA;$Nl+2qDaJH+i}U!%jQp@l zs*glM1{yjxCSD-mA~#$CxT_p0*y}D;XBlJOP!=}*Z5V-Fo;B6kDk)By5GA3wsM543 znzqyy4J61|)_W3)UTK8B&4H)G<_UT%xVpvNn;OYsHT&KrF? zjj%dI0$bDud8w0lP@1uY^Ipre;JrD`ZtiH6E6C-inqzaaNo#l~&0NxsmB`yt)Ty@+ zDCQeeHN#BBqI7w623N)IfW*s zi=>QxcRAl!fCW3NVpN?UR%tb%TKI%^G;CyVhUqo9V_BiM0ZgYB~)tWW=}?97hej@-{#8MpltfYNnXjaw07n7>7+fgM7516R^2iM z;T|rn!)%>v!w-Ghu>N8)m)Ra_i{Q?%V9qJ|PNfPs0qOQRt@oy6q4C@5S&y2}qvq2& zy+_Tb>w46D+SjjK^YNJk_z3nLo+1IdQJF7x&Ig1lpyyk!*<{#WlyDva#= zn(`u*>{}KQnAf1!PutHk(wgg$*P_eXE`TkbVqn?;o3diJT-IspVB=!IVI}Ik&Q*3(cnShQiyIT!3a+YZl}jTADEWNVLjUJ;AQn zQxA69L{~@KSBBR}c0hbC=BzPUazWA}u#stmgCjS>-ME3gv72(tR)R+duo#yoH?qY z6UktI74&4(v&|G8SPac^jTDv8XV`r94rca*iW2(T*?b!j#;J3Y%Ep@{-12r#qFRc3^g!oDst<>=TN7*Et{sCKL?c*G5 z*BA~DxyU2O(Tv51_4v)_iku4~E4#XT905B>9GL(9bo$r3J&p4=Q|D9J$z$KhvoNw!t$Zfse5`x?!86$u?a| zhIn)?j+zQ-EgIxyIB*Rg!EcEL9*GTU-xh(#iM9`^Hyvulw}#jk1yw4wOw?9ohAjvk ztwfU6=_-KJ)M&6NMmtvKB#;MMq9HdqEn0mPLAbSjxl<}kcGhc$s)a7Uy;VXbAwSSv zymytq2S0*bkk@JLK>~<;Z6tJ?AupsRTx3O^W7ie{5B8Llle5Q??uEe3s}6HraF=%$ zrdkiW5@#Ta2*WP8vMYSQvrnWKDWbs?b|>;c!nXz=S@h;8wHxhZZKdfopI3~$SWzKZ z5fs=$i(T)Ssn#@Gp^uIdLkS9|j}=)-rOBFRD=vKL6XDCG4fK&IWiRnhQMO0696Gi= zhb($FO?&XzH?4>+hO+peO#b%!O*aKg24HEB2}55Tcs7o-{Y{X_`P!ROxmd8O!sVSa z`$80MyUXv)HbGg*JCaLs$G|^oq!EZYQYn>?gV{p_Y9`lZCD&ZJlv|4v2NzZ<>S(F! z_6`@dXg45z{eoy(EtxiOh^W_d7Oh!xzqR8>w`4i?Jy}|_*uS9)A@3YSh&qHA$%U+J zD$daSrz#m&xdpP6*)10PGd7k zP)J90_=&4rcXvD=u^8}pdl6hPx>*EbR)O%G32hE{6_@o-0NN>e#j)Z7z=pEB>p2%q z`Hp4-hHO{nTsiZEVj6S8A^&ZYG$!jGZ!=p6 z=;WE(u))Yd=QnG@%kw9m^ZJ?y>^;kDE?rsFOWvFL{89THHZ9v~1;yLqTN!jXJd{Q3 z*Lv{RDF$43wjCrnE14Kb*Or33ZtxTr>`~dQLhE*#hUs!bMm4S+f>=_e ze5Dm2#`a{Xc5YYDC^8DNJ-T)bf2*0~cEiXh)S+gQs-xQ+&L%!sJJVEYJcd_)Os)gv^ zv?S4tYO0tXGRL*$v*sN`J;F0$f5P6BIiUk2!0j=$zHl`6CKhr9>!i;(n^E6m5!YL6-Nq#8{$yxJYTpt6xAec4^|BxkozR8F5wUc+>nYxiyEp4MPicG1&#TQ(E2+gFJE20d-dY%`qj(l z5l=0C8;3if@|NsEH3+Hg%mav+d#F} z>Lu3`l=Sc;=I7Q|B0w3r#AZEy=)H!9ByGyIwnD0h?v0kl-GR~+uR8$qG=oN3AjVpf}V)UuR? zTz2=?8nF<+!78CsIyEbNxLOl$cDt?7U~^luAkECYG!Ok`(_$b$`NZ#8DBNC+A^*ml z;lYCT4|)|8Sr6K9x1`0mj?^njKHLHyDn$At)-PuEcUK%FgN7o-n%TGa);--Of@Q^~ zzPK>-W0qAT81(yYMPur5mN{p~!?%mgRw!Q}i}iv96FOJ6npIJ3nm!NKIzpFN__wm2 zlCt8rykN_iA#laTGAQm5mJG2?XxfJS3-hlgJGU1woR7EPUPE6d@E0Gf=$L^-8*Wag zQ!pu={L{$Hd2vd9VD|eG8ckKu1$`xb7aW+|@Q%V!#d7}k?I#9;kw1|=w%~9E1&JH) z@j>$S9u(SH+r3m6VB{>N61JoBl&HW3Is2<46E@f*ROXM4V0Hr%`zM}xf@*&N&E+anJdNPOqb+JH$obWatMj&B zWY75NgaGQOg52j9>0_oaoBNSfvlT7RYQ1{N71V}G7f^QI4015A5tQ{t5N;Mail_~Y z7k@pwB(yfGr)?LG10{oIj#rqOHF}2jy#wawAUwqE?l*kC<;NPZyi>i0A1e;ij%>6a zEAGGg2%-1~avC_%|6P1nTyV{)menY~LF+AVLaCLOYdm3K#^i(9wr|0hcfQyK*c90W z9SGL^To|o{4#b?ba2TeR%+et*L8U&3$X4ObAFGt_Pn z9_Gq+{Q=4C(kPw0{0GhR>N)b@g_WeLsQgBJF>eT?bYAQey(w+E@aq?s1Y7L>;oqNv zB=^;q|Nb<(`5!m$WV9veqk0z)7H7uC7Pn%E&745Wy29{ULrd);2jm+d2zkRb(&ejZiG`l{ur+wsWtTTr$=b%&}>V$r(-4`lBQ%3`*ZRI@115~7+tq-`!TVe*OG zQD)}28bh9=@s6lr!Q$ZHx}+5y7F?<^8JNfTypvhX`9b1S8S-xkKT|5DV$3$yeNxOi z$lyD*^VpZnmt9R{(kNW#t6_Kj7X6DHtDtLlU;8Z=M?-BetSV_1!SDO4CEwh_J5YY* zAmRxS#zlwvSGJRndzNcRA=zC@*PXMJF4;QFQho@^eDGS62cz4bZ2vHrB^q6nPQCAb zbiQGl;c;ef%CM4Jiaj!Gx?XHM?Jwbp7rs$%zo6lPVEnRoB^g=L@kA+{Q>pDzI^c}{ zE2Y)nr%C_Nqgv)864}hdZq3hMS_#u-qA81HBmv9FE~Y&=N6*h+?tL(2rSD`lrbtKo zm8}RG``dOBX}>n|d7Zs0ID1=dfcnm0Tq5>L>&PP*_k(!9+tv~MypG=fTymf)gJ+x3 z+3Z!7;b8p(QGNu?aW7ayhS|7=;!rSJ1!w7@sfZiR-jt-0=C}S&w26p&8jbhZJEJJf zu1FDf=dgtp`orz9vBa8|w)tZ&@5CLg@<_kFq%;Ra#G$g7?L>#>9qdcGGRk(&$ee41 z(R&bQr$b#k?kW+DQ__$TLo)1F0=t+XGSjqh9k;NbX%z0$r%%!L`1JR`|2?)U&RNFS z+dv3D^>5Ff&B*^gyNHGK6GAhP%CTYYo5_~o3D zmqGYlXzQbTHP{zINKNQsffxv6m@7vaT)`2(T)7^Q2FD6MQ2!RpDQFjVHE%G?629X( z2~1%^3Vy@LLP7V(crJ;nB3v{8d`Y9<-p`x-blnQXhNUW zc7&9BFRZmg=v>y?MKIv+gJ7}#1C-*y6&ieFTpy1S#p>JP2EZQL^2vlSnyttU+f37& zQe$ayXR&x`Cfn*9*!1EwVt(?f5f}U1!YxSr3(~%*vVveLM5stE0xON}%L1X;MzOHB zv1wBx)bpLJG;}ku1g#)wwbO)!IUB#_GeA28|Dh2(IR!1F3sZkT9DXi zGVVmEgwTXq zhvBtPuc48hhWh#p*?HR2G^Vr1UM)J-iS^q%hkXokM5{2Imn=rWBz-b2Iw@_{mT^r^ z;zF($WJkeF5_(5L^aApL5IhQKRA3H-;!!{Y0s4Hg)iHfM0mLiRdmI-rn6M>DkAfLU z-2;;KD6lapd`r?E1vesBgFKbY3;?6D^VPyA{v3g>812%FQs zY^L@781W}rqu*b7+8Nqw!zvpORWjQMH&jb&R&JPYQ@}YHqKokWLi2zM8NeS1NZ-D8 zyNCn#BUjM86|FJpqsB|R5a9edHgBUsNNl35CfHYYMAJE9xfMvALF`NwcnC+yj{Ei1 zB>{gJ+e@;w$QFiMC6%q%nvRyU%=Oyr%%v4C?<%e}!wQP5S}P(KW7~j|mRIrU_SuMb zdwA{LG;`sma?sLV21)*Cvji2p7_0n{AY#i~&YE- z2lp^^*I^c!4oi%)u8i8Hm?dW7#GbLrRm>9OtSjSS-h=yb2WFBxaumoROAPGQD2T(B zn9#a13UtI0lkC&ng_-2z9Bq+`TVhC-AYX^qzm_E?5N{mUY*}T-8#=-)9A}zo@}m=C z54SJt%<)DUS+$BF9v{zU$76QK*O#$sTdfz*-ns7bWuvUT+(h*he9divB|}b`8tG>^K>)L? ztS1)OY|YlP+Dtq^^W#Xy){MqNI|X?>tE-9$U2HzGWbW55wUcakwKvpPpG6Cqm%&EM zj|*QdTumBtjJeCiExDytG{(N`+FQ8J`7Kv6n(@FJ+O zPX6|E`X7J)?a7Zb_~VPyKc9YZ|M>0c>B-68zPb4N`tsG^`RRwhiF$p5|N3z9H}>l9 zi9k+I|NK8j4z#5gaG|HNn&Or|C%Q_GL&3`}_hIL{Vf{@Bt#Jwh0VTnZ`;T;R_f!7{ zJE!!G?~`bMF)X&Yb{3&A9u*^mx)W<-z zUvZ$?m2Ec|>=wkIjTvh~lp4mLB?3*2QUe!vGU`7@sSRI`QEHD-YL8KBV_-f;sXa!i zJw~ZLMyaJ-k5OumQEK-PrG`pdN32@nwSXfxXxIU^Ozc*za_ zfRaFPAFcF7ZATjxe8w#wlFLq>5(x~^=lg?7dMY5mcu1Rv*r9gLDC7W3C8o@wzFD-b^^~Ohp z*|zQ!&)2w(IaAG^6Izy)q}ht8VC2J{{+i2{8Sx3oMZM$#l>&flq{L!FDygYX#-QCP zhFIl3X+Dw&5k)p#ZJ>mfw8IJ3fd>a6-Lxc&+?8Mc!&9UGX7;oj9s>m-wIoG)+<@Uu zi*Qt}Pki}1vXXVBW-rJ2l(Zzz;7v3sIsICbf>EW(v!`UuMQucvB!Gd+^6$SJaSYRy zm2@?wS3xPUBbC)skm{rw;sD>UW8GHZhGkvjQAKF^02C`-#!r;klLeN<2Zu%ZK|nv( za=y`N<&^BR?K(>TX>ab7cX)quRRS3vJ|7Tr9CS-nuR{FAKjq z(Mh~>z$}*xy(2U*y;H3Ig=M!(%HzcM#E*dqjjh-!!-z{-QNR3|T(7A*MJEiQ*#H-^ z^(zfutZ31!LJ<3MC~|5hsEGPb+m+7YaU$=8>lK6je%n|{5==&aq1g=+Irepj#mP5< zu5EF0-vOKDTd)TrWzjT^MRZ(UGZd>1U8!1)DC%n-9-==AP5%|Y%~*;oYG58ZiQ8y- zy`5>e?F>!@(0A|mg&N?rFk~gJkGbkmTZl5>+()bEl=jAWbb23%lG8yxTg}s|jU89> zyiLR2(OfF>mqKPY)F4i7G!I>9D$DTxHme$ z*~D8wrS7ZAbnCSJW+m^YT25=lraT(^mNelTwt+yvVC04SM%Z}wmzUqE zA-ho^RzG1ndy=to^~6HopBUZq@~oxcT2>=Evb#5s3udw?Y&AEzATLzCf)aXE~9#tg3j<6!G+_LBc3V!HbWvx{sa2e)XNiM(K;kpfI)wB;45{ zkThTOf&A#tfyW($~k~8;?9h z$h_ZC@Yutltq}!SkEP`gW!g)s*&R*i+4lLcGt*W-CWj@_DV|-Pldlof;k_g(sZsoV zDc6*zw~^g`al7eG>SQiG9AW_lO^O$MSsT68>XV{#f0ME1Po$XK$L>`WS<{^VpZHS`$jIgaMVFsFZY7-Xg%6TIp|j}2Psv& z5_Dd~J*{no_OFoF1zi#b|9FyfW&TgDuHJwrr53|2d2QmmrN2I-6p6y&XB7$EsOAj8 zJA{keOo6FZ{W$x?V6Zy%={Fc`G-u>`#j~5sz_FH_pBTzf9bDM$zI}qmNk_nSxo~T9 z@MI9LM)mV+UfJ#E=R9B|{6sF-%x<<5J8i?&X1j+Y$%CGMw*SL}dz-A_*!#O@*!@q$ zyc5I9Pc$L6%iOZ%SjT(Ty#%zSlx1DSg3xwaD<0r~!0#X8hT5lV&T*KS^Z&E==G~3l zR^n*>_NTyU`#egCNJ-u@>B(<=6m_?)=U()aWM}dmPd8Wvl8A~x4S=G=jOV+*gA1S* zvI;;^RJ%Ja&a`N?;Uj{qvYupcs_(@vl4sU5SW_SZK)>*2pu61i zze& zm1NgmBz-$Tg)_P%GqvNBI-;_&_mzv?BkVRV?>ai$x<@5SZ;o|G-gE@my)t7n9)TsB z(T6dcZ0DGL4b_Tl<~T<{?8A$J-$uaPGeVoN#4#g zj)2&bi5$!>jsRO^eN&Jq!P4f~wrv|{Y}>YN+h=Uswr$&|{zB8J@2wiRf1OM`_u-X{nb}_r5T#U$+tdb-&v=Cceba z$F4E*b(s5|abZgd^9bM_0AsYA(H~=gC)%6c^&?P8kbo(JtZ_)oax z$#|2RPjx1aJ6=!n9bFutNW+MA^<91YgXOjgH*mJ}F_866OLc1XZ-V0KrrKD_giz|5 zE-^jGzD)zaDCatd3*;QJs~D52l^FT!In)*?- z7}>^|v8VQnr*-FhN&zV<5cHKyCB0B|GT|I^o_L5tEJ*BJdzmB=M1HFaW3 zt-Xl@$D@0E4MXj5?S~EgZc5W>YI(mxO3s>^eYwtw=$46q@jd5plHSnE(^PCBw2j@S zE%o*8erpFR8x1`C>Ze7R6h@wGDgrUG4xqBo;eqb zN$ijwa`6(utar2VKZKy?ZtDl5(O;CS*B^*?;7?+vZeUqOVNGO zmNhq~9fO9ofD3IPDm5E)>_tJgxc`nF=S@WGNX#Sbnv;<7I-7&91d$FRm{FOUnqXY$ z%bdv#yan{TApx?Xe8~Y}IVU(|FN_D;-J(omoM=z??D@kan%NaeZ{?{gQNZ0*hK&KCY;rWx_@T3>y2igw81-f zwyFt2*N$v%j80$4U-6Azn_6%fL7VHXLS+YVB^9ZZ!;g>_Hm)Docz*ot$|bJ41q-sw z`5K~^jh(mhtwwK2E%eD>9}kKg;3p&DO8Ye&U3zW={7vQ4DyN7r(d_ zX&RUqrQe-Oo8;!KYOFDlKG3O8-i7@Y_rbs2AMQN^zblGoGE-MY4V?)!RVZHLL2m#r ztcBQ0iCFo^Cpv6kZETv}Wzf>5zXHfGAahJKk?8JGDkt>IK(c2eZZSazY?J#CkKAQ# z%O?IfACBB@rJ(?jW0~h`^u{Fl?mpbfk|?HM()L_bcYAnrQ}%m-gx7gzA?#BgFQGjA!2IS3u3k<&qr1BHh{9@v0v>@HarVt0h$dlG2b0V{wosiBq`no+kbb{C- zo7Aq3oonrS)MpQ{!sSAV@=?XwY2B6aP^5vZEVh{Oc$30WWZGuUeSm4?Ec1N3`s6^o z(7rQD9zT$K*nH#Im!_IZDvfqxfG-R?K8cKyVUkSq%-qu`VSG(X=2{iJzhJcoa zno9`S0;+U#FURLSI02DuL?W{C(BsS?b?HJZz?KOsPTbc7&)~^$!Hk?*<`0Pfp+78gFtwR3kO0&|gz<&d~22iNYh3!X6O)tbBqG zlg4}^jZ(faN$dI<*oGL!*G5DLk5A2_6r>%{L)(U4o^OdD`>DO*JMLja;_3@P%i zKtZc~4z*?yq)bYQ*MQ} zfiUtgm)3oi`%KqoY2F$opekB;oK3L$Z*C2u)v%OWDoB$<>aAcb)mL|wuZ%86$;GCs z1Fu|4LjIveL0!a2f;OP*ETgfg9;MYAYpBAdWydbT#zHEz=} z9K}TQ`IjHVY?9Ur3SE02rrwZ~6+<;7`Gh-}ta2@!&%|h!)WoxPr3LZIU@{&1Ay{fS zzoBQJa-6wm#k?!J>U#mdM-ktXs!2#_Eukbq&^!P*F0g4Y*zRxkz10CVYMm2%y(e0# z1bexUHiD9WeYR(F@zPSsg|2PqCt(1~;DWHiAda26vG2yS72Q6_#^Q#`S*dlO61xtQ z>BoCK%7RR%5{FB8fj}$|EKAU(g+$TgqktvNd#7nijn8s&1*j6Pc@2~K8GXGA$uwEH zBYmeTj&=lry6YrFbcl9;;nVw}SFQ|p`gNL2B1>xG`rG1!(V}@mpeAwieWSVLqXd*; zS1i}eKyE^lkC!GsGxp*;`PUm;N2iwJVyJPpLOFj|59vh;@#X6~8^l9D>CPo`rV(ug z$myes2yOOW*gDp}u!_h81eLm^FxXXVYFzXxnT$p>0t~^ivl9^uDB&5%x+4~*LJ^2A z8UYRd#oCmsgXHCiC#IQ^$WQ`P-eaF1JEq0tx#n*nd3k6-(23h5V|{Rs z3q~Tq`L4%;pquc=OMAkJ!5ioJZb8x3sl4U^fDNm(94X?)RoKv7j>D8BA-dF@5_nE; zzIeSZ@6~dUyZE&4E#K9k-5a@9Idiq-%+9oAy4K+x3h+nM#Kck1bczDB|VSLP>8Rk>oA zX4Bqk3uSHLYW>3+mjJ4fl0U^wetCKSZVMC|#c&;XAMSk%BV!pApi>um1bpv+}iK%MpluKC^?7U^gKr_N5rB){H z?pR_K7~C5jsEf{f=9Eur&7`#(hPt8n(6XJMLCxZ;%vyyY{2Pi&9X4k?&@LJ=P3fq{ z_$Y9F{z~zFUm=pFuJ{^oz)P0@pzAEgw$dW=a^JB~CvANB;UaS(YL~1r-KVx4p@Mi0 z8w_>adfB%>6(F&WL13YDbUQ}oswLasXU2m76%pBi z*K*!ZBbHPXc&V6()FqBZn{PO|HGGn8rR6EcipK`LhqH|F561ehjg}Y#>sXnOTwcKp z1gjW#_~+im#M<8X^UD737wTyE(eKqz5B%aUJ14^4l&_nOse|NzE`uMdZ->e6*7{Jm zk6dld2W68FtDaY1YrpB{B(@eysp?`mq)LOq16JaG{VnV$uONq6Cy<-rHnZA#*yut! z2DGeGiejdHY9cn$(@T1U$?2=3v3B(lrP|6#ZM2IV_+V8FX#6nVx zapAoz{wm=J8~y>Tqb)6!SW<;Rvg&2Xz znV##ymV+tZ7UMacw)!veb`FpQs?GKlV#|`#c4b?J61G&L4i+g-7#Oi; zFi|H4AM6Y{Gj=>k2b4uCGhUA}UfdN2B~0?T%wQGF^4Tq@K99=eG}wV#o`ZIfUILx( zD%M$HzVf@W0```G1jU2^oPg9|>i<4bk%KstGBbK>l(keSS2tO>XKh9LXL@DBFbT_> zbi^&HA=Q>sF+M)v?G)|9Pb86uNV83cSQV>>2fONzziVuw^2Y%^KPj5p*l}YD2)kcT zz(u@XWZep%sY7>o!K$MR8W~yGwkDf0TSVuUzgd#`*^6(yzf!TV-_=^e~e zg}fM#G^_fGC{nS2jVN3mbE$ZhP%ZCb`rH|1~`frKSs-LlWf4>U%-SFU8f(~Z37>w51=O4>QZE-T+~kh{1rO?NQ2 z?_AjCm|XO$tFLqyYb0~-qF7?z^z_NISTpx;@nHZzJA>Yi^WI$NtU9(oUOy-ukG3X>}s&JKfbll)`vb+1e}O8?v!%A=Ri zOK4HUyhXu5Z4*@Hw-SLhXnLZT5@kq(hqzelimOkjm@8T#B-MRZI7kp9j|gV3(+^XK z2@pfIbNzDvM`Z0+V}wNyW}}jkGwRb4En#N+TV|{`zAp(X_v2XOEip6=y43Y*?&v8O z4SH>$93YZe`IH5Gx7w+-pp= zWi8i&=@3JyW1}&`1!yWMf5LN?m+Ed7Y(jF++;JZO(?BN`2ty?RLf~o|#rds?S=pC{ z_=H0YYc`&iyN&K3{mdros%waOEEuUeqLMZ7A1}q0?S+evmw8>u*i4OA)Rq*|oh0;a zO7oSXLncsAJt9ZBVyM=xy1yBr5}*x)Q=ZY$S5!rj5LNGp-~g#+W`*SY1IZNIAV&<$ z01ct7HQ$BZFk*dcGI=G(TC$dRxr%i;d>dOH98?UU0=Gr=6U)YUzd%Kh6w?Mz99?V@ z#VeJ;Gc~jHbn>E2rlhgf%{U&xniu#s{3IHobu1yECIcdE9&W|`U%{h4+}J;Uc782< zSy4Za&nLTX<$AV{QwsO2Jp9G^7;NpMF8(L1N>vzv1Px=#sB4m>3z`W;-)xlf%~)JT z4dO+t1cS`#6?f}PJyyKPTjm6{zCiExf%kh|RKXl{OGGnxFUQh+nma@IHM$#rla5v8 zReLe|Q^IoPcz+HKLFgEW-L`$G3Ig#6%U%sj%TVZObTrvA_s>W(P`zfyTawSl4rBCy z{@NYBa8!R1WxhR~EupSgUhLC;c|Ljny{dg8>=fQ|5607~*J<&*_V6yP9<{h@)k^H; zrEN9Lv>k5RKA(5DdU5r0fc?R%J=*;sCjGGZY<0Tm4dDK28C#^Q2KZnH{!A^AQOIRz zP%CrNY&~72y5fwGb$tnEG=ez87JSER@e>l)J88wEj9WY6*ES6JO3C{M)h3GV)`(Y_ z4Wb~*`i)9hBBjc7@BDpBUgt)FPVV2_MzXQB@K7vAjjs7!XL z@_t8#sbB3Anp%frVjGUM?7=b`2~Djb zFdt+8m$3~+{f>OAg{IX<-)Y5S2K~*5w;8jurMr4~!U%}#CfsVkh5=%CmT*TjYiO7y z;#Lw~s_QH63kU#PI72wVGwP5k<=T*Dw<}qq{NT(~2!l#NT=(Ij-bN%R+VsS(!AJmEzWyr^j~FMB9gND?yd& zMu93q3$&LJ5wC-{ZV!@3!_g%lpAaNsg2yz8778M(Fua23if~4SvzZ}xZI5* z_zKKcb`kk=vbxZwfeP8b0=m=n3?tT%>ZGnOMLj!D^pF^dyhiO{2rFO} z0~h9uew;6~`xSLsZHa<7&aJ+{jK5aM-W6>WrMjFxHRN!J60ne*i_Q>?%=*QsqL3F7!<9oh|70l25P}>)@>-T(;uaLt>i+_)p7g4|wSp zs)km9yG_Ny)}(Ir}MP=-(1$hzHTnpoSO zTlCmQD};#b$3N)AwrhH_%`7*@crvzDm2*l=)R#%d6wJvF^(;D2GD3O+EYp(OW~V73 zc@Ssdf}U1ibKRAm7TAz#v<7>nGH*SoaqQCUI7!D5?vG<=oN{&*=1cOWhm*&P;*e7$ zBx9;N=xlM<+oW>4&4f5k@8tvpj?S)(*IVc}InFm>`2dWw0zv<@8Zyt}yQx%xQQ5+hCJSxEP8nF?4JqFBHg z8Y}h@Zutl}6}7otzb^r*>3mPc5-4%pg8k5~A$m?Hf6qG6UVXF*JmGhO7gvzbx|>CqWkm~U++){IwPN2+ zx18H8xz&7A{-m&%7?k+iHX)gP3l>iXiz{gMiRyO~tH*ov9kr4e%F5TAW!b6RH116a zy<0|4+U!oxAZFk*faq1I=LXXD<8B^{s5St4_6)U;eE!TIV#b~NDa4e=Av=eXm^@TL z|6JIqdwJoug-Vewl-3Pk)>k)U5@9kV!sU6l4AaKWZ3jvjM(?g{+#aS><>Ii~il?s8 z%qK!W57p&VOKIFwif?%W`FSy|8`k000b8>VY%Sl7@|Fmo(r>&UZ7hhYrdn4I9}Xti zUL_D0ZDyX2D|QSg%U7D=6R&AroEky5T(MYbn;MQq?{WbiCfeL$hd0A|z+lq-TD?GB zkDF55h)~hpWiRb~U-qMo140r;BgFTAMt;A~sRd)$H_(d-uHi% zyV<^PPbYq#?}w4}ynJul<0s^P++XLv#xD3>xS0Rf*p-{6@clxJnVV9>9&khERDm|! z6eqkUd@3=J+PuMJG|K*etzBpJtgEd&%(B9s)}335Y`G-b2y{XnS%-OouWOPvr8RfsIsQrqy= zKxh?a`{;NI0ZYo#u{!gv=Lp&{b49rRsbb6^<}6E!gU~dvY0Oyv~;s^7KM{NGt<03%% z;fc?IPY;s`D!~!^^nasH12c5=zMua(?k=j={O{K+aoXQDxeu&p-Bhxgb?R%k%fO(m zJujcKrj(Ar1>jhEiOY3F&laeXP1bT@pQp|AdjWVk0sRtE@~k^&gBwv1p&%1(ISYSi z47lV_nLw-A^nuP9tS6Ol)`_64NdCuMrX-^U1b3{=DVcD4)S$0i9UDL~XOX}Bc#u>j z?n-m!9VOC`_m`vs;Sl4*8Ai#B?8hhFmx|_Uy{vGVu(n_2Da*g7ZpBkssjf2G)qqZI zw$?80g9T^g_hp*Ut=FB+MZ_a6{KAe1ud;}H#KmIp z4=2Br_nb8=mCS^CF=c>qVJSM5<`dq;Oas^V#S;CCz&y^=3q?W=nx~U%h-~^KZb4^3 zGJhfv^2k2djB=U?)kp``Q1d0mn(KTHd*f~36{NjFMXl|j;uYYBVK?MtX}DPVfmDzD z$MREDxz%tgT63(H1Iu0{vy6mrx0mx8{H{CGqVmv(A?tt0qo+?{)+8{ODYP?3AyV=K zAp}e(F#5Kd1K-^sZUpd?2ao>Rb8iFL%xFVvr&wu=Hr22jmYKf9rojG^V=mJ2!ZgIL z0GPrJx}NIzI+pt>RFQ?JAh-YtimVCt<0ITRG)2}UnC7he5riX?c7tMF^35!X5_qg( zYYdxPBZ*{Rf{L>+p(-()L@8oV{H+kujD-=trjsw}HpnMjPjt&3GMQ2_b<1p{&aeog zGq0G5w#X`mRW1wJiJ&Zh1H&_7R{eumk&0GfvG;9lVtCW^MU9`6F|`5%Mv1XuQ8;&P z`uH|+x)c1T!tb`(Mok4Ia)Db%JqevDE@!R)9VYL*mL4w)>`?bi;N$2E7o4sF-|gisiF^3yjm|fef z>p_Vt5i?cHMWt=?y&6_wPvhglJ!_Ca^3mf89tg+y*)+`K0>Ci;OUhhenC1w8VVMxH zjPq^}7%RQ$lg*<*D^nS-YDZ?;EO?y0+*_)f`gP_5W2J=K02i~OQ&7^TA^+g!=41ue zHKWNt51%M-VJ~(QlP$J4NOSi4BMogB0z$jQY!Cx8urTbz`KU_+WFChth=o#%l*cyB0n zkn9cqZ^I0dVuDvB4PiTf(pspV%;*48kxnx-ipyn=Dw2 z%Rdg8Q88CZ8CW9oAUC_fH9^zbnJ;Myh4afqs3f>*r~B`<*lMg2^qE%oj&OyDnYPRF zZ|vlT*7EU%n9Zb)8PtxQdz&9?FsaANUYynEJI!DLyHjqE_Ye0h_*}c7O>5Vhr2(Lw zn%+M;2=$`gBErSn$#3fp+cN%P3K2*&u;sCc#2@FMo$mc}aLq`CCX3G*f2)^~3k^Ic zcx+;Gh}@k?M|>smVwmZl6hw(DIG&rJ9vMaRvtbm=Y89YF)PgQSk(yUxE=>vrJ1wm` z*DB;=D-w9v4GY0c!tN@_=lE47a@@JJE?nho-KmubD|dNiOL7b}_vQQgH| zrmhh6=$-VGs#WGLA@BR2*jh(r$%vXF(;=>6R==i2%38-Eo|v%nze|XhcD;jL)jjG& zPoGbfSevW@yJwp0Oy=FMcdGcY%2=^r+2j+On%~V?ZriAe%fRbTby``l9Cv~KZ5ojS zTRt9iW{N6kXGUBhDt1TBtyp^_xFr>XBLh)e-RokWLB4?c;*)2ne~9;69;I7OlV}r1 zgohtlDz9KzT_ohW#4iA7egJS|hfyM!YeII?Y{T$W=b5rJ7@Q~Qlg%TW<>QV+O(aUF z{4GiyeBJZ=eyMB1X3cbh>A26MhVO86hv{I75>Uf)KKc(aVF%Okm`euNY3KYE{9$qEKc2HMh6O2J%L8$_m+9E(1n$LK*8|+X~Ysy*tX>#4JX$U!f%O zP>dG;?FQYhWfudu`K_0RG{y^}w|+TybMTlLod<=zWCGnHCxHr{QUHvuxCiZ028JHj zwgu3Kf}1K|!5@@W>L9*S#ykgsPjN6eF;@Y_^$+4^J1~62v)p-l5ivy^CF+dc$&|R3 zD8N!u_X6_jkw=}2!e`lI*e-w=NZxjrjKaxu(Cg7oyDX{K%+iTJ7=c#pkHGAH$}(Bm zPsJ>X8D?)J@huMnk17ZD%UYwkX}u6_RA=|Do&RKRx!K@TUDKKy%Kib#ohj!J))sb_jU!jv_M{pIv3^U%2A!+T#U z!GRzUXlr~U$E5TfG`z5xGK0aGvmayX?1V4NN(&gy0YCGGG;x@zOU%26#4Otj+i@qV zL&9lR4Me5-YXJ8nG_ER146c@!lHT7SB{U z{5@g!PuAYr%r=kf9?x%v%K3RW<8QUWvrh8b*>4_R#GRb9p2KOxb&;-@0bshly%)-jGZ5bo;j<9uljiXKlkQ6sph?rccsGES;Rhc^T!|CA+31oU%HCT zUZX_3vN(pd7{Zl>U;!f|_^Le#;T|Kycx(qyLHa=sVVgbi?z0+NQ~N^yAi-s(q6Ltl zqkdbg05Ud9{eMXcqq!lr(cBQ*NM6L(g@xgl(l z`f-FN^nkTmw;-aGKwYOulSo4mUNCJ6!%MinSJr8BPP~-^uieu5%dNoxw zW|~dR@>+K?xD=!~@T5Jzi5+^Xtt;@4H$(mcMuHhjKiVx9Xlys?dwc7%-HoQd5G7yM z_I~r$YaD3LlPtN%LxIpsdY{|3oxspLcs%IyBbekJB@xC9LRXM)nai6 z;fX&ky}792oaIGv1_a;`YGp_v{B3f|Iux+bB4=2Jz7s!lM*Cc~pk-j116wLfGb?l} z`5aYx3q9O#Iv>ddsD!wr`T|ngvelqqs{Inv<~f-paD5NzH1>)p;Ou!VOY4ki9PRCd z_C4k*lR8gNd`tQRKUT)Hxu8QR9jG2sGeL+jr5qG3zENXJh+@09r*~GWoHPZWtOt=* z?;o(v+&{KcRtI{y0~Na)q+P6j)`_7rOl_8QT~)YGQQ3;jFad4i5&Z3W+VaggQ7!uK zSAVnFhG>cMI4#G2kOcUBn6eNGkjWvs8_*)@GB2TEjGnKw7UG{U_8c<&#iTtRvFImX zStx?V(=zhZa0F4!y8diQLQ=xB@6!;rCyaWF4E;^$Jm8|UrOb?d^1;DYC5;V8#qops zIa9x%&NQ>jDQIMAE;fahFcmSa?cP)sj2%A6i zyu(_;!{}s@NTAuiV?P__^_|&sVxZw_Z1)DQMqk`-nW72UWZbfR>yTfi~{8~bJ;7y06wo*P`MNYawS zK%99G+-#Com)b{Q_62Dnlvo(6h|oWMZMp(nTo4Uc^9{Wk`mA?K(9Vp~{}V7yCk@f% zHh`?WnY*OAV%OP(aBs&(2(q|xs!QB%Pg+lETb*7jL9oXS&=Sc$a+t&%sEXqDC+*MW zk^~{kc+K=k*{OP5scbo3km46y&*#kTa6JL$C?(>}hW+>SUkCxEN{jZZ0;DryWD+f6 zT?+X+>1o#-VEFb6*8qA(+Ihf_bRj+Xf5xiWlcbDf^7maGOoicS;7vnxL8j_02V^sMIvUV&BuPp_E zjieBy=*+?M6Ta2R)`96V3CP0t)*=gq9rlR`X5qy&^hw1--%VUH>uLG?XCOQWA-Y#) zx&2Te5|KlSJYm$ixSWOL2z#<^b5sIk9Bx*%HrNSPCAVKni^}`sX3cy&ce8Bh+H&LC zGL0QDo~XvB61Wo}Jgd887FTBeZHeA zA^*_NP8$Q2)GL+^+^gbYld;Rz4;*ah@UR)KbL>x&nx0{7ZZgu%U%o0?`QjbiQ%;Py z)1Z?~ZkSBvO_B`E;WWH$v=8aLpB&+`BamzIH)y1?*-X*&Dsr9bbmB#2a5EOtNjxgF%d7Ov_mW4{^@ zJD+jWB>=3&Z>p7Jhr>jgTOy} zTVfIPA3&R}i%5#Hf$;?|u?E@KHe&0h9$zIOl;^}BLG zvx&_z-*X|eiOsdk|MQRr$;b5RLdYDvNnXRbL;UA3zuzs`jjwosSXau2i=`4=7S&d7 z;4hc`i^^S_Dr`_~C;ceGVfl?*@DnrFd7p|9Ts zp=(>h^Ry@R0%<6|o;gU{(8TIe6K|QYTOgGPhIp8KMZ|2(<5d+%D+(@&7un1SWRLDbW49gqJ04mGe3W3+*}o>%q)U>HjDLx= zEJ6f%|3)P(peEOImZnwj3Tp!y1Jw~%YT5QK68{1Pm^(36cFAj&vx9P8bw_UbI_5e- zR@71(tsKIZ$`t?{sZ$cIDLv|e`~3{k9}L>T!TJ8d{5d<}!};_9;`{u*%^cL*;`_FD zcelS?e5vi}ZukCX=J`A#SEaF=?)Bw-jo|*_L>k-QXU`RpsTIJaR5GF)F}=toA|*mX zrUGi8^l^Yq%vBw#KQxW?Klf2r5N3aPhuHxpXWBRYSXi^U&0wV%Rf*^4Q5@YV&A>;T zM)$ZV1)+&#ocGBzV!#6#wcB!^3zv@4-qh!GKQ$Y$!X7k%c?Ia!i?{`RhncMQKm!w0Z5E(?}tlfawS6n{gltkDo_m;~hwz z#dJ55`m;)mO_Bfw4JvX=Xg79CcM)7tMf14@*_^aF!SYc3=Z?%uma5e|t$Uz#o8L-9 z^i(Xd7peWGuKf;Q?S!~I73Z5r%y)N$96rAan210B81tUODw8+~^8pm73j zBihFSi2p1C0*X~DtG8j8jq^Llo5TU;T?l5mHdai(-QltYyfWIXk>~P(P*N2fYgF|> z!;36UT%}{`yO1yn zWha+IyY&nR*rY6a7sI3uVqPufTc7R=uSh-|Ejm(_$1XGtXyQe5m5Om6BhT6BNG2Is z*hHjUJLj(qzaXYY^4a{@;j8-xIrX)*tLE81G802;U$i0#pF*Y>nVIlC4N>72XJ${S zIpzd2ZyPJvdK4Y%sz=xkvWiVBv>~6gd6RgPRPI8FQiUQ9O>}!vk9ZS`uvM~CSYSsd zFcplLIjaVue-JuTx=t<@MLo9n+VC}u00`OnrDV`Jlv`6{e-^y7z_%dLPp5E`tN&dE z#a}4-7nK*hz08Qk&0|WeNN6nwr%dxnNr%d7kdzNdZ4!?bk9j;nO~hFU8NI(MQ)teN zoXR(ZSt1P1JJKyTt2PyU%tX;`Q$s9FG(;7J)1{Iu|5a%bbuY<-7jFlnXI)|dVWiNX zB&1`IVkVk{S)8h{gB_MftS_Qjqa1n7=8s(RG*7VK_xzM6-OjDR2CgCstjMdtlVh#6 z7)jKMfGROJMeBc~m3(EI1@Hsx{=%+@K2PO~k>bX@pr1UDtK{pa_x9#geI$G#Ia2{S zf8&v0n+Vr$qmO$FwhIz)@TnRLaE_Xh$jN-s?pB8eGfC=KX`;kZP-UN(Na}R_fCbWu zfGt+$uP$fJ7)H~snue5Qbf}r@KXrF*Guex{T;naoWGosl&P>K8IAfY!sXJEi3@-ai zJ6ukgK#{fym^tbgOd=+^$5e{y3oS8Fz!1y~_1=G)I*&+1#VPET6^&wj0IQp|WO zv?A3{%pM9;n9Yta3!ZR+%oX+*q!S^kUO`B_JpyCprnc+TwuE)W-F1 z7Q>SXS?5xMf{#Ib48h{o{E9jQ^a2IKnb(D$7tan~#KtAPsf}vsLx&pMXboxi3+sAY zU=-JEiyh|4X*c#u+8Z&N!wm5tR%;Meb4xBs9FbuS2SGcD;gJ;lW|qmJ!r@l!eH$X~ z^%CD&xm%f%6j&yImAY(1nP6^61sjvB;rjhAIxRtVdDRX8C)B_4oc>ikpQm9pWM&y~ zNM`3>|MKP4Q|w8cA&OH4e2jb`!9Ky)xgj(-7dZc%zcmm)kV z(2mvaJA);bx~Fwzvg1)!mMC_L)zU>K#BN|7c(NkLE=pzu++y$=B+zD4`oa@N3^{1+ z3WEMQsDU_V$%&*@=M;3os`L>I4?qeL_2SEA1WUfHS`7xyPlKrHbP!3E1TRY}O+ANa z=E^p)C8tgS$bpVT6faJ=V|9{mxI#)yrP3Wn^G=Q@;Hr95lO1&5QufPZe_H@AAZu;A zw!;eXF2J4@FBL{4CI(6>_vYhg1NnoxjJ}dUo-U> z>TM`DVlbr3eqBakZ&_f2UaVk)ANub2FgHQ_J`7#(p)b*l7(3vDUYEwdWb`0tf1CaH zDNbIJ%_)8N+SvYE-v8d_jjY6_5ut{+7GJ@HKT_!oC zH}i{XrTx~d&Mh2kc>36;0f~ps@wibx@a-;$qKhmRRV%MW(n*#T)|iOnkNQktV2>^a zB+Q>fE5VeoftvuT%`=mUCkf^!M=6C1_V;C^G*egI=XuIbqGSXOoIuY7cVi`)nG(U1 z6YA~m;I3`;GK&dTZ25wVxghNqCz^XD-1gfp zMZNlGL*MVn?Dha3yaQ)p?a7!yqRVT6bVOP&pp&q46KgeA z6peW&dQtu+HaD5pw?e5VwR3n$pwcA+1>@T8M2nkhw-YgXc- zkB_(LL&K)nV@qsB+6bklsi$JR+6jI_hBG154(#N+Jp2OrX@&N*0+1cskWl!9Tn~ zc96Ajb{p(tnmH{Z?s!LSosi7iCr0V39`1Ws`Zgbj*gs??9gkF~;ZMg9Nu7Hk)>#Tr zvU>R(SeCv4^IAGN99Iye;o~q;C7I7$$KrvOTR2q9SRJ-9Dvia5xvUNj;Zc{ z>gg&#IkM)GHZ*=Y6t}42_7jZUtY2KT($9Y1e-@YCamb=QZFs{N5+qSb~+qP}n$&PI&JGO1xwr$%!d7ii4>i?z{n z;~q6X&AI0KUDvfBg=7ri6GA5y76ZDfB822Lv~)z`m{cN!krWP~*pw`IyyM}(;P>s! zS#FmGNn?;4X1K`Cv1Ax1$1!Pv)f?)+5+T!4?PlQ$@E2i43g$RjLQnXJiC>cN2>4Kq z1gc@cGSQA8e1oX$G)X^*>;weO?DVR&OG9I$#m1FZ1+C+YD5(zr?&aZNo9BCs2#i+( z@M}!BO15@OWb?9|UDqp)YoO01G5Q1uPdl(XO(jNP9i`CKBBGO}V@VlYJ1|2M6^W*s z)){s^gsl4GN%REx6Js-+p@n$sPg~#RAws=n4fv zqoww}DMjJ*P0*8v4@uC69tvBpe-zg}5HR-0x6$nrC*`cM6MM3)f3Z0^Egl%3-u@p}a2iI7F+%3!JU9Ba&DhCH4FSGycO_TI^0r9DecBaY-V z1z-U7%CkkZdEyo0@UC!CB+&Sa}@shFQrd%o9H zzrW2fz~d>$+s(zjz>go~^8CJru5kA8>*#}FCX>bDk^V9c^CLH1oVP>ZGx$(Ul2wl2 zJGp`rLv7<>=#W6vXGO{-Z2c0*e<$&ZK6yh}E6I*KZ!|fFbR@o7%;Lv6FziR-r6h^= zD0VMX@&*1tF!v{v36F3xMC|^*!_Rdlp)|O-Bf$dqVf240j^95Y?L+Y9Uo{x8|ID}I z2o}7;n{Rziz?)zFdYc86gN{2|H~hCz^Ut~~47_YNMxnIksQ>o-AMG`>Zta+t*T`tw zJe;O};0*9Khw7w>wxooC_kECwNlF3S@)}inr>3?(f}{+g(;LFy71`#Qh!*_KU|4&)ermia~dU z=flRu#rAUguDZRY#q*tt`*nvzmE3%!+l~Dpl=Y1bad2~!KHV|z;k&auFqvYf!TjD` zgeCb0E(Mp)y*Dvz0SNgif`-H+0B&EEUk;fnQKNXmw-^s1O9Slr$87wmfvSi@02;{T z6^0T)SD{c?g(hq%UhqSKKpAGZq*D{#Cp5`CQpM$~6#K|~>Eb|seK4B#9vU;E7YBng zo4w|^)8XjZ1uY;(EFRgv?2+9xe8UwJFYU!)^Uo+Kni!Uo{#US@P2S$0Hboi{=iUhF zAm4d^zi41JqA-SV@X${9o~GMdnH3r5-x_m1Xih?lscO^jr+#(_g@BZ^4WU%%@L z2uGOvyK1RhZG}2mQx6BXOv^6WA$VJ+H>q_83950IC^ci8c-!)7LNf>l<^OOlC2tm` znZOEE#3SZyb|ADAwhV1r7cl0Q%YxoFr9>KiWV!V!VF^AvEZ^m)(AQEIpZ|q<5CNHm zpYH(tG!^8>WL9rId?Ch@WFttosrm~#o)ij3y(!^C0%BGd6}UJ9q>j;{0(FS7W8f~i z;0SH0PE*5Py4v*T0%sl_Nj2d;BeSt4dNkJWZVJ$MPxMz1Cm;q+v63sR*;MJqCbG;X zUd!XeST8*AN%eiXmx=*X#=5;4U0SCprEayFpPT%0&q*O&NV`!wLazaiv}g>rV-+4TOmcdQqherw$8_Nt-zUO_i92!GaK4K@i zwu0T2?g^Dq#L3S=G;~q;x0D2HNa!8Ja&h zdVzJ*7v27lkiom{`@;Qbiua?DLFkJgz353&xA@bHtA6le4x>}VX^6hE0l7g0{W|!k zoDu;FGGzw*ej8Z0?V&;4>aB-`;*@+T&KYr0$z>5!z0b0Bv zs`7g-;4-Mk?jq6xc8qdk?aF8{TI^@=rhZ*J)ca}CU?&Yd)Q97h;uidWnZ}%@F$2j# zA3*_kZX@|H)Z_R7<0sRM-(a|*To(Xg59BXZ_;i$k+P|@ZFZ^Ja@uP2L@(a zgw5gc7SeWlc`WBF3>>9%gnYy-DShX=tO_AVH8+X_%pyizroW5lXq<$spC8V~{G5xW@Ec*=bDT_OBOb~jT zWb-7>=IX^z;)B8?UE`XWOexP1*k=#eRu^@I&&?G>?a+q_jw~)X;$W6fW+i2l=0yuU z*&|?Pb^OVS0o1nFEE*u?0_HbZ0q9d0JJLJl5b!r5otxVl(afKZ0!Y0S4Y0#uX_x*L z7UW=J3hIJK1~R>Ov8Bj-3S=^7zC!OoV>!Mun+tR zo^)rSv_tX6DR|g7%eE)QD#sic&L0Y@TpbC2Mr~)J$w z2CTV0{;a`5;M*_%XJoEd+bOO)O*Jzo4$Nv}XbDMgXG=aUEI;!6$>e?@- zE?m7URrIz*|D0`CH07sG#}27+iT54X`@(@^0XjkX4DI7D2{W_u`04WV`lIulDZW~R z&eXSS@M1|yWAZA27)}OlJMReC4rRCD_^HEM;ip<1{uWR)V|9#1;4*5(xm8!ac=1+9 zHgLr%1}T~)lE5RDzlce1CC@|VO)+QALO3Bk_MJ_bCnD^bJAInZ6X4bFf z%tBq|TY?nyqmaNrVK*x98qmCUCe}>;S^`iWO3l6-O01lAjtGH{hfrKB7EPpze9IY9 z^*fR4zAiKnc17~c6<=m=1{wZ#$;nyEO?w>EP6m) zmcSpTfO591M?DrsaBmk({<(;yXHrAG#%4MTCWk)BaunVpGmgDqtVH8!OCRVkRpys) z>JNtaO{+3^dK|(rmOR4Nqzlg`sSsd7=?9;whM_Zb3`7air{i3;c#wy!dCil zZ?a@lDtiq3qZ7luGZGe|+Qbfq8 z8dlIROu{h`_-en!DF`+qPwZH?nvo-SCr`xQn9Y@QSXLCwJ;;^2lOu2C2^JWgq(_$6 zLdVlG@A;`cLx%{A{*)_!CUGH2nz=cUB}4>sjaYvc{&&}_yvt8huAg0)Kb7g1mx1)3 zo$Kh3Byd6T|JnX;_n!kd?jl(gC<-$VzYNNXR+z#lU(uu^Rwn|8((wCKoBQC({LRvF zG4i5weoH>Drcdo%>RG!mBrcX42xWJ>%Ns~EA6<{Nx;-zWC4oD7^ta(yD;8;AF@;~UVabajERxS7-aWdR7OePWpIu*wGcn-Sc1@&B-jGzXr8=q&O-DK zT>mf>54&UCmF?|H46(mt!fGP_ae4t#T!p~((D=Md10yR1ZMOvBTxV1V{?{+n1{V}g zexSVHb#pD!hVwKr zMuJP)X|IWQYneF`2yA-f>PPJQV=PGVqLU>w4`YDUz81K>3UEq>ZK;qFCT1c8JCFi@ zqoqhNtR~rm{5z-HJb>=p(0+LGPCHE?ZR=+VJ(S?igr{y zU}@XXfMSL@Cl!{-_VTN@3Qt(a`ih@E#Cp;Kr%iL5t|>Z;@ZTid(rzhAQyDte@R)!) z^p9C*St!ueB;70QpA=5;)F{no;U6sFd%Si*(vRrz(B$QPz1yB(=GV&c;q(JnbTktr zoNBdvx}Mry;if`8%2=0IYb}Cl%SI(BLV_HaX2R7fT>6=Rm)rq(Ii)bCM-sxvCVUKy z<%cYpahy0Q4cyV=YSO0j39D!JCxrSs)E~*d$?Vt4@R;V3a_7NMB2cz;iF37sR>{~X z`LdZ?fFFDw;Z;wEw%-8D2Rv^L4|Q zw0r5KVpjgvw{kGVL&vCH$fZ&mb`wECwLm$)-m@bLZp8vO|Ilp7bPte7esO;1hH1T8 zuOk;+liG^f{ZaYjfg#OD?{@D8j+&i~ery9}62Ks5q~Vlx0g8@WjrA`9SGTB|9S}*f zApyBGOC4*eH)hiTbyC5u9Fe7Z`T>Pt8M0!sZcy`}Rjb&gWfF7dXA`=62jeKuiOVGZ z4UI*Ev$L$e>IOuyGE`3?vj}BQUU^l@!gnx694`D#}Z!V5%0W`eIdg zENx+TSNPg6;V&tP> zNlkb_l6~{n*+3Eq#HZ! zM=-9A^3RMVZ=D5ioh9$&$L;Z-zl@vL!QY;Qk-8EK=zw)KI%bL4ef8-oX$pq0kwsJd z>B2N(1$0nSIXO??lvm&ybAH(g1Y}masq60oXQo=@jD*Quq7@txZ9$S|YG^QGgQGef zE@|mF&2!!f8B;M6V z4K|@H1HHZ~)NU)ILQ-bYsdUeuPJ69#{;~hmA*eXHz-G8=tiVTmKEMSiIU0Jx4xj)Ij~zNDh7o}^0JpiWo67wVzbOf~^_Xm9)0gy#>d z0R;Im9x_Xk)%^ut_IvRzJMUiL;odrsl#*$C!KW_=VT zPXQ_vg9)U;VHKC9gdkdEzjmvnR((#**x}=S_8+#8PiJEZd_2<)z-l_K<#00&8+ zq(o4G8+v%4h)xs~Ur*ejU>|o%c!2aBw9F`S4Jg=!K^RMz$_i-aelycHVJ2n>{MfAC zDWHU?rXXcLre%DF#t9qv)TUukSqiW?bNE!yICFg9fRTps5hH0a?w}%?xPBsTXKE8X z10)dBJpAQmrtdg#j`JMKC}*zxIVK-pxPHu&31VC&m|(B@)1g33L;rQ_ACIK3a8Gy1 zrDa@igdG&?^OXejvqO~jI|v`fDjqIE|EX8^{*W=WrwJs&CyEIumxEtl^RTzRzdxVn zziis@3SPZVJL$Jwn;}-MC?HQ+sd6%mFQ|c6c{B3dmWsL^lI5w_X+KSVw_z`4(|!Kt zD{R(@j*xws>J@~_T7@K`#Tx)A)6()+9JCgD7xuHmI_A#5z$=Om7N-sDqHN3vhy&Im zwen7>338y6k6yptm_L-?qtHcAQ%!>; zJ>9i-E5`}Pq3K~aZy8k(zZhIAFzAd|YMfHUn?zw%dgYiind#~#M#yTI{-#Nvpc=_M zeuN2C{dVQlq#;Jwcjs4hu-f`w2)wCaHVxW7sa_F}X2;sM*Nm3Ow)X$ZPX|Yuilx@FTV!3F9C{wAcLJ2aJt4OGaXj8WnXGuqm`|)Tn5#pDI zc1(YKWa6st+%-;hG#QBxgEO!3!Gb@{)3Q2Q0nV&#wEnxyqCD?6ve4x(r!h)$(n{&`l95JMSZ<% z&IU7SRb-TZK**ACVq&fkPmkj}!IZx&PzWmk38*<5ZfRBWW4W*3tk&jI7;zdAs_Xta z@MdAlW|ZJv!fL0D4cY`H@g`W53_lR0#-767$c4H(3F$PWg@(y84v)tV2U!K(MZQMz z%@FeYxbilXATMl?OC}2|Gt9jsRz&eJjRp7WfYthnTkqW8x%na#a zKRT7BHzRqnAUOhqeB;P6Q?L9=*Iz8FQj0ah!9|8E=Z9t_8SpL4#Ao(07zL_lCl=hS+gFjhwwScO1S05I}*Y<^m6B4{;%+~yVj(F&gYPoNQ%!hg|N z3Bix?p#av{KkM995H^#uMJA`Qjd}iTCAz^bP&iF}wY^Fo3@&^_vGT|+_qLQKoj6{M zlG>J8d;!}a!yst5vE9Qe@u~K5iFxSBMQP#wMNzZNK61RcBzZEYS{V(^=g%S8OlYP1 zX?BY124CRs-m*@8aWARrDjWXdB(9v}YTK$Wm)E1cN9kBhCx2MEFm#KVDDg}sLMa#y zKbN1|!_QZ=O(nu`OsDCl0Eq#`0GXL}r93?5ldVY1q!VEn4uY?z5@7vua>!l8$TbB8 zR?khLM0U4C2bk&AGg0bT{8oHKV)WlJjR!MpU|dZaIWEWVWy>@3=9qop{b~N~Q zbQ%8>TlBQ&6@6$ncSz=O;;hLCFFW=(GuJUWuhUoKH1ME2T{UK<(yu{&z*>&0L@_G? z6_f=vu$2Zl@m>wpevh6Y24Ep}YpO{E7%;o7A5sIt_k_$%?gaMbIGU&SgC}p_KHlqu z?_fM1jV#(9D^OzZ|AWO6V+C>!udoc>gtA>mxS^q-bwmljjktYW9trl5RZ%Tg6Vj7B z%@#nddZE?O621kgx_7-CMUAerz|o(D;BWJAA=G48+>kTN1ZkwTd)yg>(mK05g9*Wc z9e5xz%@__P;gGMIXmBvc(y=UWFJUvFQcj;Qd)#X0WLzEW&@Rl}2Dmbx4I4!%UJP+vw^v!fxS%>_zThP%$=(?*9=~lpta9{k!)XSuB(gg8ui$KPi7I zJ`hc+x4bFz)TpayBMM71r<^2_^H50iui@kX+c926?&MO0m}H*q7$_#TchNeJvHC4rdE zpsLl&Lp)ilB4&eEybE8l5;C%I4`@19E&Y{`dgsplR~!AP^$USEm^!bJLXo0s>CAhP z^kJA&h_-d~k-jReXe^B61{g?h@?IoL*q9ik9P+9J74p^=_Tq_nv_e~6ylAez(L6ol zTO+Pz7ZUfYO70}{Yg1#nbZAhgYuvi*mE(!L+*T{hmL298RVh|4c6_!Np%8hGoJ+)vWhChUF=DrX-QGPr8+Yf!H_B3wQ;OCFe zJ$BU8ma(h71R<@ynp2KsLD5`Vmr9fzPc{>>;lxSdS-W!Be}U5V*ie8#OhBOzk@{C8 zeo;qed4@c^Qp<3<;rEs0UMJ%rD^?BOxL5h?3x~+!~!(yLR1cO@OA*qn}uVgnEMFWxjL?o?X zkvMjMMyA(n;}4ctSIL34Y*nV9KshK~1FbRB$}=2UA!Z^B+Ash;o$KKGoON5*$Ae}J zc02i^9!skI2Kpg?Lrs(!WQH(yLc>2TmUa6$C(>JZqM=`CJHsr$sZ)=Cm4e~Q-9{5l zyAydwI4%cUE&bzDdGO=ZIf<9MY$QB@(Dt|U=xk2RYxAHLSIZCppF>c#5tI-`p=jTt z^67_l?#Z+pSD}LgC%G1>k_A*q9a%P~gu~@u#o)BKpT<8GSIJza><}s{Ot#eATcjcv z^dHG!6i>s68Cylik$94WF<{e}Dr{Q{PmUR2slA-*1d{f#PmXJtecWnr8j9mSXM$el z+wD67HB2@U;uDoF?_x$4Tz+TZ>x$YQZ9)`b6L0agVjCnN?f)umqwk?4EP*UT94s^E zpXOITH|yB>KqdK;YURvqDSeNPLS4#cUCZ#xT#g3SzXnSx&J$2n5YpA+a2U__0pXoSnzSMWkcRj)V_lZhT)-U7m_GVUPU3 z6NgzY$tDlEbWpSbjGI|BDPv5W%vzcJws>21q@oFa8O0}i4*p+5P-elt#j9--mPtan zWN(hLee|y`{1S)T2pNb3yAp`l7v7<7|6kPM=6&H>pK|@1i0gboSB;oxtT}~|K!2)w zY%`vSp3ax}f{72kWj#mca%Ua8$FPr3;lzL}sxSAJ)xNeh`9+!q4<_%u&Q~i> zA|8-VULU5W<%#~hNp?CFteDN2-eo53;s232lx~NqkF!S{PfodM1U!{8j@~g~iRrIW zg^4s+RWb1 z-n6(bCy8sREBPw_!yaz-J(WY%=Ce-0Zc|K)jEOALl#JHX*K&5#@7aX4 zg%RO~dJexZkLWBIw&E{`d$!S7ttw4I;h-*3q}oeI?MasUq6sXE1*OcYAz-x38_f0?vkqi+k=le&NJ4mub zR(ZDojf40(pd4gTngi(GIWd(ri7Gy{_CWBnBQ!MPYIqfaC{lDAnn@fbin~%5_0_ZU z*1RdGFFD`p`j?BSG{C6Ix;gJ(@?bxzvDTom$(jM_=>I4l-k(_9^p~2!agd=+GUB1D zrB~)788T*l5DDP$R-V>INkuV=YO!mq5q9Q?7iut|CdUP z$-K~WyLS8YyW{<51>JCnQ91_n_2a*Q;t#a=Gx2j9?td4xa)aa9t5{Ca@QsP6>;^6; zkgmv0pT?|R1>GnpGhA==rKDdHCf0OxG3K+4x_knIz23+sJT^COtw^PgvX~b}Hvrce zQEBDnG^T!yYLMj#wyrDIW{_wJYZ70&jH3U;Y810k!yHT!WTt5}IfWjGX&7R)hThMK z*~D+NhGsZGoc&3^)r4*iblUs4*Dyw2XZ7jL3dMimZ@R^~)S_EWom7BnZD6|@6TmiJgots#Rv0-g81Qt=uvy)zF!qz41 zyFq@dF%KnKZ->rWh}`+k?tI>jUT=C5?uJaBQkPXI$5=O)wT-dWZCBEidBo&wOOEe& zPul_&G%%5i0}zajCu4nX?+)q&gwG%VwEKCbefrySFdqZ)CowJR^SL>_$;3!FT~z;! zBJW5AaI9@rVNLb7V`z>!NW+N;7UIsIUFtbL%G_8Z1A-zA@)$$AN793nHNBd2#;FuH z8ZJ-&s18aq)mku<2fCgWol13=_boP187plbnxYD~dK5Jn5@kE`h zKKJ_WeUr3#6ItTs&rD_{7;K~rrkKMVcLt@Q0(J@K5h=T&0uBR(jNdWmzT#PD$bJ-~ zw5*e)tfRQBtFr=xah68y0MawCto)HgaP__1BVWl!YN=;R{2}5_D~dRj*_lkhi`M41 zX(&#PlMEDh_$k}|xAqMsH}d)BPXXx5On-*dpmHGa8%=~9x%}>bd8TS8M-dZx@Ps48 za<#QZ#HA{t?bF?5jTrX>v+wp*4VRLoxv&KOj>C*!lI?P{!Fp=|1^BH_{>bKnGQZLf zO;r?UtoaZ{Wo9SdT+wl|)2GAdgZ-!&hlHfu#r)dK!h6+6+^X|#km%SrWW~f<-Dlu7 z@#o&?OvN8-&Rpkq-z^^MuDix9VaMVF4cV3_3xCHL{Ezy4r1qsV-EbzOMc%>R@m~67nZD7Bo(XW*=5xA*b)IWsYm+iNYeHY^YvDQCC?A23%ga zK^{Rv%8xsuglyQ+2|ZOLTF|a5u!hbU$9{x?;8KI#-_FF1U<=E0C|zE%)unDt+nNg!x8mDVxRbcE+phhJUZP?E9Set zaMg6ES2FG|!%g5!>Q)QLsr9;SZfuLO|cEkGn91M_CYc zT>fDUXJcbII4He$^|to+e)&DG?jIC#E}8nqm`O5lkSw4M1L|yNjgcfxRD}|*}*p@2uxjp2- z4wQd-Q=>6uDZJ73YI1-9#i*s?kf$5-P>TS&0*zH&p=TrDGyl zoMg5}(h^;=&npWDqt5k@4VrsDW3)<>OHJvrzP}>A!AE*!fqJ%I#~&%*j2A9D_TJ8- zko8cD@|*-vUX0+ssx*;&n>b}`7g6i23sjXuwT3bzgApg)FP&f&LyFHno=gweUw^&w zeE~>JT=ywRw&imxZ}{7)`hNAVH)H27g zvEXI5?CA__jcL;kn|evA$6yLOc#3#}113!zyLFsLAL3}vbqi@=vwupj)YbC?4W8n} z3sWJ}K%03-<$6U#!N`SCEACB|TC9#piRc|o&b~2N6rXdxNUi_Af)t;oUSO@Ro^mqu z&GGuAyplXs?(BAVMk$p^XL?J#6YIR!|A_Tw6ZlO22W#|=^%-Z$2UF8{HnE9I(2dNb zIjzzDF&8*PP?OL!#w>tGHc1IHTb|V|{L}e`}M@|89ucb_`{LV}6T($zc;p zyVm)^7k@fV{S@J}y18zT;LUlQaQi`1}0eUH6-d^K;Af%=hYh)&)fN}cs~0QubYRPoBefNciY?D z(fQWZ`|W;PNONmXmpA((O8bMmHg0~N_tUdR6Au34t>G1=BH8qN`A>cUp5FYtLQ`I^ z7!Cez&C1>g)*tljKtAy@Ig1PiFpkflnNoYZYq>J^tfxYy@lgIAr|X^Yk$6@Oqs91= z6l}y~NcVZi$h1z7+mMFdKC`Mj+2h}Ec#|r)%*q#T>L_w*&`%>1Zxg$2-U|t@`Imzy zG~ve9lj)OG6qrx?3okRpDd%5;q%C}@IZz)_Bar!f0&NLs`EAM8ae_VORD7a9W%3pL z!oUMM;CkyGuV(8i&U`ed!^zvT0#xqWlB)PQZ&BRHh^@J2O`6&)^Mol_`;45-Hz@Bj zTw;j6hC5|Pt{r+*F1ZXE0L+CEEyqMOJ!K1ZavJ%IoWv7Jm6+lTlq2B|e&R|=)@r#` zWvrIMmYqN<*U(K+7SjS_eZVQG=~cR1Z@)K;LNy?#YWm$9#0v@FdD(_SmjM2BSv(Sbu9C>b^6$wd=5q+P$9 ze1iF59lzFhB%`>jU|Mqw{XKxT7lyDJK(6?#`>s=#26?INlezu!NW+A?A8`mw`m+?6 zvX4-4z?6+j@11o_eLX3AaOx#ssIz*+s+~pH`t}n=`<;1bVi3w;qv?H-V<5m}ER(kU zOd7HG`yas3g-o(d@Yaj!t@S_~t00j*J)xX<-~L;iZZKvk@?==JKbNVB%vMOsEB0e% zJnSKLIUdy_a2#0k5(nJVF_kH+dU~o^Lr_cy_8E9IH8v^^`*9s#w;>sWKZ%IJl?A$B zI9;0EWc;qR4#~rK8$`v#VJ?vIGfc+KB>H! zC#h;stFoI1_d^F=*F?k>8$4?u!}Us=!Pd9BWNQx?w@$pSjz5*BLXtT-@L@$;e6ZWF zYNCIWD|0@gwg@ym^Xjh?nwPV8^8OmopGgoyPG z78GH5j3bv?B-dtyQB(fnhzExXm>)f~$06ZUxh|N&Voa@}tqhrf-8TKZW66jq3+~|W z%L+Tyt5S=(8RtB1*Z8$YtCnjTixxz)Hk1det}};1)L@UiCe{~dVOcNNLC|WX&490b zkUqZLOPZa4v^lJpLsRm&BnJJsy7c%H4Nu+$XoX5>X~bnJB$l^kb%|D7kE?_Pi&~D| z+<1-ntWq;-Z8;s-iL(B*E`5DbIlP;^O|Bwqki5;sQ2!vzA-(T)6)>Pt7ktzJagA(L zQx&=scoZ@+A-LO6|%~3ZlOhaGv&zE)RK%1MFMO2!wJG zEes&mL_hw3)Q09#nhT*Y143n&BNjjE7avB<*;^ntZlLkh|t%=64eLJOJ4WkJJU02M_CJ*(<@uqR><8Bw@cvhNw|CLp}(8 z&LCQX72U~GzqTBK$J7qzzF9OPldn-P(vTjx(G=G*g^2o=o?TiO-CAB$$U<@IqsCF^ z5|+81gjAnkgE%BokI>X0B(=6_feDC+MjDQL1ji;le7(DQgSO7?9O5Aq4*MN<#}w&~e>P({UO4bwB}p zgl$NH$qu+`^(^ecS_1PAwYGq*r!Fo=$B5v^WlK)M7ELCKJ^ zT>_*v@8BppA2c8HA)qI=Zl;SJ1!(sp=Rwu7%_*NX>t-#YvJBvW*rnBPDkn8Es@DfL zaq1V4wl#QDqx=@PxJpJxAn5{BJ`xKt!r(Tdsn0d<6`5MP8hK4-WzFqJT;2fYDcm*g zDKHpvy_v?PxbQI(p2_&Gbv$L=Br@(pbw;s)kU4Jr_&P!XpLTfF?VFHNf(Cq@@$~!o zT8@M)CH5m{(WA%8X8IJW$+VwLQ*z3utcHJi1OC#qUhr{N072c(sTU z;L43PkgQm4iFNvess}cnwlUwf5Dz@Lw9{1Ne_#z$lL?dmYEp!`W@a3sKl07i8L$wG z;~W2c4#N&Wg6A8b&*%Xlf!lr#koX5cf(L-)eHoCv)#qnLbJa8~<_$n1*H-^8xjR5| z_hXKKH-y-51CS*AlB?^4R{%Hnt#BB$?7#GqZySfl_WCE& zE{uE^&kcm=y1e(vjkWHZ?Pz;nVT4Hpvb#|CqnqNyaV7NW9WzlBf@Z1g2G!kpgzRf6(Q~B*>mS+5Utkp=?EKqTWbdOaEJP%+f;1JPtMQr23o&&Y z>S!=^T2qh)@iXwI#aQj(HRnLWvv%onNf`SRcy!Q8!b|~fHB=@+4L6}F%_=eDQ8GaM zIch_|slcy!>+`~Y6RDVt5Q2$!SRg|X(mUbws3v4k<>Qnx3Sf=TJ2xOnKB^WnO^L|E>}4tYljwh7Z?sO>0sgYEa{Q=bY=b zyN)`}jI&Kua2l)vN`9h0M2WoIGyrHt{ySsVje%o~rs`bbQD+nOX@9roHJKjV3-rrK zT`#D0MMjM0O4K^8x#_k_-b*BuYlbq-29?=VvhD!h4un9;G?Z9y_9F+mJoj6qvSuIr zHwPT^y+WJN`rO`>40|%ru?Q;nk&P+eJixWFk03nfyw?M^|(dT?U_8a znTsb8ZxYorOMCg{x~u2ToV4<4O?$}$HaqF%N`b0|2erm17#`d-IjL@z5V2UeXRt}0 z8yOsC2+9(m*J~WpzF^QbXWob+ZvWP@h)^Ek}sC}(k`3kix{i%TYh3H*)4wem=cqteAwGShU;q=->51t%KYva7$&AaaHHKG>RM{Eh+n#YFZO6E=RQA1 z3m^bDv$d@5=mr;XIx2!GSYSiL$eR%=m$9PHdQ@q5@gu8^;=Z|U?$PIq3H1-d z#;t>WZga!+(ya@-m6gu#$a@2i2q}jiri*V|Ul=v#IYA+GI&z$uIDa%4n8PBd)`EXq z_8XB&kYHN#nZc<2()d}5lAXUl57zIWcZi~m9rWwUFnr3bB)WMauZ_b1NsWxcF3}&} z0Aw2s=Q_#Piwbj*wK`TpD$G#2J)ys12?bk`9+AT3NF{HX@e0o>D@JW@T@bN**;c1V z(XZcPM?chBKVY`mEfS38j^R-=0W*3zz1h zU<=`ZAZ^c%X5D77RG=(s`KZR|=XXq`gOsk<4b+;rTrJh>%XToBr@v=<9KTB*1HONE zf6nm#%{>kgP{w3HH#OG;x|ceBcL8L1trf6_RM6(s7_%&O?(##0X`Jz2ob+X9tg!(h zw*aVU;(YvUy}TkO@g+D>-6KZ)3q6k>=Z~zQ0+qH2YmHCh3OI?Tbr4uZj#eg!-+~>w z36uF0q}9o9Yhpy~V7!sFgZprj6G7SV4nNj*$uum)DF)zF~YmZ z_n_pv0@eCvfsW>C>J~b7PcM>EFWlZvWx8{O`|_5x)NYuS8{{yim8jKW%7h0Oiwn(x z!kOj6Ej*G*fpF2a5(|A_EGmVdGt2ww5kep|D8f39wcYS{QK^yuRM}tD={G|G-k`)a zIbFlV92#VHLW0CmG!iuT?*TJpc?F;gq@!WMY}xi2BtLu^q+haCSZg)Lnp*Wo);<`7 z;tjiM0dj$UvFz<8AQ|&``}poK6iLAhTjE#+P9%=d9}25!ht_~L+u|Rn-5oYY(yA9I z7X#pz-ktA4%|S`#vE(1x*}yIPU@;9qG6E5n95Z)VI=>&7(e7~lc^&OK9Mtbk03h(K zJX(#3)(&Mbmuc3WQajZrr9AR=>}KdIDvju)pJ7KdpH4MdfHdI|B8HM4E8z+02$#sw zCXEJ=+=tINr>?Rw#GQ&#N1@S_xaYOyr@vsnFfi^Wm#e#jf!siii-<^{7J-Ob93 zU=WfE7T*-M!$*t_>)ZtUi;|BYWsy5v3jQJE=^!aXIcUR|ayrQzgs-rK-&h~-K?V~& zn8(ghrDoTIi6x>Ig}poiMy=3(SsICrDz%$A$1JW_+u2p)l~3JT*m2#la#0?8A)I-P zF$*@+(Cpgw&uV-8E8lhhN2-f0&en<|WtQk9q-NotPK12&Im`~6SSjWhzQ&RQ6uA#G zt{{lErp6NaR`Sw-B^@gcl$yA@-}1_7sxhA5mP-p6oAd(bAhs47w%A`dY;&ympk|AQ z!^w+SH4OaTa^NvY^5X>7EW@#P3P*odiFz&tjb5P5QOZrrlaDV_n?nhfc*dlM8XWH$ z&cBmY@I(4PJzDLFdeYJXs(G5$Ml6jx^ZdU9>$7KrD~&3@x#UwTCenPd?a!p?H*=i5 z3Z#vRr9*}2F}?SbQfaC83i%1-A#yv#EyFBpmld5IckIA$%R6RA^GwYq^bMg+EE#_UL0YU6OChtuLvkU#@rGw}Uq zB<>s2=7(A`&DTfbS-A3r>BCix`Tfh&QW}m1hlI=YyXYf@Ap0CuRO)7nu@RGeCZaJh zQexq$24ll)?y`;m#5xWtRLB9Y2pT|63gev_p;b8m#0pGdnsUY`fl0&;j0!>2#9Hj( zg^AR{c+N(-HKpQlBCl@CawH0!);44rJ^^wiZj(+R<;`Ji*3OMf4_8oy5S?!csMazL zZ7tMwLJKQG2rJ*Qt|-@lQd?Uxu8s)OE4{Z=WSA#&z=HV&m)f%gaiE%0U_$WLP`5|l zR1nTb`&HDwrg$KLB;Z(5ItrU3`Q-cJ2>-MtQ~@$k{I!*$O;-I=p4UNIotjMf$m3pm zYLz42#A!^9gbAO%IZ2OMQzK0IHTE}%9+SmiF5nkN<)F+V?namm`GbyHEn1%Gn~JBumRwEjUEGkT_4`b0 z#U{?j2b&k9r-YSH0Iz^6?sPT~qG|l)$lux~QD zpA%6!Ay~CVoGy+t?cgEi`;y?7H2q25p7-OGRlJe%YFda&RT>V}?j^3VRgTJY;5M)4 zKVN%;AFnyy2?@-)l_A(%VB?Fu4v*C)t#7r>l0+SbmXiB+NzN~)MQr5|=TCoVLj1X; zb0V2G25}X7pf0AeCvQYYJ^|M*pt0K1^&jkH;X^`JO66}1?r*0A-R~}FqqWa^b9&F{ zaqwh3!ym3X=j$#6c0pNL5Ck1a3RL-Ih!1vQr1DT>webU`he7kx^Tp5tIIL3C$%!Mgf{ zF9e*JI};y%y@*qYppKK^-!w-R55~}dEFq=-iGL7EFeO288Z@~wuBUB1Th&oPkgNyO zcu#CF;d5~F^(ca6c#G? zdBMePEQ%d~>Al{;Q6J+-B%elM|JrLlH*`|49TZ=-T10JFDpxUdV14bK@hbIJ;Z5PK z_L=hO`gV7G>*t#%BcX$|`CkBzKykkVL+rmL7i|Nf?`^F}PB)6&Gj^-UhS6FL#O8V% z`1#Lv-<@u{J$QQWea^klYwvUEeOCVW>hw>iyIVEDSZdie?5bu<{vjemXmtkeCKcS? zTdA6@xPk&Efc?3qA}@xKhL_To5;KszMqc{n(%3wV14If!iE0k&knv8Kkt8251YdJ^K81^s5dB}vknk@I)2$D&Y!c|*Ofo(H;v-FCld z);!WA>3pl`M>-=^YgEWAs4&%ROe0w1*PKN|dY8RAztq^oq+}qCniK}6L+hC~V-w~@ zf8jC#-BhGh6%Qu}CdGo=SxfG|o93ENex`ACV=L7r#afwi*A-5%{ zsGbqpu+@$UqkomGUO~`I2umgj<9$-Owh+vAEOBDk#AAQ6(;hTc2NEX2{euJHt54$Z z_g;?Fb&P0yp!7U%D%WWQ(F45@)d;%htGg;?ctPT>@L82O+j)O!6*FmBh zy$%u;IY?Af`*)D24ieR)NmQy|k~$((Z56O4Ygy!q01zxB|Jl|EZJL@dz=$~<@0`i9 z?4ws=t$bye%2v%Xmlr!w#CxI$wp3r%vaJ#%ucLD`z|(@Lwz7uxf~@rOc#xa38>n*> zf7?UrgNWBk=5ul_ONVHn*T+~cVX%K~7p!0-xqHy!8ldZefbVs~`WTkRg|-RX_XY8m5hWfp;m^tawfqJb>w@kNp$D?hzQsB{}xd z9PB$>N6m}IkP?1_@2^;0ax}LuxRGJMet&H6=j!`eeLtJcW~TTdfJBcuWc(RN*iPA) zAo;Bs)=Z##u^LC+qhe#mH`gaRvr?Y}ed$V9>q&yBW`MnxY92r@;ELR1!TynYi*c;$ zn9tDcj#ofnA*{x}w{74t7ZhO2N`kXZ-!OhSk4?1=9x^mzN(3#KCRFyJ3LwTB0JJM< zimLlzV!*Za4&La^(%Xw6coEGDWZl@QA`7Zm-gW(Ci%L%Cufjdy+zcU_A8B=LC-^kKS1eZDit}WSuQ`ime2s*X-hkvFntB7_Mms2a zuHeDQn7&a=Ib>^lfH}x|$Or*AM>BRVw$InOp6vj0>|Ko_Q4-(0>X&&dg3{{V?BDK9WDjgA(M z*#d-F+J@)uSk)S(0nQc@d6en@cxYFYwz9A(fxSKvE2}o;m2{pfC9f! zY{V$D!<)W&G~VO`J!o?5?|q~>n|x?^mc3!D$q6eswpM{-EO7+~-a1Cv3Hc^AXzKlbXQ0D%#JE4OrL6+q+nmcAS^&!vDqCmH){IfRig z*!TAmVb7`Syngr-_8o3X>-){7+UmQ0_!IWM9X--^`mRsEe&69Z`c!?7*k%>WGFetv zwOkIuZ|n>YZ6)xKNei(@?hd@ULd(dgIR(r{dvbwMIqYG)L1wgAOVtGDV`ombAL|OE z5vtz`--S|L@B&*J+-smUpyWmemW8V54VKn|`;7M@(5B|h9aNd$ZX*G_u`AR{VNguu zVP8WHN1ki=<>rAUvyDiw688!rxq`cRllM6Ek_F|Hru^cQfDYEFf z3uyrJy7EQPu7;J-YV-_`>uP-=@eEemxWvc+Y&~K#Y{-|FuS2q8PrMb3`D>_0S6<% z3aI3-Raxf|bFjN3=1{ji> zW&-#QAF`Tj=6tb_V+OE?_HKENW0h#D>6iTt+mOZ-_W%Ql+mLqceUHr83$6O>D~GR2 zMPJnn0DNe1U9l`tT>J9v^?Cn6;7e;}V664;gyzPT6_VH3VlCw@Ik9Kb@C9lNM{H1%sujRs*z2cfd;5vfFROeGU)@>5g z=&>(i^^2(hx-iD01X|WiqH_W-7+FgI4AEM-)={JYa)X_WxI8fThE;mQbLjq|@Hq=n zye#MnDzGC8itmp`NN13Ay_;hyQdjI3(5w|Q7wdm9)g!?9eXhs49K{16n1+9>e0v1w z%WHbax|}m~Qfp1tn}a}`$XZVj3R*KkleDeL5?ZFDaP9H|41EAWbV`H-P=*wWTV6y4eAP{U|77Uzltn&tR*q&p`faR>^y#pl1Z0=E$N?3Txxzem1 zYD`^+YBlwvoqYH>Q{#xVy?V!{VK`3mp#2C)p7s=l749bQ&+vyKcFEl2n41hlBbQ#g9?#1tgMQ`x&e4iL`*P5`OMTyeD-fP z_-c4I}^&Fcs{hf&3??iBlqL@|Km*!-uU|WXVXH#H{#F43Is> zmo5nIk_zvV2=KB^D6V_ii$*IVJ>?3}=fSNo9UnM-e|&au zLVE%VMQm{{N;m1Svo}BPY3HiyU=2~D1PgNedqrN%zj$#5TL|l_+#$N58nc8!7@*b} z!HSM8y!g*E!kgU+J!ptvg4I0R8rqou!LQP@FOPOpK65AR3Og~|`Dx!HcR)Hwb$)*5 znJEFk7cw3o4~~j_Mhhtzx%&21IFtbZ@F42F3wbwwYzd2N>ws!*J49At7Qo5{17DS& z-2u_-RvWnb_SG5Tb2hhs!L4C44HUkBQLhll(iNHm76#q-HIB4afB1tQvVQK5vp*R5 z6#n|b7%6|)`WBC~K%>wHR~q;SttVD2o8$DUA53BwD9>{&eI1KsC7mq>5O*58P)@hj znu37T7i;YmvcZN!HY0!&)AMu!k#8SpSruA0kdhQqtWM|TdqK_xVIR1HeT#D%D2-_> z)6ffxHm19>og{?g`IV5(uPix7I%*$mB%ybb+uEg2VFMGibCBVypwfF3qXS6$_ne_5vc_ClG7Z|pk^(1 z^-2s5F4hL20;;|O!hxxc5*EQJS=92D36d49YFG_a*|*4crJK=YXVTX~!jI)mCC!0i z{^||~2de+=D?*nIgCci&$`@?Z@k-Dvj&%{f7l7WUrPebv=^5XQ;6)&sBC+86XlWk* z>emVE5=jkezQH-@YSqAAjc`k1%lcJlz4E51Uo?X^nt}`lGJ+pRnx$H-HD+69H-~D3 z|3ox1lkeTJ%9LWj!4ka02-Xwk5%dh>!R$cCVGgw5qZ%ffZ@aTSUZEXqsJwR0Cr!#` zdRu`c+@I)3i?E7NXvy*$a}H_NX6+C-4CjxEE{#bk9UP)rI3*3*w>~k$!%ib|cyXq= z$y=GnZ`~G!y+^wYYRvIfpiANn(N+m5U6I6oY40_?+h4}4+*Ddg^53vSGkD$X#kk^Q zuLPt5yb|#W{F_6l1@_V$2t~oEYtud1xvqVQ)>vQ@hQ_#NFfYlkdLoJuQw6QmS_>Ws zQh1`w7jv@+^V{FU3iWS~6;IdKuXZG&lfk`lngY*(my^e#HYHf6Y-z8qHWN&gi91SJ z__gnA05=K3s-C&x%T9R^>^`AID_vnSsu=W|_urV%DE7%hnUHg_!6|(0k%_0*CA7$m z4vE1b{d`n62JObWCUDp9ia-;h{ZzJK*;<*(eL-Y!^IBfe((J=pu$S!%C0uzfa0_Gs3; z&Sw>csH{!J)S2JmgobJG2j_$`6B3>N{bXl(w;Q^GfOW)I?6YKmOS%?CO<$cg{07yh8|mphiXNJa8|r=b&tA3K4s}$ zsd;KpH24x@M^MQ!byW>vq@0q?+E(T1hesKpeSOd^Xm)!~>s&!R8EyE2>j9*+?sx6P zleGZ$&fmMIDOLf}WhEP3wd-oXilv?wDcCr!&DIo@XiwIg$-2GpXa~W=0@Z1icBF(| z|I(|_+)KhsEk#q&ken~g8KInCZqmx{%$Qb4zJnTf-n^o13W6k327WKQP-#SFkxN_X z^=wQ0Om0eEde5|EtcF=lLc6X2?ZCLPNE%?!Tl-D|5+GhDc`d8TSZK@B z?Qz?h!vJe#aXb?j%+y{5>@bXL#Pegjn`u&_7kPH_XV%bD@{QVxzI5)bG=(sx2HA3% zsOKDXkAv<(4!TGGI_Mtcm#cdO>ceeceZ9jkrsP7@{0N*_#joCQN6fxZoe4^+Ad|lM zP*I^J(KnPAW3_bqmu2RivC7os^*teztf9k2@etmiW+p(RSyQ18&O0tg!$17X#4EK8 za!Az}zYLZi#pW_v(M-$D2w9#)?}pE+hBmD|B5kBNxK19?wC+~?u}{fRy}#{i3SjQe zv*9ql{*@ZkH~dsGcO{=^5LW-%D1jqmR`i+`bNb;lIXS zNUDGMklip=4>7gPk*bNq^$`;rO|PODD_YQ96RIjEw)olej0rcZ(P}K)SC2eh@e3?w z0$@*Ak68&AQP*@BsNL5tH^|^#iX|xsVb9DJ!bowMqqeFs>^I&>D6W?G)i_V(?qJzY+rjE z&}r3awy)G`HIoLvRoP~wdd;LJBb9C@RkPW?_L}T?wVUlLx%$ndiRo9$8Lfshsm+AS zIpY{l<4q=1(s`V2?o`&9)NHs4Pim!|omQvtd^5g%=yFPoo4wMlZFq$$_w3#qDG#%K zP1MY$jyGEQXZw1HGB4XU&cV99;RRD0)j%LZN|PnRnL$E?kPYD?E7}|( z0-$lk<$C;y+=CRm2Q3ZSKp^M5N;6^%r9eP*it~bJDG3o1MxC_V;|KQ)R%7(%M5w&c z8fsQ>dV3|^ z*rduS~xHB6sJ|fUI}~n`|E1m`m|SMv|A*bDf#~5`X;!j#tn%8GZ_EZDa_r%>Z#{q ziIEaPWHXye%S4WqNRUx;Ek&tz0S+1Kfgp{XlUJeY83be1Ozz}dtAVd*$%Zyj!-m;DIY`@0f3^{R;*V~LaS+A%*wxE@aHf$wp z{pX3wWM$S8?5(G6Ng^%3r5jXEouO7?92CIjz-is(r780Uz;}x`J&_i0?H+{;;t^r} zbppZI8qHe4e{5NYFzj%|V*|eQl#G6wv^(R^2rZhmY*%ZHkWYW~M;3&!&==2HS;+>}3m{qF z5s5-2$I=t8EUIyAC)mS$Ds^$dv0ZXjtAB$PO(C?%hEJfzaNc>J3IQ_Nm$YpTbGD{6 z%^Fs7)$lA(Dwcpp%LK9+>Oh48(4X`ouVww1=7R1k{06kYz}zNE3aJU~Thq)R-0s>D;INlZzRn2`K1!C0D>^e{2uVM?(7gubSw zFicEec#^)RxkwX|5fTy+k{vz!I(a4}K0ig$^OGbzPfT`B>1TSP^H>^llKo&O$vK$9 z9GlyL1m|e_a#RZ&N^VY0U5>PVvvmwUUfS}+lx0Xrj!nf=q$^KMRSwsovR87Naz6<< zwk2XI%DV~3F_A8qo;;X*9BDC>njA?yjs3<-q!T2C0AEe}il>9FyB`2onK2dt^6Q$-(Ov_DWJ;^D#UBYdO zq?MMA+fTMl5(+G_!ZT!qY~o3-saXrjX-renR=sr8gqBG(WpyVNHL2ByrK?U&Rh^Kg znx3LMZ5oL!6gDY6b#hMA)U?#pl+<3DY4U(a(@}R*Oq1JuoK)1w6FV&hH97rsVvaV6 zrJi~@SLtLqo_4zZuH-gHQcj1{NRwL~N;e%$Ax&=aL8+z_(@ghBF`brfIVH6;HLY|~ zO6eXcb}3_EoeY}LXe3>9FcCDN+2K^t z2Z>XUEluxl?VOhMIXU68m(H1HO#H;jo{5_;HOX_21kd>4HYvH&rfx=~GaiuGIcd@+ zl37|ZXF|l^gA+L)ElL^;*2!_wh$Xq@SIFAB1}#b~WnH3%A2o!_C84C-7?$1)`MkHa z=Kl2}rfJr+v@p$LNOZcNrXwR)#PDA!aGDl{go6I>(;R2}NJUAe+-YXqNhaLS)_nUQ z(`|2a>@K5ax3>0uqEFCd5}oh1*VHt-HSEhe^W1A=L@TkP>zrJ={KA6PE9)KfcCl?c z<-l|4Bh30!johd|an)s&K%`fsMLSVHwz8pT##RGeMDHh-QQ4W7DD6Xa)j6EPwZAy6 zGIsFEZQ6hk)|?Hsjn$UT8^BBoLyFl|X_2*Pn!#S}&aLOeg?Wv46MV@mKO3P>5XsFw zCgd^d3&-%kp%sUUkX0>LS_^@RH&K*GS&??_X==xaIuR|1D{YCe>6U7quJxbA+OMk4p!Zue))fQ@|_)iS%705EWzInmG z`dB=E54S?qZp%hCSs&Fq!P8&AUFMMnxp&r>ytr@-U z(%zHPZ=yFtsX(c2UbHl58nfk`wa{TICxCqr+~=Rg{FO{8x|%*|<~ci4>$ap~Rx_Fd z7=`&YY2LWV0ii&wwsw+U7jG_8rfXtJ7OFxZN+To3$Iu9k5Cdnun)}hzcf6>+zi!S* zNwYN<<{VKC6*z^)b%pi8c|45WOaO>r&_PJ=w={XIcBV5G@RS1>kgOX;j(QCF$Uven zYHSVN>5$a$_z7aqh(}-Yn0;!`X5r?kbQC^*ZZ=io_R(~RrQGkVl{DP+JhANwQ;_Nu zo27-cqQIJD>`C@C#Ed=FnyM+rTeyGBc8NcZt;C~PoNK!kqXfOhy}HK`k4V{*2_x$H zPpnfLV;k_j-&&`7lf}lyok1lcFi1G9G_D$FvaAY*(Q&=@jbMlwjD|xlHs-dN#@S`- z%bI9iqF_yu`Wb21DyNU9em>L?AG{&(V|#)=%|R^ zYx)Ko!!z~9wFhg|G&EadJimDpq-hQB>xLro9PO&vf>2lUs$k>~&h>l7mP?j3e(|9GSOUw&{TCO&nXKTCgJ?8w!L$JpbLe>SVd#0y}5qD%rhtv{CE! z7ddney0z~l$Aa5>GyQd9Fk8%xX`+CK!q&VPw2L2GTFl97FQOU08g17+yaVce&x<^x zHCliTxTumQ&qs5g(H1i*9P8;QK^X@mts9=T1+6Wp+h*@|=*aY%X)%8Eknc0k?WeXS zl#10{V!BXBo7~zcCx~!hX;v;R-*KDT3keT2>H`8`+HZd9adNj~=j6{$f;c05T)y+o z{ics*+n6Qln7~rj?2gssB$p(FHOlUI)|}4C|Hob6EK8Pi+OT4CYGWHPj$w>Itme-}^Sj>$i6~?QhAt;j&bOv`vNvO>Pi&EbIZA^( zb0B^accK@Y=z&$nSfkoR>Y zNuoEPuly&R^RAI(3Ax**_t4WhxsrK+IGr$v%xEi!U!OObY(<03=d59M$wk-(DB5zN zb&4fU6^0S=nq;!BnW`j4^tg4-Wl8y{VwGYNo|1|2G&vZLk$sV1uYa11i^;hbHjg6O z?zAk6bnAaoX2sLwR7}aHm|!nT&!Bj!+=&TU6X^w~_Rg1>QaU0rN8%~6BhvFC(yJGx zWJGkC4hbzLePW)%^w5jM zd47tlgr~_zn4XCci_e~%gW%5DV6%^reULz53AaqoHRzV-uWXa1qrdGvVr7@cNRZuu zjU@x%tPbkwAa+8cFlxXk)obQtHB}qGh8Tjd;9+B)6|f5_Z`;Z)sVnQjMJ7ulavP|A zO%*br``@z74D~*|zgtxVNbG3^-N1vAH)2Z3Wry}VWhzFSwIEggDCY#$3Ph+}isJ^~ zX*a38b+pZHqFhgm{bNqfiD(Omr9;042;^wS&^sf{2?<^4{nhj*aBQbU4xI!R7-!MF zJ*3ByvWY%i-?ICp-E;gFZ{EUS$l0o9kS>8M50p>_&JidO!0wplJvmoc8b&V%*ua0d zUo-KIP_*G{$?2kC|MzS8PBzw}{lmC8$QvtqTlAAkYmrdbyu$``(14Iu1S;gX0c6=% zaxK+0v|fQj%&3-@1Fs6XpoN8{80VTdJNLStn?F0>`1cF* z?{CP@3>b{{ zkiYov_F3I5Z-^A8g%>9AW{|J;X20(%7=WweLm^y{2LVuN>zGmKB_f%{|>j8G=1hSNzVQe^oWhSnl$?a36D9 zQ3hW&&cS#yCzs9fu>zB*$(uunbn7EArdGY+K-GiH&9Ds$vB=>`Zr(fZIv$F z4o+hEN8OJ1e4uoIOIe08I=5hqr|faqsZ3dRNXkw$d$~UWtO06 zcD%`&(MFN)kNW)L`y=o*I8obxRd&enYu@Z!uy+*%XS=nO4AxoJ+m7r}g^;9)*+Xn%3$%|fWB=KlXkI^(;u zJbd82G>3BHA&vhA<*|-T2dVQvPP);ag3sDAiE%Z-F%K==c|Oaz6HpXyT3{e&P;x|{ zih6Q_`)Od(GF$tpJ50~C+(28!Ea2HV(^Gn3t#b66li!dTd48&?T;Mi>Mf61zz}pQM zUvAb~DbOpTy7DSp!UK?`ivrVfIgQ{l^iq@bTy$%4DXwJRcfj>PnnT;O18*iL_Cps) zcRD|M(lc4eoSiSBkk{r)7CcLfwK;mX^PG1^ujM_tXE^OZdZXjN0&l3_iD&u`M_X8j z%^r0|YPCmBOe9yZ%wJcS8bfRLlFZ0oq>%MXlzIguPP?OQv>QRAi4QNZbucR}o?e2& z|8n2fow=YWOn>7q{eJvRyj=PRt$*CW8N<=E*AJV~zrwTy1ZqtIUQ{IpRn7`_#|pO- zJnGg>Y7)BC<4{}*q@jU*G;Xgktu70?8r>vO(}teX8}PeNrGEdZ5beV&#u)&{5e4ad zke)+SijArYyZOq%wARl&RPD1?^%kjpbamsla;W@`-h#TWc>e8=eZ zJX*BbEo-btquxMapSB?jT2PVc9kLzq5dh|dmt%|2D@4a3)Y``2m0sK+RHG3Y3#oF{ z8|ZnVk`_pPIz24L^EIqICS`3l^-TcmZ= z#}8Rc)d*U$qj41xW>r%jZb55jG`P0j?1&tyE_p8u=?F*e$$pQt0C{wVZ~e&t2zYM| zh@ZXQTsA|tntk{4jzucXe7>uw^^Yuwd!lBajsqiThR>G(Ff5r8$H&fIW$J5eMYCxE zsHX{zwtQj*Ued6lFhCtv5eJ2{k)6iPq2<0UZhSok8uS}8-ZV60V|gt+$U^A;D}JX> zz$B9%nm^a9s7TH7Hh?$+orGil+}HmhM%;2na@1SMBgB6m-{ig1nlFLBf%wapgx#@v zLn^5hunu7HR4Kj^WysIGv@uekGs5VsdUa&iKhCX7vT%Aod4#uv*eTCJp1C=ixWNO zFHZI7JzM-2vlpi%kf_yb(Tt3#o;Chbm|8gZvd&H1uUW*=W}%3xEs()Wd)~tK^LFgL z1a<>X1m2Kem9l2zCA7MBMy=&z#_k9U4#m*StO0QFwZhASk|(f6Y|!WImeDlOwBb< z-?EW}kO--b7fc$ce{VW$Fa(>)h;8zY)pwlTKf9N;p5mG2o-C28zba&xiCUkg<){;5E9omT1~ueXrt-Jcp>n$5ck9aev@Y|{k;n|-HnceEKC+a~%PxeMC$oY5IySR2`<8vEqzYR&?A+`NyXmkt?-j%0F7$9A^M6e4Ny$uDs+$Pt zMSr^Serrftb}+tv8DH#dpiMK=8`+Y3iZ(l_xicp>yn0D4glf?N+ibli+@|T7Dd683 zreW0#7=l>YggtVB8cHOd_=tUIetUFAjy^1v{zo)Rbu=fJrK_jI#gf%EQpM_R=z zm@PML7|jL)Ur&QIYg4?OB+1iB5c?_rS`@*&s_i=O|42Jl$q^0j5~9^_Nb)opO1Ty@pT>SakbnEmJOAUZcc+%hd`PF{i&^)mP?WZ9e4M(aHGG6lG!NZ00#;g zecXYl#g_$S-ohiEkn#X|A_KO+ij^Jc)0+2?W!ZnxJ$!}V1Kjyb@=gjC&fa^^XioIb zJ;RUR)x2RrZj^KcX%df>1n3ROK41sqtMj>DDSd-Ei^7LRAr}_3`7uWE&HKwqgZSqC zr5%I*D3?3khc1R0<>PdJqM+Kzsc*Jq)b1!ZCZAyM>5hz>C|w3yz4$Qy7;@!9=R8)6 zyuXX}-C!cj8rF+Vyk1kyNJ-fi4X*+&BVtKj6BvZJI>d6CK|((vuNiu|@|QN*E=qww zu7i!eg^H88;jRP=Q0747@fJ_e=OWx=^p5g^E_lJ4jg}8i=e??(3Z~(~+-S1M<0Kv* zaA!#i#mI^N?b*F%lc!i-BP#c-S-m4L*#}33q6f+^@w3=++~5u0yuWyfb2>Fj*)^lvQ7t#Ikr#e)qQW*g1q&< zr6iL@!Lq1TKrV42T$2N;n{oZC`R(sPc{6I-GhBdY?=Q|@zrC23`M<5@{j8C*R&|Qxh!q8olPMRoK&VF@K^PU7#3lpij9QKe68ygA3o% zD$SB)@7|IH+}DT^gCN?KtN_$(bP6yPG8A5wvUd#+pXQ|X9_q8KQh&4Hjbk`rcOYvJ zNXuXsOF=RNW&nq}mnuK+g0dqvsjRVChYpYH`Sa(`5i94v{kQ){WKAf^S;oul8-opF z3;*%r#hm=l^S56?_+!;5vXIT%yj9=OYdC<2qObqw`s;Tc>OlWV&>G!z_9`TVyZO7{ z{&r4Y^BOW?xXBN;5t+1En~Jgxp@6^wVp3SQU1MTS0|+gU6j1Wjx-prWntBVKH5d|% ztW!qPn2C-0qo&k}5d&esXQbe_j4aicwQQ>ngc8GHgKZkzC+tK=i;6WLF%BtepX}`= zxO+ZxPGd4)Ms^vd{1j3GFVXPZgSy4vl zwi%|LtH9>}Y9Sy8*rXBEL^ZTFdtQJ4n6U6~X{8W^SqrfMb1osHx>;LhMw9CD zek3?{(?dg>kBSUlMvmoyjvbfgRQ`I;QyH&;v)R;oDVTp4`v-19vyXZtg>`cs=sDjMtv0GhV4@G+t9msmDBgc=ZcK z8aE#f->_6a!D>02{j}3f3HaX~6Pc5@hAHFbq6;O9w!y|zh~aIYr*~4H#S8~#?QR1u zp3rQUY>iAg#`4YZQW@RjLD%cCCb)(l1-3qB^7wqs9glA6NLuM|X6L}1*#~UWLod;@ z6K65kGdN%n^->Z+{o#ZyeNO^CpDCVZNl=`eymAS5Hh}0-G^}3IjGg)x23DTnutcl<%S4w6)2->eNZ`n}A0_mt% z#ml)NQ&R&alt2nP-;?HA$edlZi-N25^<<;}(8CyfL`xh5_5ZVo((R4o?>? zJ8><-OIU{3{A_O+*_Cg$KfUYv|7|?Q6N|;>>>E+}&yb83g+-chRU|Fj56kGfNl;;J z;UB;im3BrTbOWUdSAJ-+6GeIGzf#x+poXq}k6W$qq|%z14GR`jm^viXWlkTuu!{=yUQ3m zp#bCo{PH;5XfCrBP;-DE>ox1*72;Lf5=b{8K?3@@uUMPQdvQnOfiQc;PBe@j(tim*pPv(k(!n5c(Q^dG3ezAneT{-oML-8)zDHFZ1vz& zJ2=&T8BVoJbgKEKaC`j5SY0Dlx~YDP*Bz%r@pe1y%ag=ol8iK)wp&F{wzrs;VMu#2 z$pZpyMHHBUF&}p+B#6F<&Oy%uS;>f54=)gQpx6Kcou9cPbi!ssfn-_HjOpE$VrsVs zYl3+>eWD1I(6!=e%H7)tO!%Y&q082+r10^Lp0}%*1+%-jB3hVh7=oFwx>Q6i&w6or zM|UqGy1Tt56(uzUbCqN}C)-nS)iafAV(BpW5>ny2t8>7@w#J!#g5jnas0oNy%j8u| zR7CLFPaVN)A2(t|{_tgtr0w0g56_$>w(madi54llFfM#-3m7S_CW#<@F(aYb6!)R$ z987T@CAbeg$s)-)o8mt7O}Sf6?wk@6fz5p2S^OV!|Z&_HCGF1cxx< zzLD;&GtqbwQ}$}za~~cb&*#S&_|Yj0ZL^&Ho-mQgV9|!|x`95{ssXX8Ge!B%)6({L zl3Nn{7bk%U4Bz4uz1dSJ>O>)jcw`3bKoXN>BHMx@1uwZNMXwnTQ;i}g_~ksS+7zB) z?k-s=>&=;KMnB$b*W58)y|KHXvO`c}=Pm>DsfQTtz}Aw& zLWt+eglCz!BX_h`@ez~hVph)YxROcaqEE!l*9pW*5V3j!lj}c|-1$MgUJFQTuvL*W z$EtqyF9WjK?Tsy&_eB^Q$6gg}8di&!>;N zbo#UX)S7Ht}g!0Pk;WKXv*tfdv5UVaHgbCUr+-iCUQgoVqM6Bh zhEx9%u<8;Tu@khF@@3I z-A`O-SXs%M)*F%=)7WML{)GpC@nFzD=wHt3hh#C9Eaz=G0Z0Sx_h;lDKxz&6t&OR` z+rmy3Z6k_6VOy)nH|~Oc!ls=rHNb$b5F2UvtwKb zvAE0vY1CGJ?(6R{nc&*ZWM>M3j8dy!suq+DGXj)xc8m7sT?Bu&Sn(6NZ*YX zz}hWBD@-8T&h{DJJ*{H(Y}N97+P?WLg`7P_oA{|_L*3F!J-XZb=b8rz?M1_l4rj3; z)CO=lwr@SeH_l39&bw@r8ORo%1QUaQ}EV`7X~g{y9bIwOly8yh&@ zFw~~2ORq=+n2&0vCl~;1YFQE`XrV0Iiq8V=jrNU~OL46r;)wb`!fip9y;`*D88^@gO26UXzZ|axf8-jx`q*(btTip)T=YW7hdUrf;X@MsIc7C+USJ#)=EMKjg**yc7 zE*|C`ud$uv+mWf`yG+3e(-;FEn&F}N9Nhq5o(Y8P#g;@gYD1Aj6ad3Kca_Ps6baUF zy|gM+9C3GyP{yr82zOuOS%?)rwE{4vu%`&_cOCuQu(B#>lODuz^qpre_uQ^}Pg9Z4 zXramO>f2Yb5COMSdofryUG+g*VOL9NllCY<0$}BW<+)d425czz0;`tlKd!!gbw>D{ z&Fx>oCT%nJ7--}QgX>+c2M|0~u->UgC4kk@{hAWcx4X$_=7%iYJEdO zZ4mQ>AJ7eqU;p*PmCUazmd#BWo*z&nCQLMS1X}A#=2*oGw+i&RX8*Kd6Q-D=;er`$ zn21Iy+ca1q8&e<*N;krsoPe3o7C~&J1io_`Eat7Rb3xb#t^fiB=N6WRVwAZPQ1mM` zB}9u86C(z|cz-2i^D9UCHhK_iQxke8c}`j=;h|Z8084#j3d|}mHl*aLp|@;KuC>77 z$1LDV3IgE#EGMmKcsT4xKk42N)8Gp!mrJ7BOdpC>&1O~{iV^=1Lp5kYl{De&`b}$! zAzf7-9Xu9S6=q9r2XVvLrN}x_2o^SiWb+Orpkiafh0si396Ew!OetD#u$H2!yoZBX zw&5uSk*}npg~n@!)kxb6R*I5EEpM4{ISKkSxCse?#D)Xl%Gl6Vh}n>)7+{yfUD6zA zv zf~f7XT9!dwi=nsKkCZjgzFW9@PfV4zqpOmr)du=`UU zw@7QAy<&~(qB8M3(~QwbFUDj8haD_Kv{h=X6vK)EE~1j=T+*j+Z17Wjb65!Ot<2LL zd0RExx`(^03(V?KVAsMMlKc&&saxOSC?LmN)4Tmm%$@W)b0ub=OEY-g>_v(%Bq5a0 zUK3qxDT%7~(md)q*&nn8`Qx57tlk7{xR9WPx>WXJT(h8;4yRerFRS7#CRDMWK0_-i~2JaX;KWD zb&Z$;efD2+esyVB^@)%Im|s$>*xo6|G!KjOcnj9$PLw75^lA@*$O;5pbjfOXxqq=Har9C6kmFt zpVF01unzvTO$DL@2sdV0LB*~ilZZ6;lJHVX<`jyUE|W6)?PYyq9u@4&idJ>LTc=?{ z_2>!hXjsTzchh;itSiQrgHIFG`U0=FT+v-m@ZXMYI=HK}MzKcJyQ- zb@7$J{B5qR4a%fXoaD8vDq~kJPtW=zOH|vKV)Y$U5bj~qI`q=nGW<}d4e=L~Pnqqm z_89IQ0dr2tH!4-Y@mROZX?-*$3yt4K&pK#62hC@2cn8gA=sIXV{p%O5`M6Ai=~_>h zpyLIR`4NrohGfIvkoo#@N#3(e-mw}P|HX%j3N8D-p}a_C`{s!7%xh5Vr|;*PXw7xR zYsjTE6JWqobekr?rmEQ;mu=cSm^je3L2jxVBUi5Z%9X~mAz$Fatn8V9z&-5a)Uy2d=T!UQu*CS-}kR9Ih6{ZB@ zmkT2s+R$toE-1Wx$t7rpZNt2rL&Fm$Z;2jk)ibP$J#)>@y6Ea)`AYwqcpMOy%Xruy z|GN9*z0H%|+97x{r!THc%-G!_J(NNFz9|B^EP2x~8!qN{2Kf2%M1UgyGCTpyau^@R zKDc6Kp79T~ihCZ2W&w&Gb>ZE<8VMn>WrbMMu(xL>iNTtqIzExy>@R|!Ou8MDB-65X zw`z;yg9f{ZE?7Z+Y+1b_TJW>svuSc96#G&J9`>)P#EDl@fe-&0EB1kF18Dter4;-NL1@jA zK#rZBHkm$nvX7|(FS}+Bmg%;+>YZ%h`H!+lIQ$1Jk+F|+uw7#~Jmexz>_^uZ7uMqz zpDS`Hh^)=*?y&_-C$W3hR@%8{Cbl@xKeOPn?ycaXb=7&0A~Dgs7q$adSa3YL_*p1- zL>snBuvPW1lGQ7~&&iSsgml-$GYYRg>3Uk6qP?crAMuKp2LQyrNTsQAv0{XFi=V)X`ZgVb~=m(lVY@E zWln1HftF~M# z$vp%AsFp?`=18R=A$y~T2-HlLRV_=dY|5>ni3bBK6>Sjey1&9jEBXaU-@GCXtyiqk z?GVwH3l_wzd0g!H$t78iT}_tWDE2R?LdbjDB19cRjO0?*Ci2w%XbJ-8mH*b9jjd=@ zFf=QzSPM@7ci+8B2hl(7o*7%sds!FxJDEMXx31q%aea<$51<=3d}W})|Wnucj}LM9cijI~%%r5w@#sW{g-c%$_x;sYNQVFEF4A0q$1caT`ld>Q;#HMN1ORXhRiiV&=HMeAYdq zxre_;%uiUGGAA^`G7Pj3nf=j-d7{_KQr2!_ER1xR|JtA+c((EAc7T8ng!*1(taDD}T-`ndvVDKkwFfkivy-ypqhWJ~T1 zew@)$Zg};QTnN?HtOI-AdQFUm6SWp{flz|#rPWV@4No*-Zks8CZ9g_O1Yee9FWt6@ z#Wrf}s;-S=exwLV{X9iu%p_tp7!b-M-P0**1Pl0b7>@Inh zv%6<1r!UT+3n@uN{c%!7j$Rzi$u%!|LF)pu-yH;iy)SGMm$mB%xZK23L-k&$6|Zr(aqmgK1p;r#{0b^)i*C)z3!Y<5w+yYR~K{gjZtOHv5Z_z zk6+U?-0?9dHfifVz(SCs&xg-|EnR=t=kO^eENoARYbHohd&^Y3dC#iRAc-wW3UA)B z|MEtEAr*%bHMAiVnMu|bG~_oz>+hVrzPNgS@#_5M;`K}NrFjGQh37fBk)63Xpzydv z1%nss`gVp)m{8Hx#{ghSD#)eajjLW3mi@9q3KNR0TQ%0|6>rW^(!-bNpL=f!G|I>o z7VB|cA2l>2X;ZeeO}dicY75!iaCAO(-{X-KH53c!>@-BySPQ|F5S-7C=MYJHJU_OA zSi#L<8En*6;B=kw;ko;SHz(w4n{yME*!a><)ad~qb)`uL-Voz0SP*65ch+EiulOAk zIFS$MpSC;6OYCh zoVy~(^Kx{{Zt+)p`<904zBZY1;m`$8Yk*%(st9Hf)Rlz*HAmVr*QsakFV0`Ty_lEz zLs%3Oc_q0BXO3>wN3)Mu+Oc1FKs3)5i1Nq4Z{>Vb%_H4xb8w-XT3(7w}1SVw4SPjua|p-(OpMcb^GXHJdqbVW`I}>rOBj_uY!l)MG7k z){ciy7n`k6K2H`KZ5B-ETp2X0qS$nG9*lK_t`PXQHlC8I=6AectB@gZ&Be+C_wXS@ zOc9#CA^)xZtIN(EZ5Z~)yRUDcE))2RAFSx;jsy#C&So<(DV_XN%glLkO1@*}{SpdI z)lda}EnO8H7+ZWrVXGpXzj^wZZb8eRcs%yn;S2zYJMZy>sDldfm|tY(^WXHdMNRvh$>ugMo#hj5mUCJ<&lz?b>+tm-8z^TRnS?slxF<$)FqK zHD+c_9-(>e0pqhRJjLi9H+;V5$BJ0qX|snPE8M2(*A_~Huo z^9GmWE0+Tk>e3@S0fDO1En4P&w?2OVui!WG6;}|N6Dv0eH*;mG{($6mrIk)z{e$Lt z{SwdMrIDoSp!`N0m^Xw`S}*oVv#AVR_|2;;f+co;|GVcP$^GVw-#rg5{>RNbnQTe= zsNTf~i!;+>i+eG|dQ2c?U1NBy=A~wn1Mx)|Lf)Mk>2lg3Hue`D8*9_skTE3_p8pV} zd+#zi%=Ey|;jJ}^zj-_?*y`i(w?|R*`Hy2=o$14F^f!N!_cauRHeP?j`JpPd2J>5! z7DvhJch~>;>io|aU%|X#bW6Y@*{r$PJMJ-k*N*M4=(34ar=8ex*9r3O&2`UHx8so; zx1el$>K>{(iRI`FJdwTAgT?GAsd`eNBt$)bO50puz~mFTqx8seHe#Nm>5izlVDaGK zx}+J63oiAP49sJ^-^wg{{~+_JHT&0$pDC4s7_&`vpOms5WbmDtaqP?HtDzz?X&i3w zQNOx=kN*X>l~=WUto)XZqoF1kR#h|$(D&nE$#-AjHvqr#Am#}zOp6Y6k8CF&k4#rg zA=#ZuTb;9#Hrd+GQho}~{NTC%|Jl2?=eTX|-sk)kn4~k38m%OE+$8l&#*X5QnkZH+ zH=T}?&R|JM%Zeme09<9~^Y_1h0~`PZxm;YhE7^*dFSd5cv&8M(&-tMTP|97xT z)H^AocHeGnzA(?QduB3cSWrRh36(Wt5B7uhSL%)*EKu)eMZ*$-{AC+TlDedkiL^FB zrG7|hi!=H;mDTUkrN8Y_ZSsUdHhHs|^NVw>VVX)bt~VKq*kx#ar#-kt&o9m=A56-` zJ!L6PI{J?cGHCR7!y?kn*2t%I_O8|08^8cgN6;S!v2nM-5&X1{-gGG$ z=*nQlrgt=xx-x8Sf54Vc;5p_2?Z}V|*Kr)IoL0_RdT1_U-exiV};G>_5dtV;YNc$=JdY4$1L~Y^+lCAz(x$U#q1Y4bk6`T=|URY zIYIM6aLC?Md3M6Jwf&)zx(iB5HX>Pu{uReBRwJ4TTpGkJz%%vIJv%!C-{aZ0-+l{F z#W~4}YDfgZUH|s%*$MiemzTbjeuQwQv~uWJ1#hGl#3js+(SKaO`L)Bs)Zcmu>;AcN zkJ#xvLFZ2To$KqvViClJsHs5s<_314V#5VDoWV34DQ*`A15)m>vIz9QRqhn<3!9!d zkY=fV$4U}4gkw||cLd#Vh5qQTB{5xu8R1gp&u}f@3Vx{DU7UEE5bZV?83DD|j}Ym@ zY#dOL;z6mQHM#LvoXf%XkOMm`PO0Wc&l)nh&%M2+3jdt4FU++f7)w!gB(EJWjmBkB zso2u6&>sV6QzX@kwW>6E5=kzJ^Yu|gQnh34(t9Ki*5bIWo!>~APu2U7Agzh3gd%Kk zT%1IrMUjQhuYFNK2-XzMPOyu89GiI8U7U!vf!76?>9#Hj=crKYQ(<;TOanAr#;($p zoIKkL5`b2gkkcx8m|FxH+squP0!a{!BZZSuG?LIJdPn#YHyrRWq~5KNcc?1h6$>r6 zBdH>UBYGXW*A%RwM5LijI751z`ZT2=drX4ps3QBfkG}SS(j$6};jCtnKof1!aUn>l zySAi(ocNVotH>tNj8b}&C|U(Mpadro^(xGcQk+E8QJ^nw28ikD3=l5Y?lDqC?u3CR zokY{ox&xYZ5?P-XKG3w2=z6sBSUf|Npr)@4YZ4Y|DD*zO`WC!X8)}7=E=QWSB!ViK zmyUU!pi3?A5;qT)IxweYD{PtSWXTDa8khl_fo=naWG79Nhpk0oyb#uprVEHvB&HM` zCT~o6cqQX6rJ*myqxz>M`bCcJk<$U;N=*ukBpZ`D2v_2MGz#&Abuy2d)>O6>sHe$C z4v;b=d9*TqjJmDCa1(|datrtGj@?fVbhgrU0VNj{#RU0dx*&X{EGICwGQ95yB+haT=~J za^yoQmSk^|H4QUMJX??oC+k`AdM#JxN~@RmtPp}g2Svgvj_6G)3@GYF6~A0QOMu&h zYcgo&jTy>TQ+wqU`Ge0=nb>8j@q?U$; zbF5Q_vR)I!gM*WkgOt^={bjV-){2|icZk#(&0(|4_hyMNDV6lj2jF_N6q~fZp7av~ z<4YOfB++WJW?pZtFn4OQ?aY#Zoog=JHcIQuT~<$7RE2J^B-tt9iE)NQAaIg3?Xf0y zQjvqtM@k>hTE<8%%GD!l=63#Kk!0Pgzo54JEO^MQ4wkGx-q>#8 z{J3++$WKYVL`%%PIrd}MWaT<9mIbHYil4Cy*Q+;4M*~3}dw_Y?J>e>wX8=!5y=>!YK?!}mX5{`BtZ$3Kdr5AW-?x`V$y9KI(%{t-#!=;-@@ zPJGdxT~O3wM%fIO^aTrV7{Ews`M1f+}v$?hNN;7&9Dg2*YG-tVJW zf9c_tPbGwsKpawY6Mf7Yb#2_etiQ~$@M8<=SN+sH%O4|G4ccr(Qr#!2J>f*PYYjK( ztd`0@OL^CbEH%hKizFJGrKUpM(OZ9?r53*SS!(+%wSAUa3eA0%+CEEdpQX0XQj59v zS!(+%wH;)sfzdXSs}?yp-9xcURQoICscD$!msv*JTD+roBq9N2;7eGepxOWqeG4(Z z1m%5LC-<-^Y`Dz}X~s|3W9Ct}0dJj?+R|a*68dNp|q0Pwu>3DQ3!-w&|bKa(xHKeB44k7JD zaZ~92U=P8k$e&HY)MmdX8Ljgvj;l2qoyw%bVbLy@yah6=2-C3c7|vJP z!o1|&nj_pajN)uTxbx;=q`&UcZANSXa@pP%H5dhoveXjG6=GCi5xs-fhv@Dq^ON#N zA}NB(CfG`~(4zWvoOvjmgAngpqGfK{FMoF??cYqFb}3_2Lr6hU8J{;GrPGc3R=uD2 z*XL+KTgFe$(_%`3qG#$!a48x4TA-3(F3__xR26kARhKA&j>+=P^Mqp+?dDH{!x z1`w&Vl>%HR))A-n8z9yV5;wHyO5ZBApbx;Y;&uE;jXhdniM%na$_EMke@o{p5!X&h z-`nq_^q=~}wHB`%-+LX5vFq@<;h`8^<(ARb+mxU;7In9=kWEA(%LrF8^t^*Cul+Gf zMJKpK57vZetbdSMk&g)X7o3ogiqFVx!9^0RD?HHOoV=)S0UNJsOHnkXogMEW-WpF@H%0-6D zZWf%}mu)wvAT_SNvD1F1b9FnS_qD-_0leS-u0#aVgn~-ySpFqXrvgzS;#;bPT*P9G#6fc-d)C&a!!2x3L2@&-*;=EzT)N zuS%NTp>v{?_KyL4%;x!SVC427WPbq=GB)?y*-r_Atirbm+fm(aoet`3!bgfr?FwYN zciO&M(EFL7vzC)t;mv)E8t@%isYJl&A?pWvwx6KN$*Vq>%EsGQ=l{h+ej`V%J|!YM z%}BF2)zqgmqa9CAMe9OPmVn4^o&YJB(X!Ot+~^JZfiw$CG8vg*$K!uo^r?o@cz~P( zd3XONcU5WL=bxF}r&tu_uotmSis;D1))bAk`;p3=-#65vY&;Bv@nw z4w%%dHhyb_>Qsx2XSgI6Z=$R+eQ)0|yVf5%J@f-+$R-?zpY7&BHO)5%3icQdeT&H1 z_0U>=TcLf61-ZxZGF#v8A~Ox}n9MHm4)NvH1^Nj{ZI30vr~vizoL0DqhmjpVI0W5^ z9n6)bgES!rr1++|ZKd6+?a7bWzJE<6dOGdk{80qW(C|3oidf*Vn(ONJ?9WA$fJ-yXL>Pq7<0I_Gc9sT@pBh^9(BLTn>ToUjI0K zM=)6(+wvPoHkuRkZc$`+S5EMRGWl8Ud=%}(Nq8=5pXp&;r8o|pXN<%$irKZECG{QV zoMr_MB?zg#puAk27|WFy+kSLP*@+(*=usn5*sQ+31N&KrBXx_~%+1z=fqj*P=hr>5 zj^}40UL8P4_nogK4gpqbes;R%RycOuCwmdMOQFZ5uwb&c0&BJx%e!6ar`A)0H6KclYYH%;jm`d^W8t~w9 zZK@9sRDS#UU%x7RH>f02u}D%8pu!d1k(Ju8-Ke zP&XYxcBjG^t_KjwaHR-lsCJH-*HEv>evNYk#S~erAEY?blN=cb-7}-^8Phi^s9xxN%&w)npppkZuyu^$_G#FM0KZB%sv2W{x;NK@ zAgcX%A`bUHcIvsF2>$(N3BtWi6p#0MYZ_;qVjfCOqw^Sq48?uz`FnYFfigx^9Y!= zReAnYNgKe|$23X9omAdotVGe9Fh8Ij3{HJUo2c2*f>rww>|B`x?x-KGb0@?(K+(QZ z)teoKezz(x)NonSdmemESCGqfm*dmrY%~fiQ`46lyE_owm?Qh-^v;GlqpXh6j%WL)3RJ~DmIe-dI zQ%v$PbVtA581m45)_S9j{g*|yx;RhI8Gz@E^x-yNKz@#1VIE+Em$=5a;AA+wzIt_e zBzKDP9$lOdBhaq41wZiX_GOKODM*sOYfnDFH0+>q)5xz7G}VaYyc#yFfaCzv8;2Vf z5+gmARerbKK*C7@-CIv%u)bfIskIpS!b@|LPiVtKTwu`-cw}E4zm&X)5$NMGSG|UE z{g~0x-z{)W`P;T63}t1@#d=lQThpW91pP_{D(b9ka}|#wwP^&T3ey;^`hAU#K@S|M zHXQ!+8^tgwQ+5$_h#IvUZVRz!=O-Dh924SHC!dMc>Aa-#(+bOBNP|MKV%`eM_-Rg- zq&(%t?F_SQQ3#TWmXTB36tj%hOSwsD%}=WQE6FftxXx!VQGIJ*vuwgkT$ECeeoG|F z#v{V3w>a;pz$L2NYEGES$}!KQhUOd$Yv?Rsd)rh+E!SeDdodl2?Jvx3iIqkwX{?*% zoHQj}NzSp(CH(GhpS!!kQvD2!_HCz27;mIh0dXd};15A;4a+Fh;y8Yny3<%+iTwwu|ZFg|j3oFlXFFgvwva=8{0$F$o8Rl;$Bm&UT; zQj{3HMP(Q6Zsg;VYPQa@2H(TU9(>5g?;Y7Q!#n!3AC_?IuRT)}^To!eK*BkgD_Xg7 nKWXO{&*laC+uy(Vr~S3R_SgRUbgzE|00960S`!%l0GJv8!?>!^ literal 0 HcmV?d00001 diff --git a/charts/rancher-monitoring-crd/102.0.5+up40.1.2/templates/_helpers.tpl b/charts/rancher-monitoring-crd/102.0.5+up40.1.2/templates/_helpers.tpl new file mode 100644 index 0000000000..edac2b315e --- /dev/null +++ b/charts/rancher-monitoring-crd/102.0.5+up40.1.2/templates/_helpers.tpl @@ -0,0 +1,50 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# CRD Installation + +{{- define "crd.established" -}} +{{- if not (regexMatch "^([a-zA-Z]+[.][a-zA-Z]*)+$" .) -}} +{{ required (printf "%s is not a valid CRD" .) "" }} +{{- else -}} +echo "beginning wait for {{ . }} to be established..."; +num_tries=1; +until kubectl get crd {{ . }} -o=jsonpath='{range .status.conditions[*]}{.type}={.status} {end}' | grep -qE 'Established=True'; do + if (( num_tries == 30 )); then + echo "timed out waiting for {{ . }}"; + exit 1; + fi; + num_tries=$(( num_tries + 1 )); + echo "{{ . }} is not established. Sleeping for 2 seconds and trying again..."; + sleep 2; +done; +echo "successfully established {{ . }}"; +{{- end -}} +{{- end -}} \ No newline at end of file diff --git a/charts/rancher-monitoring-crd/102.0.5+up40.1.2/templates/jobs.yaml b/charts/rancher-monitoring-crd/102.0.5+up40.1.2/templates/jobs.yaml new file mode 100644 index 0000000000..51b512adf4 --- /dev/null +++ b/charts/rancher-monitoring-crd/102.0.5+up40.1.2/templates/jobs.yaml @@ -0,0 +1,152 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ .Chart.Name }}-create + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Chart.Name }} + annotations: + "helm.sh/hook": post-install, post-upgrade, post-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed +spec: + template: + metadata: + name: {{ .Chart.Name }}-create + labels: + app: {{ .Chart.Name }} + spec: + serviceAccountName: {{ .Chart.Name }}-manager + securityContext: + runAsNonRoot: false + runAsUser: 0 + initContainers: + - name: set-preserve-unknown-fields-false + image: {{ template "system_default_registry" . }}{{ .Values.image.repository }}:{{ .Values.image.tag }} + imagePullPolicy: IfNotPresent + command: + - /bin/sh + - -c + - > + {{- range $path, $_ := (.Files.Glob "crd-manifest/**.yaml") }} + {{- $crd := get (get ($.Files.Get $path | fromYaml) "metadata") "name" }} + if [[ -n "$(kubectl get crd {{ $crd }} -o jsonpath='{.spec.preserveUnknownFields}')" ]]; then + patch='{"spec": {"preserveUnknownFields": false}}'; + if [[ -z "$(kubectl get crd {{ $crd }} -o jsonpath='{.spec.versions[0].schema}')" ]]; then + patch='{"spec": {"preserveUnknownFields": false, "versions": [{"name": "v1", "served": false, "storage": true, "schema": {"openAPIV3Schema": {"description": "placeholder", "type": "object"}}}]}}'; + fi + echo "Applying patch to {{ $crd }}: ${patch}" + if kubectl patch crd {{ $crd }} -p "${patch}" --type="merge"; then + {{- include "crd.established" $crd | nindent 18 }} + fi; + fi; + {{- end }} + containers: + - name: create-crds + image: {{ template "system_default_registry" . }}{{ .Values.image.repository }}:{{ .Values.image.tag }} + imagePullPolicy: IfNotPresent + command: + - /bin/sh + - -c + - > + echo "Applying CRDs..."; + mkdir -p /etc/crd; + base64 -d /etc/config/crd-manifest.tgz.b64 | tar -xzv -C /etc/crd; + kubectl replace -Rf /etc/crd || kubectl create -Rf /etc/crd; + + echo "Waiting for CRDs to be recognized before finishing installation..."; + + {{- range $path, $_ := (.Files.Glob "crd-manifest/**.yaml") }} + {{- $apiGroup := get (get ($.Files.Get $path | fromYaml) "spec") "group" }} + rm -rf $HOME/.kube/cache/discovery/*/{{ $apiGroup }}; + {{- end }} + + {{- range $path, $_ := (.Files.Glob "crd-manifest/**.yaml") }} + {{- $crd := get (get ($.Files.Get $path | fromYaml) "metadata") "name" }} + {{- include "crd.established" $crd | nindent 12 }} + {{- end }} + volumeMounts: + - name: crd-manifest + readOnly: true + mountPath: /etc/config + restartPolicy: OnFailure + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} + {{- if .Values.nodeSelector }} + {{- toYaml .Values.nodeSelector | nindent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} + {{- if .Values.tolerations }} + {{- toYaml .Values.tolerations | nindent 8 }} + {{- end }} + volumes: + - name: crd-manifest + configMap: + name: {{ .Chart.Name }}-manifest +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ .Chart.Name }}-delete + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Chart.Name }} + annotations: + "helm.sh/hook": pre-delete + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed +spec: + template: + metadata: + name: {{ .Chart.Name }}-delete + labels: + app: {{ .Chart.Name }} + spec: + serviceAccountName: {{ .Chart.Name }}-manager + securityContext: + runAsNonRoot: false + runAsUser: 0 + initContainers: + - name: remove-finalizers + image: {{ template "system_default_registry" . }}{{ .Values.image.repository }}:{{ .Values.image.tag }} + imagePullPolicy: IfNotPresent + command: + - /bin/sh + - -c + - > + {{- range $path, $_ := (.Files.Glob "crd-manifest/**.yaml") }} + {{- $crd := get (get ($.Files.Get $path | fromYaml) "metadata") "name" }} + if kubectl patch crd {{ $crd }} -p '{"metadata": {"finalizers": []}}'; then + {{- include "crd.established" $crd | nindent 14 }} + fi; + {{- end }} + volumeMounts: + - name: crd-manifest + readOnly: true + mountPath: /etc/config + containers: + - name: delete-crds + image: {{ template "system_default_registry" . }}{{ .Values.image.repository }}:{{ .Values.image.tag }} + imagePullPolicy: IfNotPresent + command: + - /bin/sh + - -c + - > + echo "Deleting CRDs..."; + mkdir -p /etc/crd; + base64 -d /etc/config/crd-manifest.tgz.b64 | tar -xzv -C /etc/crd; + kubectl delete --ignore-not-found=true -Rf /etc/crd; + volumeMounts: + - name: crd-manifest + readOnly: true + mountPath: /etc/config + restartPolicy: OnFailure + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} + {{- if .Values.nodeSelector }} + {{- toYaml .Values.nodeSelector | nindent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} + {{- if .Values.tolerations }} + {{- toYaml .Values.tolerations | nindent 8 }} + {{- end }} + volumes: + - name: crd-manifest + configMap: + name: {{ .Chart.Name }}-manifest diff --git a/charts/rancher-monitoring-crd/102.0.5+up40.1.2/templates/manifest.yaml b/charts/rancher-monitoring-crd/102.0.5+up40.1.2/templates/manifest.yaml new file mode 100644 index 0000000000..8dc9dfb447 --- /dev/null +++ b/charts/rancher-monitoring-crd/102.0.5+up40.1.2/templates/manifest.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Chart.Name }}-manifest + namespace: {{ .Release.Namespace }} +data: + crd-manifest.tgz.b64: + {{- .Files.Get "files/crd-manifest.tgz" | b64enc | indent 4 }} diff --git a/charts/rancher-monitoring-crd/102.0.5+up40.1.2/templates/rbac.yaml b/charts/rancher-monitoring-crd/102.0.5+up40.1.2/templates/rbac.yaml new file mode 100644 index 0000000000..a4d498b0fa --- /dev/null +++ b/charts/rancher-monitoring-crd/102.0.5+up40.1.2/templates/rbac.yaml @@ -0,0 +1,76 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ .Chart.Name }}-manager + labels: + app: {{ .Chart.Name }}-manager +rules: +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: ['create', 'get', 'patch', 'delete', 'update', 'list'] +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ .Chart.Name }}-manager +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ .Chart.Name }}-manager + labels: + app: {{ .Chart.Name }}-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ .Chart.Name }}-manager +subjects: +- kind: ServiceAccount + name: {{ .Chart.Name }}-manager + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Chart.Name }}-manager + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Chart.Name }}-manager +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ .Chart.Name }}-manager + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Chart.Name }}-manager +spec: + privileged: false + allowPrivilegeEscalation: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'configMap' + - 'secret' +{{- end }} diff --git a/charts/rancher-monitoring-crd/102.0.5+up40.1.2/templates/validate-psp-install.yaml b/charts/rancher-monitoring-crd/102.0.5+up40.1.2/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring-crd/102.0.5+up40.1.2/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring-crd/102.0.5+up40.1.2/values.yaml b/charts/rancher-monitoring-crd/102.0.5+up40.1.2/values.yaml new file mode 100644 index 0000000000..0f30e302f2 --- /dev/null +++ b/charts/rancher-monitoring-crd/102.0.5+up40.1.2/values.yaml @@ -0,0 +1,17 @@ +# Default values for rancher-monitoring-crd. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + +image: + repository: rancher/shell + tag: v0.1.25 + +nodeSelector: {} + +tolerations: [] diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/.helmignore b/charts/rancher-monitoring/102.0.5+up40.1.2/.helmignore new file mode 100644 index 0000000000..1937f42c7f --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/.helmignore @@ -0,0 +1,28 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +# helm/charts +OWNERS +hack/ +ci/ +kube-prometheus-*.tgz + +unittests/ diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/CHANGELOG.md b/charts/rancher-monitoring/102.0.5+up40.1.2/CHANGELOG.md new file mode 100644 index 0000000000..8178169b91 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/CHANGELOG.md @@ -0,0 +1,47 @@ +# Changelog +All notable changes from the upstream Prometheus Operator chart will be added to this file. + +## [Package Version 00] - 2020-07-19 +### Added +- Added [Prometheus Adapter](https://github.com/helm/charts/tree/master/stable/prometheus-adapter) as a dependency to the upstream Prometheus Operator chart to allow users to expose custom metrics from the default Prometheus instance deployed by this chart +- Remove `prometheus-operator/cleanup-crds.yaml` and `prometheus-operator/crds.yaml` from the Prometheus Operator upstream chart in favor of just using the CRD directory to install the CRDs. +- Added support for `rkeControllerManager`, `rkeScheduler`, `rkeProxy`, and `rkeEtcd` PushProx exporters for monitoring k8s components within RKE clusters +- Added support for a `k3sServer` PushProx exporter that monitors k3s server components (`kubeControllerManager`, `kubeScheduler`, and `kubeProxy`) within k3s clusters +- Added support for `kubeAdmControllerManager`, `kubeAdmScheduler`, `kubeAdmProxy`, and `kubeAdmEtcd` PushProx exporters for monitoring k8s components within kubeAdm clusters +- Added support for `rke2ControllerManager`, `rke2Scheduler`, `rke2Proxy`, and `rke2Etcd` PushProx exporters for monitoring k8s components within rke2 clusters +- Exposed `prometheus.prometheusSpec.ignoreNamespaceSelectors` on values.yaml and set it to `false` by default. This value instructs the default Prometheus server deployed with this chart to ignore the `namespaceSelector` field within any created ServiceMonitor or PodMonitor CRs that it selects. This prevents ServiceMonitors and PodMonitors from configuring the Prometheus scrape configuration to monitor resources outside the namespace that they are deployed in; if a user needs to have one ServiceMonitor / PodMonitor monitor resources within several namespaces (such as the resources that are used to monitor Istio in a default installation), they should not enable this option since it would require them to create one ServiceMonitor / PodMonitor CR per namespace that they would like to monitor. Relevant fields were also updated in the default README.md. +- Added `grafana.sidecar.dashboards.searchNamespace` to `values.yaml` with a default value of `cattle-dashboards`. The namespace provided should contain all ConfigMaps with the label `grafana_dashboard` and will be searched by the Grafana Dashboards sidecar for updates. The namespace specified is also created along with this deployment. All default dashboard ConfigMaps have been relocated from the deployment namespace to the namespace specified +- Added `monitoring-admin`, `monitoring-edit`, and `monitoring-view` default `ClusterRoles` to allow admins to assign roles to users to interact with Prometheus Operator CRs. These can be enabled by setting `.Values.global.rbac.userRoles.create` (default: `true`). In a typical RBAC setup, you might want to use a `ClusterRoleBinding` to bind these roles to a Subject to allow them to set up or view `ServiceMonitors` / `PodMonitors` / `PrometheusRules` and view `Prometheus` or `Alertmanager` CRs across the cluster. If `.Values.global.rbac.userRoles.aggregateRolesForRBAC` is enabled, these ClusterRoles will aggregate into the respective default ClusterRoles provided by Kubernetes +- Added `monitoring-config-admin`, `monitoring-config-edit` and `monitoring-config-view` default `Roles` to allow admins to assign roles to users to be able to edit / view `Secrets` and `ConfigMaps` within the `cattle-monitoring-system` namespace. These can be enabled by setting `.Values.global.rbac.userRoles.create` (default: `true`). In a typical RBAC setup, you might want to use a `RoleBinding` to bind these roles to a Subject within the `cattle-monitoring-system` namespace to allow them to modify Secrets / ConfigMaps tied to the deployment, such as your Alertmanager Config Secret. +- Added `monitoring-dashboard-admin`, `monitoring-dashboard-edit` and `monitoring-dashboard-view` default `Roles` to allow admins to assign roles to users to be able to edit / view `ConfigMaps` within the `cattle-dashboards` namespace. These can be enabled by setting `.Values.global.rbac.userRoles.create` (default: `true`) and deploying Grafana as part of this chart. In a typical RBAC setup, you might want to use a `RoleBinding` to bind these roles to a Subject within the `cattle-dashboards` namespace to allow them to create / modify ConfigMaps that contain the JSON used to persist Grafana Dashboards on the cluster. +- Added default resource limits for `Prometheus Operator`, `Prometheus`, `AlertManager`, `Grafana`, `kube-state-metrics`, `node-exporter` +- Added a default template `rancher_defaults.tmpl` to AlertManager that Rancher will offer to users in order to help configure the way alerts are rendered on a notifier. Also updated the default template deployed with this chart to reference that template and added an example of a Slack config using this template as a comment in the `values.yaml`. +- Added support for private registries via introducing a new field for `global.cattle.systemDefaultRegistry` that, if supplied, will automatically be prepended onto every image used by the chart. +- Added a default `nginx` proxy container deployed with Grafana whose config is set in the `ConfigMap` located in `charts/grafana/templates/nginx-config.yaml`. The purpose of this container is to make it possible to view Grafana's UI through a proxy that has a subpath (e.g. Rancher's proxy). This proxy container is set to listen on port `8080` (with a `portName` of `nginx-http` instead of the default `service`), which is also where the Grafana service will now point to, and will forward all requests to the Grafana container listening on the default port `3000`. +- Added a default `nginx` proxy container deployed with Prometheus whose config is set in the `ConfigMap` located in `templates/prometheus/nginx-config.yaml`. The purpose of this container is to make it possible to view Prometheus's UI through a proxy that has a subpath (e.g. Rancher's proxy). This proxy container is set to listen on port `8081` (with a `portName` of `nginx-http` instead of the default `web`), which is also where the Prometheus service will now point to, and will forward all requests to the Prometheus container listening on the default port `9090`. +- Added support for passing CIS Scans in a hardened cluster by introducing a Job that patches the default service account within the `cattle-monitoring-system` and `cattle-dashboards` namespaces on install or upgrade and adding a default allow all `NetworkPolicy` to the `cattle-monitoring-system` and `cattle-dashboards` namespaces. +### Modified +- Updated the chart name from `prometheus-operator` to `rancher-monitoring` and added the `io.rancher.certified: rancher` annotation to `Chart.yaml` +- Modified the default `node-exporter` port from `9100` to `9796` +- Modified the default `nameOverride` to `rancher-monitoring`. This change is necessary as the Prometheus Adapter's default URL (`http://{{ .Values.nameOverride }}-prometheus.{{ .Values.namespaceOverride }}.svc`) is based off of the value used here; if modified, the default Adapter URL must also be modified +- Modified the default `namespaceOverride` to `cattle-monitoring-system`. This change is necessary as the Prometheus Adapter's default URL (`http://{{ .Values.nameOverride }}-prometheus.{{ .Values.namespaceOverride }}.svc`) is based off of the value used here; if modified, the default Adapter URL must also be modified +- Configured some default values for `grafana.service` values and exposed them in the default README.md +- The default namespaces the following ServiceMonitors were changed from the deployment namespace to allow them to continue to monitor metrics when `prometheus.prometheusSpec.ignoreNamespaceSelectors` is enabled: + - `core-dns`: `kube-system` + - `api-server`: `default` + - `kube-controller-manager`: `kube-system` + - `kubelet`: `{{ .Values.kubelet.namespace }}` +- Disabled the following deployments by default (can be enabled if required): + - `AlertManager` + - `kube-controller-manager` metrics exporter + - `kube-etcd` metrics exporter + - `kube-scheduler` metrics exporter + - `kube-proxy` metrics exporter +- Updated default Grafana `deploymentStrategy` to `Recreate` to prevent deployments from being stuck on upgrade if a PV is attached to Grafana +- Modified the default `SelectorNilUsesHelmValues` to default to `false`. As a result, we look for all CRs with any labels in all namespaces by default rather than just the ones tagged with the label `release: rancher-monitoring`. +- Modified the default images used by the `rancher-monitoring` chart to point to Rancher mirrors of the original images from upstream. +- Modified the behavior of the chart to create the Alertmanager Config Secret via a pre-install hook instead of using the normal Helm lifecycle to manage the secret. The benefit of this approach is that all changes to the Config Secret done on a live cluster will never get overridden on a `helm upgrade` since the secret only gets created on a `helm install`. If you would like the secret to be cleaned up on an `helm uninstall`, enable `alertmanager.cleanupOnUninstall`; however, this is disabled by default to prevent the loss of alerting configuration on an uninstall. This secret will never be modified on a `helm upgrade`. +- Modified the default `securityContext` for `Pod` templates across the chart to `{"runAsNonRoot": "true", "runAsUser": "1000"}` and replaced `grafana.rbac.pspUseAppArmor` in favor of `grafana.rbac.pspAnnotations={}` in order to make it possible to deploy this chart on a hardened cluster which does not support Seccomp or AppArmor annotations in PSPs. Users can always choose to specify the annotations they want to use for the PSP directly as part of the values provided. +- Modified `.Values.prometheus.prometheusSpec.containers` to take in a string representing a template that should be rendered by Helm (via `tpl`) instead of allowing a user to provide YAML directly. +- Modified the default Grafana configuration to auto assign users who access Grafana to the Viewer role and enable anonymous access to Grafana dashboards by default. This default works well for a Rancher user who is accessing Grafana via the `kubectl proxy` on the Rancher Dashboard UI since anonymous users who enter via the proxy are authenticated by the k8s API Server, but you can / should modify this behavior if you plan on exposing Grafana in a way that does not require authentication (e.g. as a `NodePort` service). +- Modified the default Grafana configuration to add a default dashboard for Rancher on the Grafana home page. \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/CONTRIBUTING.md b/charts/rancher-monitoring/102.0.5+up40.1.2/CONTRIBUTING.md new file mode 100644 index 0000000000..f6ce2a3235 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/CONTRIBUTING.md @@ -0,0 +1,12 @@ +# Contributing Guidelines + +## How to contribute to this chart + +1. Fork this repository, develop and test your Chart. +1. Bump the chart version for every change. +1. Ensure PR title has the prefix `[kube-prometheus-stack]` +1. When making changes to rules or dashboards, see the README.md section on how to sync data from upstream repositories +1. Check the `hack/minikube` folder has scripts to set up minikube and components of this chart that will allow all components to be scraped. You can use this configuration when validating your changes. +1. Check for changes of RBAC rules. +1. Check for changes in CRD specs. +1. PR must pass the linter (`helm lint`) diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/Chart.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/Chart.yaml new file mode 100644 index 0000000000..f1a7782c5d --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/Chart.yaml @@ -0,0 +1,128 @@ +annotations: + artifacthub.io/links: | + - name: Chart Source + url: https://github.com/prometheus-community/helm-charts + - name: Upstream Project + url: https://github.com/prometheus-operator/kube-prometheus + artifacthub.io/operator: "true" + catalog.cattle.io/auto-install: rancher-monitoring-crd=match + catalog.cattle.io/certified: rancher + catalog.cattle.io/deploys-on-os: windows + catalog.cattle.io/display-name: Monitoring + catalog.cattle.io/kube-version: '>= 1.16.0-0 < 1.28.0-0' + catalog.cattle.io/namespace: cattle-monitoring-system + catalog.cattle.io/permits-os: linux,windows + catalog.cattle.io/provides-gvr: monitoring.coreos.com.prometheus/v1 + catalog.cattle.io/rancher-version: '>= 2.7.0-0 < 2.8.0-0' + catalog.cattle.io/release-name: rancher-monitoring + catalog.cattle.io/requests-cpu: 4500m + catalog.cattle.io/requests-memory: 4000Mi + catalog.cattle.io/type: cluster-tool + catalog.cattle.io/ui-component: monitoring + catalog.cattle.io/upstream-version: 19.0.3 +apiVersion: v2 +appVersion: 0.59.1 +dependencies: +- condition: grafana.enabled + name: grafana + repository: file://./charts/grafana +- condition: hardenedKubelet.enabled + name: hardenedKubelet + repository: file://./charts/hardenedKubelet +- condition: hardenedNodeExporter.enabled + name: hardenedNodeExporter + repository: file://./charts/hardenedNodeExporter +- condition: k3sServer.enabled + name: k3sServer + repository: file://./charts/k3sServer +- condition: kubeStateMetrics.enabled + name: kube-state-metrics + repository: file://./charts/kube-state-metrics +- condition: kubeAdmControllerManager.enabled + name: kubeAdmControllerManager + repository: file://./charts/kubeAdmControllerManager +- condition: kubeAdmEtcd.enabled + name: kubeAdmEtcd + repository: file://./charts/kubeAdmEtcd +- condition: kubeAdmProxy.enabled + name: kubeAdmProxy + repository: file://./charts/kubeAdmProxy +- condition: kubeAdmScheduler.enabled + name: kubeAdmScheduler + repository: file://./charts/kubeAdmScheduler +- condition: prometheus-adapter.enabled + name: prometheus-adapter + repository: file://./charts/prometheus-adapter +- condition: nodeExporter.enabled + name: prometheus-node-exporter + repository: file://./charts/prometheus-node-exporter +- condition: rke2ControllerManager.enabled + name: rke2ControllerManager + repository: file://./charts/rke2ControllerManager +- condition: rke2Etcd.enabled + name: rke2Etcd + repository: file://./charts/rke2Etcd +- condition: rke2IngressNginx.enabled + name: rke2IngressNginx + repository: file://./charts/rke2IngressNginx +- condition: rke2Proxy.enabled + name: rke2Proxy + repository: file://./charts/rke2Proxy +- condition: rke2Scheduler.enabled + name: rke2Scheduler + repository: file://./charts/rke2Scheduler +- condition: rkeControllerManager.enabled + name: rkeControllerManager + repository: file://./charts/rkeControllerManager +- condition: rkeEtcd.enabled + name: rkeEtcd + repository: file://./charts/rkeEtcd +- condition: rkeIngressNginx.enabled + name: rkeIngressNginx + repository: file://./charts/rkeIngressNginx +- condition: rkeProxy.enabled + name: rkeProxy + repository: file://./charts/rkeProxy +- condition: rkeScheduler.enabled + name: rkeScheduler + repository: file://./charts/rkeScheduler +- condition: global.cattle.windows.enabled + name: windowsExporter + repository: file://./charts/windowsExporter +description: Collects several related Helm charts, Grafana dashboards, and Prometheus + rules combined with documentation and scripts to provide easy to operate end-to-end + Kubernetes cluster monitoring with Prometheus using the Prometheus Operator. +home: https://github.com/prometheus-operator/kube-prometheus +icon: https://raw.githubusercontent.com/prometheus/prometheus.github.io/master/assets/prometheus_logo-cb55bb5c346.png +keywords: +- operator +- prometheus +- kube-prometheus +- monitoring +kubeVersion: '>=1.16.0-0' +maintainers: +- email: andrew@quadcorps.co.uk + name: andrewgkew +- email: cedric@desaintmartin.fr + name: desaintmartin +- email: gianrubio@gmail.com + name: gianrubio +- email: github.gkarthiks@gmail.com + name: gkarthiks +- email: kube-prometheus-stack@sisti.pt + name: GMartinez-Sisti +- email: scott@r6by.com + name: scottrigby +- email: miroslav.hadzhiev@gmail.com + name: Xtigyro +- email: arvind.iyengar@suse.com + name: Arvind +- email: amangeet.samra@suse.com + name: Geet + url: https://github.com/geethub97 +name: rancher-monitoring +sources: +- https://github.com/prometheus-community/helm-charts +- https://github.com/prometheus-operator/kube-prometheus +type: application +version: 102.0.5+up40.1.2 diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/README.md b/charts/rancher-monitoring/102.0.5+up40.1.2/README.md new file mode 100644 index 0000000000..dd2abe9c02 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/README.md @@ -0,0 +1,739 @@ +# kube-prometheus-stack + +Installs the [kube-prometheus stack](https://github.com/prometheus-operator/kube-prometheus), a collection of Kubernetes manifests, [Grafana](http://grafana.com/) dashboards, and [Prometheus rules](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/) combined with documentation and scripts to provide easy to operate end-to-end Kubernetes cluster monitoring with [Prometheus](https://prometheus.io/) using the [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator). + +See the [kube-prometheus](https://github.com/prometheus-operator/kube-prometheus) README for details about components, dashboards, and alerts. + +_Note: This chart was formerly named `prometheus-operator` chart, now renamed to more clearly reflect that it installs the `kube-prometheus` project stack, within which Prometheus Operator is only one component._ + +## Prerequisites + +- Kubernetes 1.16+ +- Helm 3+ + +## Get Helm Repository Info + +```console +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update +``` + +_See [`helm repo`](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + +## Install Helm Chart + +```console +helm install [RELEASE_NAME] prometheus-community/kube-prometheus-stack +``` + +_See [configuration](#configuration) below._ + +_See [helm install](https://helm.sh/docs/helm/helm_install/) for command documentation._ + +## Dependencies + +By default this chart installs additional, dependent charts: + +- [prometheus-community/kube-state-metrics](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-state-metrics) +- [prometheus-community/prometheus-node-exporter](https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-node-exporter) +- [grafana/grafana](https://github.com/grafana/helm-charts/tree/main/charts/grafana) + +To disable dependencies during installation, see [multiple releases](#multiple-releases) below. + +_See [helm dependency](https://helm.sh/docs/helm/helm_dependency/) for command documentation._ + +## Uninstall Helm Chart + +```console +helm uninstall [RELEASE_NAME] +``` + +This removes all the Kubernetes components associated with the chart and deletes the release. + +_See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall/) for command documentation._ + +CRDs created by this chart are not removed by default and should be manually cleaned up: + +```console +kubectl delete crd alertmanagerconfigs.monitoring.coreos.com +kubectl delete crd alertmanagers.monitoring.coreos.com +kubectl delete crd podmonitors.monitoring.coreos.com +kubectl delete crd probes.monitoring.coreos.com +kubectl delete crd prometheuses.monitoring.coreos.com +kubectl delete crd prometheusrules.monitoring.coreos.com +kubectl delete crd servicemonitors.monitoring.coreos.com +kubectl delete crd thanosrulers.monitoring.coreos.com +``` + +## Upgrading Chart + +```console +helm upgrade [RELEASE_NAME] prometheus-community/kube-prometheus-stack +``` + +With Helm v3, CRDs created by this chart are not updated by default and should be manually updated. +Consult also the [Helm Documentation on CRDs](https://helm.sh/docs/chart_best_practices/custom_resource_definitions). + +_See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documentation._ + +### Upgrading an existing Release to a new major version + +A major chart version change (like v1.2.3 -> v2.0.0) indicates that there is an incompatible breaking change needing manual actions. + +### From 39.x to 40.x + +This version upgrades Prometheus-Operator to v0.59.1, Prometheus to v2.38.0, kube-state-metrics to v2.6.0 and Thanos to v0.28.0. +This version also upgrades the Helm charts of kube-state-metrics to 4.18.0 and prometheus-node-exporter to 4.2.0. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +Starting from prometheus-node-exporter version 4.0.0, the `node exporter` chart is using the [Kubernetes recommended labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/common-labels/). Therefore you have to delete the daemonset before you upgrade. + +```console +kubectl delete daemonset -l app=prometheus-node-exporter +helm upgrade -i kube-prometheus-stack prometheus-community/kube-prometheus-stack +``` + +If you use your own custom [ServiceMonitor](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#servicemonitor) or [PodMonitor](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#podmonitor), please ensure to upgrade their `selector` fields accordingly to the new labels. + +### From 38.x to 39.x + +This upgraded prometheus-operator to v0.58.0 and prometheus to v2.37.0 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 37.x to 38.x + +Reverted one of the default metrics relabelings for cAdvisor added in 36.x, due to it breaking container_network_* and various other statistics. If you do not want this change, you will need to override the `kubelet.cAdvisorMetricRelabelings`. + +### From 36.x to 37.x + +This includes some default metric relabelings for cAdvisor and apiserver metrics to reduce cardinality. If you do not want these defaults, you will need to override the `kubeApiServer.metricRelabelings` and or `kubelet.cAdvisorMetricRelabelings`. + +### From 35.x to 36.x + +This upgraded prometheus-operator to v0.57.0 and prometheus to v2.36.1 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 34.x to 35.x + +This upgraded prometheus-operator to v0.56.0 and prometheus to v2.35.0 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 33.x to 34.x + +This upgrades to prometheus-operator to v0.55.0 and prometheus to v2.33.5. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 32.x to 33.x + +This upgrades the prometheus-node-exporter Chart to v3.0.0. Please review the changes to this subchart if you make customizations to hostMountPropagation. + +### From 31.x to 32.x + +This upgrades to prometheus-operator to v0.54.0 and prometheus to v2.33.1. It also changes the default for `grafana.serviceMonitor.enabled` to `true. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 30.x to 31.x + +This version removes the built-in grafana ServiceMonitor and instead relies on the ServiceMonitor of the sub-chart. +`grafana.serviceMonitor.enabled` must be set instead of `grafana.serviceMonitor.selfMonitor` and the old ServiceMonitor may +need to be manually cleaned up after deploying the new release. + +### From 29.x to 30.x + +This version updates kube-state-metrics to 4.3.0 and uses the new option `kube-state-metrics.releaseLabel=true` which adds the "release" label to kube-state-metrics labels, making scraping of the metrics by kube-prometheus-stack work out of the box again, independent of the used kube-prometheus-stack release name. If you already set the "release" label via `kube-state-metrics.customLabels` you might have to remove that and use it via the new option. + +### From 28.x to 29.x + +This version makes scraping port for kube-controller-manager and kube-scheduler dynamic to reflect changes to default serving ports +for those components in Kubernetes versions v1.22 and v1.23 respectively. + +If you deploy on clusters using version v1.22+, kube-controller-manager will be scraped over HTTPS on port 10257. + +If you deploy on clusters running version v1.23+, kube-scheduler will be scraped over HTTPS on port 10259. + +### From 27.x to 28.x + +This version disables PodSecurityPolicies by default because they are deprecated in Kubernetes 1.21 and will be removed in Kubernetes 1.25. + +If you are using PodSecurityPolicies you can enable the previous behaviour by setting `kube-state-metrics.podSecurityPolicy.enabled`, `prometheus-node-exporter.rbac.pspEnabled`, `grafana.rbac.pspEnabled` and `global.rbac.pspEnabled` to `true`. + +### From 26.x to 27.x + +This version splits prometheus-node-exporter chart recording and altering rules in separate config values. +Instead of `defaultRules.rules.node` the 2 new variables `defaultRules.rules.nodeExporterAlerting` and `defaultRules.rules.nodeExporterRecording` are used. + +Also the following defaultRules.rules has been removed as they had no effect: `kubeApiserverError`, `kubePrometheusNodeAlerting`, `kubernetesAbsent`, `time`. + +The ability to set a rubookUrl via `defaultRules.rules.rubookUrl` was reintroduced. + +### From 25.x to 26.x + +This version enables the prometheus-node-exporter subchart servicemonitor by default again, by setting `prometheus-node-exporter.prometheus.monitor.enabled` to `true`. + +### From 24.x to 25.x + +This version upgrade to prometheus-operator v0.53.1. It removes support for setting a runbookUrl, since the upstream format for runbooks changed. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 23.x to 24.x + +The custom `ServiceMonitor` for the _kube-state-metrics_ & _prometheus-node-exporter_ charts have been removed in favour of the built-in sub-chart `ServiceMonitor`; for both sub-charts this means that `ServiceMonitor` customisations happen via the values passed to the chart. If you haven't directly customised this behaviour then there are no changes required to upgrade, but if you have please read the following. + +For _kube-state-metrics_ the `ServiceMonitor` customisation is now set via `kube-state-metrics.prometheus.monitor` and the `kubeStateMetrics.serviceMonitor.selfMonitor.enabled` value has moved to `kube-state-metrics.selfMonitor.enabled`. + +For _prometheus-node-exporter_ the `ServiceMonitor` customisation is now set via `prometheus-node-exporter.prometheus.monitor` and the `nodeExporter.jobLabel` values has moved to `prometheus-node-exporter.prometheus.monitor.jobLabel`. + +### From 22.x to 23.x + +Port names have been renamed for Istio's +[explicit protocol selection](https://istio.io/latest/docs/ops/configuration/traffic-management/protocol-selection/#explicit-protocol-selection). + +| | old value | new value | +|-|-----------|-----------| +| `alertmanager.alertmanagerSpec.portName` | `web` | `http-web` | +| `grafana.service.portName` | `service` | `http-web` | +| `prometheus-node-exporter.service.portName` | `metrics` (hardcoded) | `http-metrics` | +| `prometheus.prometheusSpec.portName` | `web` | `http-web` | + +### From 21.x to 22.x + +Due to the upgrade of the `kube-state-metrics` chart, removal of its deployment/stateful needs to done manually prior to upgrading: + +```console +kubectl delete deployments.apps -l app.kubernetes.io/instance=prometheus-operator,app.kubernetes.io/name=kube-state-metrics --cascade=orphan +``` + +or if you use autosharding: + +```console +kubectl delete statefulsets.apps -l app.kubernetes.io/instance=prometheus-operator,app.kubernetes.io/name=kube-state-metrics --cascade=orphan +``` + +### From 20.x to 21.x + +The config reloader values have been refactored. All the values have been moved to the key `prometheusConfigReloader` and the limits and requests can now be set separately. + +### From 19.x to 20.x + +Version 20 upgrades prometheus-operator from 0.50.x to 0.52.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRDs manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 18.x to 19.x + +`kubeStateMetrics.serviceMonitor.namespaceOverride` was removed. +Please use `kube-state-metrics.namespaceOverride` instead. + +### From 17.x to 18.x + +Version 18 upgrades prometheus-operator from 0.49.x to 0.50.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRDs manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 16.x to 17.x + +Version 17 upgrades prometheus-operator from 0.48.x to 0.49.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRDs manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 15.x to 16.x + +Version 16 upgrades kube-state-metrics to v2.0.0. This includes changed command-line arguments and removed metrics, see this [blog post](https://kubernetes.io/blog/2021/04/13/kube-state-metrics-v-2-0/). This version also removes Grafana dashboards that supported Kubernetes 1.14 or earlier. + +### From 14.x to 15.x + +Version 15 upgrades prometheus-operator from 0.46.x to 0.47.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRDs manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 13.x to 14.x + +Version 14 upgrades prometheus-operator from 0.45.x to 0.46.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRDs manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 12.x to 13.x + +Version 13 upgrades prometheus-operator from 0.44.x to 0.45.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRD manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.45.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.45.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.45.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +``` + +### From 11.x to 12.x + +Version 12 upgrades prometheus-operator from 0.43.x to 0.44.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRD manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/release-0.44/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +``` + +The chart was migrated to support only helm v3 and later. + +### From 10.x to 11.x + +Version 11 upgrades prometheus-operator from 0.42.x to 0.43.x. Starting with 0.43.x an additional `AlertmanagerConfigs` CRD is introduced. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRD manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/release-0.43/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +``` + +Version 11 removes the deprecated tlsProxy via ghostunnel in favor of native TLS support the prometheus-operator gained with v0.39.0. + +### From 9.x to 10.x + +Version 10 upgrades prometheus-operator from 0.38.x to 0.42.x. Starting with 0.40.x an additional `Probes` CRD is introduced. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRD manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/release-0.42/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +``` + +### From 8.x to 9.x + +Version 9 of the helm chart removes the existing `additionalScrapeConfigsExternal` in favour of `additionalScrapeConfigsSecret`. This change lets users specify the secret name and secret key to use for the additional scrape configuration of prometheus. This is useful for users that have prometheus-operator as a subchart and also have a template that creates the additional scrape configuration. + +### From 7.x to 8.x + +Due to new template functions being used in the rules in version 8.x.x of the chart, an upgrade to Prometheus Operator and Prometheus is necessary in order to support them. First, upgrade to the latest version of 7.x.x + +```console +helm upgrade [RELEASE_NAME] prometheus-community/kube-prometheus-stack --version 7.5.0 +``` + +Then upgrade to 8.x.x + +```console +helm upgrade [RELEASE_NAME] prometheus-community/kube-prometheus-stack --version [8.x.x] +``` + +Minimal recommended Prometheus version for this chart release is `2.12.x` + +### From 6.x to 7.x + +Due to a change in grafana subchart, version 7.x.x now requires Helm >= 2.12.0. + +### From 5.x to 6.x + +Due to a change in deployment labels of kube-state-metrics, the upgrade requires `helm upgrade --force` in order to re-create the deployment. If this is not done an error will occur indicating that the deployment cannot be modified: + +```console +invalid: spec.selector: Invalid value: v1.LabelSelector{MatchLabels:map[string]string{"app.kubernetes.io/name":"kube-state-metrics"}, MatchExpressions:[]v1.LabelSelectorRequirement(nil)}: field is immutable +``` + +If this error has already been encountered, a `helm history` command can be used to determine which release has worked, then `helm rollback` to the release, then `helm upgrade --force` to this new one + +## Configuration + +See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To see all configurable options with detailed comments: + +```console +helm show values prometheus-community/kube-prometheus-stack +``` + +You may also run `helm show values` on this chart's [dependencies](#dependencies) for additional options. + +### Rancher Monitoring Configuration + +The following table shows values exposed by Rancher Monitoring's additions to the chart: + +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `nameOverride` | Provide a name that should be used instead of the chart name when naming all resources deployed by this chart |`"rancher-monitoring"`| +| `namespaceOverride` | Override the deployment namespace | `"cattle-monitoring-system"` | +| `global.rbac.userRoles.create` | Create default user ClusterRoles to allow users to interact with Prometheus CRs, ConfigMaps, and Secrets | `true` | +| `global.rbac.userRoles.aggregateToDefaultRoles` | Aggregate default user ClusterRoles into default k8s ClusterRoles | `true` | +| `prometheus-adapter.enabled` | Whether to install [prometheus-adapter](https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-adapter) within the cluster | `true` | +| `prometheus-adapter.prometheus.url` | A URL pointing to the Prometheus deployment within your cluster. The default value is set based on the assumption that you plan to deploy the default Prometheus instance from this chart where `.Values.namespaceOverride=cattle-monitoring-system` and `.Values.nameOverride=rancher-monitoring` | `http://rancher-monitoring-prometheus.cattle-monitoring-system.svc` | +| `prometheus-adapter.prometheus.port` | The port on the Prometheus deployment that Prometheus Adapter can make requests to | `9090` | +| `prometheus.prometheusSpec.ignoreNamespaceSelectors` | Ignore NamespaceSelector settings from the PodMonitor and ServiceMonitor configs. If true, PodMonitors and ServiceMonitors can only discover Pods and Services within the namespace they are deployed into | `false` | + +The following values are enabled for different distributions via [rancher-pushprox](https://github.com/rancher/dev-charts/tree/master/packages/rancher-pushprox). See the rancher-pushprox `README.md` for more information on what all values can be configured for the PushProxy chart. + +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `rkeControllerManager.enabled` | Create a PushProx installation for monitoring kube-controller-manager metrics in RKE clusters | `false` | +| `rkeScheduler.enabled` | Create a PushProx installation for monitoring kube-scheduler metrics in RKE clusters | `false` | +| `rkeProxy.enabled` | Create a PushProx installation for monitoring kube-proxy metrics in RKE clusters | `false` | +| `rkeIngressNginx.enabled` | Create a PushProx installation for monitoring ingress-nginx metrics in RKE clusters | `false` | +| `rkeEtcd.enabled` | Create a PushProx installation for monitoring etcd metrics in RKE clusters | `false` | +| `rke2IngressNginx.enabled` | Create a PushProx installation for monitoring ingress-nginx metrics in RKE2 clusters | `false` | +| `k3sServer.enabled` | Create a PushProx installation for monitoring k3s-server metrics (accounts for kube-controller-manager, kube-scheduler, and kube-proxy metrics) in k3s clusters | `false` | +| `kubeAdmControllerManager.enabled` | Create a PushProx installation for monitoring kube-controller-manager metrics in kubeAdm clusters | `false` | +| `kubeAdmScheduler.enabled` | Create a PushProx installation for monitoring kube-scheduler metrics in kubeAdm clusters | `false` | +| `kubeAdmProxy.enabled` | Create a PushProx installation for monitoring kube-proxy metrics in kubeAdm clusters | `false` | +| `kubeAdmEtcd.enabled` | Create a PushProx installation for monitoring etcd metrics in kubeAdm clusters | `false` | + + +### Multiple releases + +The same chart can be used to run multiple Prometheus instances in the same cluster if required. To achieve this, it is necessary to run only one instance of prometheus-operator and a pair of alertmanager pods for an HA configuration, while all other components need to be disabled. To disable a dependency during installation, set `kubeStateMetrics.enabled`, `nodeExporter.enabled` and `grafana.enabled` to `false`. + +## Work-Arounds for Known Issues + +### Running on private GKE clusters + +When Google configure the control plane for private clusters, they automatically configure VPC peering between your Kubernetes cluster’s network and a separate Google managed project. In order to restrict what Google are able to access within your cluster, the firewall rules configured restrict access to your Kubernetes pods. This means that in order to use the webhook component with a GKE private cluster, you must configure an additional firewall rule to allow the GKE control plane access to your webhook pod. + +You can read more information on how to add firewall rules for the GKE control plane nodes in the [GKE docs](https://cloud.google.com/kubernetes-engine/docs/how-to/private-clusters#add_firewall_rules) + +Alternatively, you can disable the hooks by setting `prometheusOperator.admissionWebhooks.enabled=false`. + +## PrometheusRules Admission Webhooks + +With Prometheus Operator version 0.30+, the core Prometheus Operator pod exposes an endpoint that will integrate with the `validatingwebhookconfiguration` Kubernetes feature to prevent malformed rules from being added to the cluster. + +### How the Chart Configures the Hooks + +A validating and mutating webhook configuration requires the endpoint to which the request is sent to use TLS. It is possible to set up custom certificates to do this, but in most cases, a self-signed certificate is enough. The setup of this component requires some more complex orchestration when using helm. The steps are created to be idempotent and to allow turning the feature on and off without running into helm quirks. + +1. A pre-install hook provisions a certificate into the same namespace using a format compatible with provisioning using end user certificates. If the certificate already exists, the hook exits. +2. The prometheus operator pod is configured to use a TLS proxy container, which will load that certificate. +3. Validating and Mutating webhook configurations are created in the cluster, with their failure mode set to Ignore. This allows rules to be created by the same chart at the same time, even though the webhook has not yet been fully set up - it does not have the correct CA field set. +4. A post-install hook reads the CA from the secret created by step 1 and patches the Validating and Mutating webhook configurations. This process will allow a custom CA provisioned by some other process to also be patched into the webhook configurations. The chosen failure policy is also patched into the webhook configurations + +### Alternatives + +It should be possible to use [jetstack/cert-manager](https://github.com/jetstack/cert-manager) if a more complete solution is required, but it has not been tested. + +You can enable automatic self-signed TLS certificate provisioning via cert-manager by setting the `prometheusOperator.admissionWebhooks.certManager.enabled` value to true. + +### Limitations + +Because the operator can only run as a single pod, there is potential for this component failure to cause rule deployment failure. Because this risk is outweighed by the benefit of having validation, the feature is enabled by default. + +## Developing Prometheus Rules and Grafana Dashboards + +This chart Grafana Dashboards and Prometheus Rules are just a copy from [prometheus-operator/prometheus-operator](https://github.com/prometheus-operator/prometheus-operator) and other sources, synced (with alterations) by scripts in [hack](hack) folder. In order to introduce any changes you need to first [add them to the original repository](https://github.com/prometheus-operator/kube-prometheus/blob/main/docs/customizations/developing-prometheus-rules-and-grafana-dashboards.md) and then sync there by scripts. + +## Further Information + +For more in-depth documentation of configuration options meanings, please see + +- [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator) +- [Prometheus](https://prometheus.io/docs/introduction/overview/) +- [Grafana](https://github.com/grafana/helm-charts/tree/main/charts/grafana#grafana-helm-chart) + +## prometheus.io/scrape + +The prometheus operator does not support annotation-based discovery of services, using the `PodMonitor` or `ServiceMonitor` CRD in its place as they provide far more configuration options. +For information on how to use PodMonitors/ServiceMonitors, please see the documentation on the `prometheus-operator/prometheus-operator` documentation here: + +- [ServiceMonitors](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/getting-started.md#include-servicemonitors) +- [PodMonitors](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/getting-started.md#include-podmonitors) +- [Running Exporters](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/running-exporters.md) + +By default, Prometheus discovers PodMonitors and ServiceMonitors within its namespace, that are labeled with the same release tag as the prometheus-operator release. +Sometimes, you may need to discover custom PodMonitors/ServiceMonitors, for example used to scrape data from third-party applications. +An easy way of doing this, without compromising the default PodMonitors/ServiceMonitors discovery, is allowing Prometheus to discover all PodMonitors/ServiceMonitors within its namespace, without applying label filtering. +To do so, you can set `prometheus.prometheusSpec.podMonitorSelectorNilUsesHelmValues` and `prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues` to `false`. + +## Migrating from stable/prometheus-operator chart + +## Zero downtime + +Since `kube-prometheus-stack` is fully compatible with the `stable/prometheus-operator` chart, a migration without downtime can be achieved. +However, the old name prefix needs to be kept. If you want the new name please follow the step by step guide below (with downtime). + +You can override the name to achieve this: + +```console +helm upgrade prometheus-operator prometheus-community/kube-prometheus-stack -n monitoring --reuse-values --set nameOverride=prometheus-operator +``` + +**Note**: It is recommended to run this first with `--dry-run --debug`. + +## Redeploy with new name (downtime) + +If the **prometheus-operator** values are compatible with the new **kube-prometheus-stack** chart, please follow the below steps for migration: + +> The guide presumes that chart is deployed in `monitoring` namespace and the deployments are running there. If in other namespace, please replace the `monitoring` to the deployed namespace. + +1. Patch the PersistenceVolume created/used by the prometheus-operator chart to `Retain` claim policy: + + ```console + kubectl patch pv/ -p '{"spec":{"persistentVolumeReclaimPolicy":"Retain"}}' + ``` + + **Note:** To execute the above command, the user must have a cluster wide permission. Please refer [Kubernetes RBAC](https://kubernetes.io/docs/reference/access-authn-authz/rbac/) + +2. Uninstall the **prometheus-operator** release and delete the existing PersistentVolumeClaim, and verify PV become Released. + + ```console + helm uninstall prometheus-operator -n monitoring + kubectl delete pvc/ -n monitoring + ``` + + Additionally, you have to manually remove the remaining `prometheus-operator-kubelet` service. + + ```console + kubectl delete service/prometheus-operator-kubelet -n kube-system + ``` + + You can choose to remove all your existing CRDs (ServiceMonitors, Podmonitors, etc.) if you want to. + +3. Remove current `spec.claimRef` values to change the PV's status from Released to Available. + + ```console + kubectl patch pv/ --type json -p='[{"op": "remove", "path": "/spec/claimRef"}]' -n monitoring + ``` + +**Note:** To execute the above command, the user must have a cluster wide permission. Please refer to [Kubernetes RBAC](https://kubernetes.io/docs/reference/access-authn-authz/rbac/) + +After these steps, proceed to a fresh **kube-prometheus-stack** installation and make sure the current release of **kube-prometheus-stack** matching the `volumeClaimTemplate` values in the `values.yaml`. + +The binding is done via matching a specific amount of storage requested and with certain access modes. + +For example, if you had storage specified as this with **prometheus-operator**: + +```yaml +volumeClaimTemplate: + spec: + storageClassName: gp2 + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 50Gi +``` + +You have to specify matching `volumeClaimTemplate` with 50Gi storage and `ReadWriteOnce` access mode. + +Additionally, you should check the current AZ of your legacy installation's PV, and configure the fresh release to use the same AZ as the old one. If the pods are in a different AZ than the PV, the release will fail to bind the existing one, hence creating a new PV. + +This can be achieved either by specifying the labels through `values.yaml`, e.g. setting `prometheus.prometheusSpec.nodeSelector` to: + +```yaml +nodeSelector: + failure-domain.beta.kubernetes.io/zone: east-west-1a +``` + +or passing these values as `--set` overrides during installation. + +The new release should now re-attach your previously released PV with its content. + +## Migrating from coreos/prometheus-operator chart + +The multiple charts have been combined into a single chart that installs prometheus operator, prometheus, alertmanager, grafana as well as the multitude of exporters necessary to monitor a cluster. + +There is no simple and direct migration path between the charts as the changes are extensive and intended to make the chart easier to support. + +The capabilities of the old chart are all available in the new chart, including the ability to run multiple prometheus instances on a single cluster - you will need to disable the parts of the chart you do not wish to deploy. + +You can check out the tickets for this change [here](https://github.com/prometheus-operator/prometheus-operator/issues/592) and [here](https://github.com/helm/charts/pull/6765). + +### High-level overview of Changes + +#### Added dependencies + +The chart has added 3 [dependencies](#dependencies). + +- Node-Exporter, Kube-State-Metrics: These components are loaded as dependencies into the chart, and are relatively simple components +- Grafana: The Grafana chart is more feature-rich than this chart - it contains a sidecar that is able to load data sources and dashboards from configmaps deployed into the same cluster. For more information check out the [documentation for the chart](https://github.com/grafana/helm-charts/blob/main/charts/grafana/README.md) + +#### Kubelet Service + +Because the kubelet service has a new name in the chart, make sure to clean up the old kubelet service in the `kube-system` namespace to prevent counting container metrics twice. + +#### Persistent Volumes + +If you would like to keep the data of the current persistent volumes, it should be possible to attach existing volumes to new PVCs and PVs that are created using the conventions in the new chart. For example, in order to use an existing Azure disk for a helm release called `prometheus-migration` the following resources can be created: + +```yaml +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pvc-prometheus-migration-prometheus-0 +spec: + accessModes: + - ReadWriteOnce + azureDisk: + cachingMode: None + diskName: pvc-prometheus-migration-prometheus-0 + diskURI: /subscriptions/f5125d82-2622-4c50-8d25-3f7ba3e9ac4b/resourceGroups/sample-migration-resource-group/providers/Microsoft.Compute/disks/pvc-prometheus-migration-prometheus-0 + fsType: "" + kind: Managed + readOnly: false + capacity: + storage: 1Gi + persistentVolumeReclaimPolicy: Delete + storageClassName: prometheus + volumeMode: Filesystem +``` + +```yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + labels: + app.kubernetes.io/name: prometheus + prometheus: prometheus-migration-prometheus + name: prometheus-prometheus-migration-prometheus-db-prometheus-prometheus-migration-prometheus-0 + namespace: monitoring +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: prometheus + volumeMode: Filesystem + volumeName: pvc-prometheus-migration-prometheus-0 +``` + +The PVC will take ownership of the PV and when you create a release using a persistent volume claim template it will use the existing PVCs as they match the naming convention used by the chart. For other cloud providers similar approaches can be used. + +#### KubeProxy + +The metrics bind address of kube-proxy is default to `127.0.0.1:10249` that prometheus instances **cannot** access to. You should expose metrics by changing `metricsBindAddress` field value to `0.0.0.0:10249` if you want to collect them. + +Depending on the cluster, the relevant part `config.conf` will be in ConfigMap `kube-system/kube-proxy` or `kube-system/kube-proxy-config`. For example: + +```console +kubectl -n kube-system edit cm kube-proxy +``` + +```yaml +apiVersion: v1 +data: + config.conf: |- + apiVersion: kubeproxy.config.k8s.io/v1alpha1 + kind: KubeProxyConfiguration + # ... + # metricsBindAddress: 127.0.0.1:10249 + metricsBindAddress: 0.0.0.0:10249 + # ... + kubeconfig.conf: |- + # ... +kind: ConfigMap +metadata: + labels: + app: kube-proxy + name: kube-proxy + namespace: kube-system +``` diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/app-README.md b/charts/rancher-monitoring/102.0.5+up40.1.2/app-README.md new file mode 100644 index 0000000000..3920854384 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/app-README.md @@ -0,0 +1,46 @@ +# Rancher Monitoring and Alerting + + This chart is based on the upstream [kube-prometheus-stack](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack) chart. The chart deploys [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator) and its CRDs along with [Grafana](https://github.com/grafana/helm-charts/tree/main/charts/grafana), [Prometheus Adapter](https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-adapter) and additional charts / Kubernetes manifests to gather metrics. It allows users to monitor their Kubernetes clusters, view metrics in Grafana dashboards, and set up alerts and notifications. + +For more information on how to use the feature, refer to our [docs](https://rancher.com/docs/rancher/v2.x/en/monitoring-alerting/v2.5/). + +The chart installs the following components: + +- [Prometheus Operator](https://github.com/coreos/prometheus-operator) - The operator provides easy monitoring definitions for Kubernetes services, manages [Prometheus](https://prometheus.io/) and [AlertManager](https://prometheus.io/docs/alerting/latest/alertmanager/) instances, and adds default scrape targets for some Kubernetes components. +- [kube-prometheus](https://github.com/prometheus-operator/kube-prometheus/) - A collection of community-curated Kubernetes manifests, Grafana Dashboards, and PrometheusRules that deploy a default end-to-end cluster monitoring configuration. +- [Grafana](https://github.com/grafana/helm-charts/tree/main/charts/grafana) - Grafana allows a user to create / view dashboards based on the cluster metrics collected by Prometheus. +- [node-exporter](https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-node-exporter) / [kube-state-metrics](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-state-metrics) / [rancher-pushprox](https://github.com/rancher/charts/tree/dev-v2.7/packages/rancher-monitoring/rancher-pushprox/charts) - These charts monitor various Kubernetes components across different Kubernetes cluster types. +- [Prometheus Adapter](https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-adapter) - The adapter allows a user to expose custom metrics, resource metrics, and external metrics on the default [Prometheus](https://prometheus.io/) instance to the Kubernetes API Server. + +For more information, review the Helm README of this chart. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. +​ +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Upgrading from 100.0.0+up16.6.0 to 100.1.0+up19.0.3 + +### Noticeable changes: +Grafana: +- `sidecar.dashboards.searchNamespace`, `sidecar.datasources.searchNamespace` and `sidecar.notifiers.searchNamespace` support a list of namespaces now. + +Kube-state-metrics +- the type of `collectors` is changed from Dictionary to List. +- `kubeStateMetrics.serviceMonitor.namespaceOverride` was replaced by `kube-state-metrics.namespaceOverride`. + +### Known issues: +- Occasionally, the upgrade fails with errors related to the webhook `prometheusrulemutate.monitoring.coreos.com`. This is a known issue in the upstream, and the workaround is to trigger the upgrade one more time. [32416](https://github.com/rancher/rancher/issues/32416#issuecomment-828881726) diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/.helmignore b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/.helmignore new file mode 100644 index 0000000000..8cade1318f --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.vscode +.project +.idea/ +*.tmproj +OWNERS diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/Chart.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/Chart.yaml new file mode 100644 index 0000000000..9b0ca17926 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/Chart.yaml @@ -0,0 +1,29 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.16.0-0 < 1.27.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-grafana +apiVersion: v2 +appVersion: 9.1.5 +description: The leading tool for querying and visualizing time series and metrics. +home: https://grafana.net +icon: https://raw.githubusercontent.com/grafana/grafana/master/public/img/logo_transparent_400x.png +kubeVersion: ^1.8.0-0 +maintainers: +- email: zanhsieh@gmail.com + name: zanhsieh +- email: rluckie@cisco.com + name: rtluckie +- email: maor.friedman@redhat.com + name: maorfr +- email: miroslav.hadzhiev@gmail.com + name: Xtigyro +- email: mail@torstenwalter.de + name: torstenwalter +name: grafana +sources: +- https://github.com/grafana/grafana +type: application +version: 6.38.6 diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/README.md b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/README.md new file mode 100644 index 0000000000..45046f0d8a --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/README.md @@ -0,0 +1,574 @@ +# Grafana Helm Chart + +* Installs the web dashboarding system [Grafana](http://grafana.org/) + +## Get Repo Info + +```console +helm repo add grafana https://grafana.github.io/helm-charts +helm repo update +``` + +_See [helm repo](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + +## Installing the Chart + +To install the chart with the release name `my-release`: + +```console +helm install my-release grafana/grafana +``` + +## Uninstalling the Chart + +To uninstall/delete the my-release deployment: + +```console +helm delete my-release +``` + +The command removes all the Kubernetes components associated with the chart and deletes the release. + +## Upgrading an existing Release to a new major version + +A major chart version change (like v1.2.3 -> v2.0.0) indicates that there is an +incompatible breaking change needing manual actions. + +### To 4.0.0 (And 3.12.1) + +This version requires Helm >= 2.12.0. + +### To 5.0.0 + +You have to add --force to your helm upgrade command as the labels of the chart have changed. + +### To 6.0.0 + +This version requires Helm >= 3.1.0. + +## Configuration + +| Parameter | Description | Default | +|-------------------------------------------|-----------------------------------------------|---------------------------------------------------------| +| `replicas` | Number of nodes | `1` | +| `podDisruptionBudget.minAvailable` | Pod disruption minimum available | `nil` | +| `podDisruptionBudget.maxUnavailable` | Pod disruption maximum unavailable | `nil` | +| `deploymentStrategy` | Deployment strategy | `{ "type": "RollingUpdate" }` | +| `livenessProbe` | Liveness Probe settings | `{ "httpGet": { "path": "/api/health", "port": 3000 } "initialDelaySeconds": 60, "timeoutSeconds": 30, "failureThreshold": 10 }` | +| `readinessProbe` | Readiness Probe settings | `{ "httpGet": { "path": "/api/health", "port": 3000 } }`| +| `securityContext` | Deployment securityContext | `{"runAsUser": 472, "runAsGroup": 472, "fsGroup": 472}` | +| `priorityClassName` | Name of Priority Class to assign pods | `nil` | +| `image.repository` | Image repository | `grafana/grafana` | +| `image.tag` | Overrides the Grafana image tag whose default is the chart appVersion (`Must be >= 5.0.0`) | `` | +| `image.sha` | Image sha (optional) | `` | +| `image.pullPolicy` | Image pull policy | `IfNotPresent` | +| `image.pullSecrets` | Image pull secrets (can be templated) | `[]` | +| `service.enabled` | Enable grafana service | `true` | +| `service.type` | Kubernetes service type | `ClusterIP` | +| `service.port` | Kubernetes port where service is exposed | `80` | +| `service.portName` | Name of the port on the service | `service` | +| `service.appProtocol` | Adds the appProtocol field to the service | `` | +| `service.targetPort` | Internal service is port | `3000` | +| `service.nodePort` | Kubernetes service nodePort | `nil` | +| `service.annotations` | Service annotations (can be templated) | `{}` | +| `service.labels` | Custom labels | `{}` | +| `service.clusterIP` | internal cluster service IP | `nil` | +| `service.loadBalancerIP` | IP address to assign to load balancer (if supported) | `nil` | +| `service.loadBalancerSourceRanges` | list of IP CIDRs allowed access to lb (if supported) | `[]` | +| `service.externalIPs` | service external IP addresses | `[]` | +| `headlessService` | Create a headless service | `false` | +| `extraExposePorts` | Additional service ports for sidecar containers| `[]` | +| `hostAliases` | adds rules to the pod's /etc/hosts | `[]` | +| `ingress.enabled` | Enables Ingress | `false` | +| `ingress.annotations` | Ingress annotations (values are templated) | `{}` | +| `ingress.labels` | Custom labels | `{}` | +| `ingress.path` | Ingress accepted path | `/` | +| `ingress.pathType` | Ingress type of path | `Prefix` | +| `ingress.hosts` | Ingress accepted hostnames | `["chart-example.local"]` | +| `ingress.extraPaths` | Ingress extra paths to prepend to every host configuration. Useful when configuring [custom actions with AWS ALB Ingress Controller](https://kubernetes-sigs.github.io/aws-alb-ingress-controller/guide/ingress/annotation/#actions). Requires `ingress.hosts` to have one or more host entries. | `[]` | +| `ingress.tls` | Ingress TLS configuration | `[]` | +| `resources` | CPU/Memory resource requests/limits | `{}` | +| `nodeSelector` | Node labels for pod assignment | `{}` | +| `tolerations` | Toleration labels for pod assignment | `[]` | +| `affinity` | Affinity settings for pod assignment | `{}` | +| `extraInitContainers` | Init containers to add to the grafana pod | `{}` | +| `extraContainers` | Sidecar containers to add to the grafana pod | `""` | +| `extraContainerVolumes` | Volumes that can be mounted in sidecar containers | `[]` | +| `extraLabels` | Custom labels for all manifests | `{}` | +| `schedulerName` | Name of the k8s scheduler (other than default) | `nil` | +| `persistence.enabled` | Use persistent volume to store data | `false` | +| `persistence.type` | Type of persistence (`pvc` or `statefulset`) | `pvc` | +| `persistence.size` | Size of persistent volume claim | `10Gi` | +| `persistence.existingClaim` | Use an existing PVC to persist data (can be templated) | `nil` | +| `persistence.storageClassName` | Type of persistent volume claim | `nil` | +| `persistence.accessModes` | Persistence access modes | `[ReadWriteOnce]` | +| `persistence.annotations` | PersistentVolumeClaim annotations | `{}` | +| `persistence.finalizers` | PersistentVolumeClaim finalizers | `[ "kubernetes.io/pvc-protection" ]` | +| `persistence.subPath` | Mount a sub dir of the persistent volume (can be templated) | `nil` | +| `persistence.inMemory.enabled` | If persistence is not enabled, whether to mount the local storage in-memory to improve performance | `false` | +| `persistence.inMemory.sizeLimit` | SizeLimit for the in-memory local storage | `nil` | +| `initChownData.enabled` | If false, don't reset data ownership at startup | true | +| `initChownData.image.repository` | init-chown-data container image repository | `busybox` | +| `initChownData.image.tag` | init-chown-data container image tag | `1.31.1` | +| `initChownData.image.sha` | init-chown-data container image sha (optional)| `""` | +| `initChownData.image.pullPolicy` | init-chown-data container image pull policy | `IfNotPresent` | +| `initChownData.resources` | init-chown-data pod resource requests & limits | `{}` | +| `schedulerName` | Alternate scheduler name | `nil` | +| `env` | Extra environment variables passed to pods | `{}` | +| `envValueFrom` | Environment variables from alternate sources. See the API docs on [EnvVarSource](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#envvarsource-v1-core) for format details. Can be templated | `{}` | +| `envFromSecret` | Name of a Kubernetes secret (must be manually created in the same namespace) containing values to be added to the environment. Can be templated | `""` | +| `envFromSecrets` | List of Kubernetes secrets (must be manually created in the same namespace) containing values to be added to the environment. Can be templated | `[]` | +| `envFromConfigMaps` | List of Kubernetes ConfigMaps (must be manually created in the same namespace) containing values to be added to the environment. Can be templated | `[]` | +| `envRenderSecret` | Sensible environment variables passed to pods and stored as secret | `{}` | +| `enableServiceLinks` | Inject Kubernetes services as environment variables. | `true` | +| `extraSecretMounts` | Additional grafana server secret mounts | `[]` | +| `extraVolumeMounts` | Additional grafana server volume mounts | `[]` | +| `createConfigmap` | Enable creating the grafana configmap | `true` | +| `extraConfigmapMounts` | Additional grafana server configMap volume mounts (values are templated) | `[]` | +| `extraEmptyDirMounts` | Additional grafana server emptyDir volume mounts | `[]` | +| `plugins` | Plugins to be loaded along with Grafana | `[]` | +| `datasources` | Configure grafana datasources (passed through tpl) | `{}` | +| `alerting` | Configure grafana alerting (passed through tpl) | `{}` | +| `notifiers` | Configure grafana notifiers | `{}` | +| `dashboardProviders` | Configure grafana dashboard providers | `{}` | +| `dashboards` | Dashboards to import | `{}` | +| `dashboardsConfigMaps` | ConfigMaps reference that contains dashboards | `{}` | +| `grafana.ini` | Grafana's primary configuration | `{}` | +| `ldap.enabled` | Enable LDAP authentication | `false` | +| `ldap.existingSecret` | The name of an existing secret containing the `ldap.toml` file, this must have the key `ldap-toml`. | `""` | +| `ldap.config` | Grafana's LDAP configuration | `""` | +| `annotations` | Deployment annotations | `{}` | +| `labels` | Deployment labels | `{}` | +| `podAnnotations` | Pod annotations | `{}` | +| `podLabels` | Pod labels | `{}` | +| `podPortName` | Name of the grafana port on the pod | `grafana` | +| `lifecycleHooks` | Lifecycle hooks for podStart and preStop [Example](https://kubernetes.io/docs/tasks/configure-pod-container/attach-handler-lifecycle-event/#define-poststart-and-prestop-handlers) | `{}` | +| `sidecar.image.repository` | Sidecar image repository | `quay.io/kiwigrid/k8s-sidecar` | +| `sidecar.image.tag` | Sidecar image tag | `1.19.2` | +| `sidecar.image.sha` | Sidecar image sha (optional) | `""` | +| `sidecar.imagePullPolicy` | Sidecar image pull policy | `IfNotPresent` | +| `sidecar.resources` | Sidecar resources | `{}` | +| `sidecar.securityContext` | Sidecar securityContext | `{}` | +| `sidecar.enableUniqueFilenames` | Sets the kiwigrid/k8s-sidecar UNIQUE_FILENAMES environment variable. If set to `true` the sidecar will create unique filenames where duplicate data keys exist between ConfigMaps and/or Secrets within the same or multiple Namespaces. | `false` | +| `sidecar.dashboards.enabled` | Enables the cluster wide search for dashboards and adds/updates/deletes them in grafana | `false` | +| `sidecar.dashboards.SCProvider` | Enables creation of sidecar provider | `true` | +| `sidecar.dashboards.provider.name` | Unique name of the grafana provider | `sidecarProvider` | +| `sidecar.dashboards.provider.orgid` | Id of the organisation, to which the dashboards should be added | `1` | +| `sidecar.dashboards.provider.folder` | Logical folder in which grafana groups dashboards | `""` | +| `sidecar.dashboards.provider.disableDelete` | Activate to avoid the deletion of imported dashboards | `false` | +| `sidecar.dashboards.provider.allowUiUpdates` | Allow updating provisioned dashboards from the UI | `false` | +| `sidecar.dashboards.provider.type` | Provider type | `file` | +| `sidecar.dashboards.provider.foldersFromFilesStructure` | Allow Grafana to replicate dashboard structure from filesystem. | `false` | +| `sidecar.dashboards.watchMethod` | Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. | `WATCH` | +| `sidecar.skipTlsVerify` | Set to true to skip tls verification for kube api calls | `nil` | +| `sidecar.dashboards.label` | Label that config maps with dashboards should have to be added | `grafana_dashboard` | +| `sidecar.dashboards.labelValue` | Label value that config maps with dashboards should have to be added | `""` | +| `sidecar.dashboards.folder` | Folder in the pod that should hold the collected dashboards (unless `sidecar.dashboards.defaultFolderName` is set). This path will be mounted. | `/tmp/dashboards` | +| `sidecar.dashboards.folderAnnotation` | The annotation the sidecar will look for in configmaps to override the destination folder for files | `nil` | +| `sidecar.dashboards.defaultFolderName` | The default folder name, it will create a subfolder under the `sidecar.dashboards.folder` and put dashboards in there instead | `nil` | +| `sidecar.dashboards.searchNamespace` | Namespaces list. If specified, the sidecar will search for dashboards config-maps inside these namespaces.Otherwise the namespace in which the sidecar is running will be used.It's also possible to specify ALL to search in all namespaces. | `nil` | +| `sidecar.dashboards.script` | Absolute path to shell script to execute after a configmap got reloaded. | `nil` | +| `sidecar.dashboards.resource` | Should the sidecar looks into secrets, configmaps or both. | `both` | +| `sidecar.dashboards.extraMounts` | Additional dashboard sidecar volume mounts. | `[]` | +| `sidecar.datasources.enabled` | Enables the cluster wide search for datasources and adds/updates/deletes them in grafana |`false` | +| `sidecar.datasources.label` | Label that config maps with datasources should have to be added | `grafana_datasource` | +| `sidecar.datasources.labelValue` | Label value that config maps with datasources should have to be added | `""` | +| `sidecar.datasources.searchNamespace` | Namespaces list. If specified, the sidecar will search for datasources config-maps inside these namespaces.Otherwise the namespace in which the sidecar is running will be used.It's also possible to specify ALL to search in all namespaces. | `nil` | +| `sidecar.datasources.resource` | Should the sidecar looks into secrets, configmaps or both. | `both` | +| `sidecar.datasources.reloadURL` | Full url of datasource configuration reload API endpoint, to invoke after a config-map change | `"http://localhost:3000/api/admin/provisioning/datasources/reload"` | +| `sidecar.datasources.skipReload` | Enabling this omits defining the REQ_URL and REQ_METHOD environment variables | `false` | +| `sidecar.notifiers.enabled` | Enables the cluster wide search for notifiers and adds/updates/deletes them in grafana | `false` | +| `sidecar.notifiers.label` | Label that config maps with notifiers should have to be added | `grafana_notifier` | +| `sidecar.notifiers.searchNamespace` | Namespaces list. If specified, the sidecar will search for notifiers config-maps (or secrets) inside these namespaces.Otherwise the namespace in which the sidecar is running will be used.It's also possible to specify ALL to search in all namespaces. | `nil` | +| `sidecar.notifiers.resource` | Should the sidecar looks into secrets, configmaps or both. | `both` | +| `smtp.existingSecret` | The name of an existing secret containing the SMTP credentials. | `""` | +| `smtp.userKey` | The key in the existing SMTP secret containing the username. | `"user"` | +| `smtp.passwordKey` | The key in the existing SMTP secret containing the password. | `"password"` | +| `admin.existingSecret` | The name of an existing secret containing the admin credentials (can be templated). | `""` | +| `admin.userKey` | The key in the existing admin secret containing the username. | `"admin-user"` | +| `admin.passwordKey` | The key in the existing admin secret containing the password. | `"admin-password"` | +| `serviceAccount.autoMount` | Automount the service account token in the pod| `true` | +| `serviceAccount.annotations` | ServiceAccount annotations | | +| `serviceAccount.create` | Create service account | `true` | +| `serviceAccount.name` | Service account name to use, when empty will be set to created account if `serviceAccount.create` is set else to `default` | `` | +| `serviceAccount.nameTest` | Service account name to use for test, when empty will be set to created account if `serviceAccount.create` is set else to `default` | `nil` | +| `rbac.create` | Create and use RBAC resources | `true` | +| `rbac.namespaced` | Creates Role and Rolebinding instead of the default ClusterRole and ClusteRoleBindings for the grafana instance | `false` | +| `rbac.useExistingRole` | Set to a rolename to use existing role - skipping role creating - but still doing serviceaccount and rolebinding to the rolename set here. | `nil` | +| `rbac.pspEnabled` | Create PodSecurityPolicy (with `rbac.create`, grant roles permissions as well) | `true` | +| `rbac.pspUseAppArmor` | Enforce AppArmor in created PodSecurityPolicy (requires `rbac.pspEnabled`) | `true` | +| `rbac.extraRoleRules` | Additional rules to add to the Role | [] | +| `rbac.extraClusterRoleRules` | Additional rules to add to the ClusterRole | [] | +| `command` | Define command to be executed by grafana container at startup | `nil` | +| `testFramework.enabled` | Whether to create test-related resources | `true` | +| `testFramework.image` | `test-framework` image repository. | `bats/bats` | +| `testFramework.tag` | `test-framework` image tag. | `v1.4.1` | +| `testFramework.imagePullPolicy` | `test-framework` image pull policy. | `IfNotPresent` | +| `testFramework.securityContext` | `test-framework` securityContext | `{}` | +| `downloadDashboards.env` | Environment variables to be passed to the `download-dashboards` container | `{}` | +| `downloadDashboards.envFromSecret` | Name of a Kubernetes secret (must be manually created in the same namespace) containing values to be added to the environment. Can be templated | `""` | +| `downloadDashboards.resources` | Resources of `download-dashboards` container | `{}` | +| `downloadDashboardsImage.repository` | Curl docker image repo | `curlimages/curl` | +| `downloadDashboardsImage.tag` | Curl docker image tag | `7.73.0` | +| `downloadDashboardsImage.sha` | Curl docker image sha (optional) | `""` | +| `downloadDashboardsImage.pullPolicy` | Curl docker image pull policy | `IfNotPresent` | +| `namespaceOverride` | Override the deployment namespace | `""` (`Release.Namespace`) | +| `serviceMonitor.enabled` | Use servicemonitor from prometheus operator | `false` | +| `serviceMonitor.namespace` | Namespace this servicemonitor is installed in | | +| `serviceMonitor.interval` | How frequently Prometheus should scrape | `1m` | +| `serviceMonitor.path` | Path to scrape | `/metrics` | +| `serviceMonitor.scheme` | Scheme to use for metrics scraping | `http` | +| `serviceMonitor.tlsConfig` | TLS configuration block for the endpoint | `{}` | +| `serviceMonitor.labels` | Labels for the servicemonitor passed to Prometheus Operator | `{}` | +| `serviceMonitor.scrapeTimeout` | Timeout after which the scrape is ended | `30s` | +| `serviceMonitor.relabelings` | MetricRelabelConfigs to apply to samples before ingestion. | `[]` | +| `revisionHistoryLimit` | Number of old ReplicaSets to retain | `10` | +| `imageRenderer.enabled` | Enable the image-renderer deployment & service | `false` | +| `imageRenderer.image.repository` | image-renderer Image repository | `grafana/grafana-image-renderer` | +| `imageRenderer.image.tag` | image-renderer Image tag | `latest` | +| `imageRenderer.image.sha` | image-renderer Image sha (optional) | `""` | +| `imageRenderer.image.pullPolicy` | image-renderer ImagePullPolicy | `Always` | +| `imageRenderer.env` | extra env-vars for image-renderer | `{}` | +| `imageRenderer.serviceAccountName` | image-renderer deployment serviceAccountName | `""` | +| `imageRenderer.securityContext` | image-renderer deployment securityContext | `{}` | +| `imageRenderer.hostAliases` | image-renderer deployment Host Aliases | `[]` | +| `imageRenderer.priorityClassName` | image-renderer deployment priority class | `''` | +| `imageRenderer.service.enabled` | Enable the image-renderer service | `true` | +| `imageRenderer.service.portName` | image-renderer service port name | `http` | +| `imageRenderer.service.port` | image-renderer port used by deployment | `8081` | +| `imageRenderer.service.targetPort` | image-renderer service port used by service | `8081` | +| `imageRenderer.appProtocol` | Adds the appProtocol field to the service | `` | +| `imageRenderer.grafanaSubPath` | Grafana sub path to use for image renderer callback url | `''` | +| `imageRenderer.podPortName` | name of the image-renderer port on the pod | `http` | +| `imageRenderer.revisionHistoryLimit` | number of image-renderer replica sets to keep | `10` | +| `imageRenderer.networkPolicy.limitIngress` | Enable a NetworkPolicy to limit inbound traffic from only the created grafana pods | `true` | +| `imageRenderer.networkPolicy.limitEgress` | Enable a NetworkPolicy to limit outbound traffic to only the created grafana pods | `false` | +| `imageRenderer.resources` | Set resource limits for image-renderer pdos | `{}` | +| `imageRenderer.nodeSelector` | Node labels for pod assignment | `{}` | +| `imageRenderer.tolerations` | Toleration labels for pod assignment | `[]` | +| `imageRenderer.affinity` | Affinity settings for pod assignment | `{}` | +| `networkPolicy.enabled` | Enable creation of NetworkPolicy resources. | `false` | +| `networkPolicy.allowExternal` | Don't require client label for connections | `true` | +| `networkPolicy.explicitNamespacesSelector` | A Kubernetes LabelSelector to explicitly select namespaces from which traffic could be allowed | `{}` | +| `networkPolicy.ingress` | Enable the creation of an ingress network policy | `true` | +| `networkPolicy.egress.enabled` | Enable the creation of an egress network policy | `false` | +| `networkPolicy.egress.ports` | An array of ports to allow for the egress | `[]` | +| `enableKubeBackwardCompatibility` | Enable backward compatibility of kubernetes where pod's defintion version below 1.13 doesn't have the enableServiceLinks option | `false` | + + + +### Example ingress with path + +With grafana 6.3 and above +```yaml +grafana.ini: + server: + domain: monitoring.example.com + root_url: "%(protocol)s://%(domain)s/grafana" + serve_from_sub_path: true +ingress: + enabled: true + hosts: + - "monitoring.example.com" + path: "/grafana" +``` + +### Example of extraVolumeMounts + +Volume can be type persistentVolumeClaim or hostPath but not both at same time. +If neither existingClaim or hostPath argument is given then type is emptyDir. + +```yaml +- extraVolumeMounts: + - name: plugins + mountPath: /var/lib/grafana/plugins + subPath: configs/grafana/plugins + existingClaim: existing-grafana-claim + readOnly: false + - name: dashboards + mountPath: /var/lib/grafana/dashboards + hostPath: /usr/shared/grafana/dashboards + readOnly: false +``` + +## Import dashboards + +There are a few methods to import dashboards to Grafana. Below are some examples and explanations as to how to use each method: + +```yaml +dashboards: + default: + some-dashboard: + json: | + { + "annotations": + + ... + # Complete json file here + ... + + "title": "Some Dashboard", + "uid": "abcd1234", + "version": 1 + } + custom-dashboard: + # This is a path to a file inside the dashboards directory inside the chart directory + file: dashboards/custom-dashboard.json + prometheus-stats: + # Ref: https://grafana.com/dashboards/2 + gnetId: 2 + revision: 2 + datasource: Prometheus + local-dashboard: + url: https://raw.githubusercontent.com/user/repository/master/dashboards/dashboard.json +``` + +## BASE64 dashboards + +Dashboards could be stored on a server that does not return JSON directly and instead of it returns a Base64 encoded file (e.g. Gerrit) +A new parameter has been added to the url use case so if you specify a b64content value equals to true after the url entry a Base64 decoding is applied before save the file to disk. +If this entry is not set or is equals to false not decoding is applied to the file before saving it to disk. + +### Gerrit use case + +Gerrit API for download files has the following schema: where {project-name} and +{file-id} usually has '/' in their values and so they MUST be replaced by %2F so if project-name is user/repo, branch-id is master and file-id is equals to dir1/dir2/dashboard +the url value is + +## Sidecar for dashboards + +If the parameter `sidecar.dashboards.enabled` is set, a sidecar container is deployed in the grafana +pod. This container watches all configmaps (or secrets) in the cluster and filters out the ones with +a label as defined in `sidecar.dashboards.label`. The files defined in those configmaps are written +to a folder and accessed by grafana. Changes to the configmaps are monitored and the imported +dashboards are deleted/updated. + +A recommendation is to use one configmap per dashboard, as a reduction of multiple dashboards inside +one configmap is currently not properly mirrored in grafana. + +Example dashboard config: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: sample-grafana-dashboard + labels: + grafana_dashboard: "1" +data: + k8s-dashboard.json: |- + [...] +``` + +## Sidecar for datasources + +If the parameter `sidecar.datasources.enabled` is set, an init container is deployed in the grafana +pod. This container lists all secrets (or configmaps, though not recommended) in the cluster and +filters out the ones with a label as defined in `sidecar.datasources.label`. The files defined in +those secrets are written to a folder and accessed by grafana on startup. Using these yaml files, +the data sources in grafana can be imported. + +Secrets are recommended over configmaps for this usecase because datasources usually contain private +data like usernames and passwords. Secrets are the more appropriate cluster resource to manage those. + +Example values to add a datasource adapted from [Grafana](http://docs.grafana.org/administration/provisioning/#example-datasource-config-file): + +```yaml +datasources: + datasources.yaml: + apiVersion: 1 + datasources: + # name of the datasource. Required + - name: Graphite + # datasource type. Required + type: graphite + # access mode. proxy or direct (Server or Browser in the UI). Required + access: proxy + # org id. will default to orgId 1 if not specified + orgId: 1 + # url + url: http://localhost:8080 + # database password, if used + password: + # database user, if used + user: + # database name, if used + database: + # enable/disable basic auth + basicAuth: + # basic auth username + basicAuthUser: + # basic auth password + basicAuthPassword: + # enable/disable with credentials headers + withCredentials: + # mark as default datasource. Max one per org + isDefault: + # fields that will be converted to json and stored in json_data + jsonData: + graphiteVersion: "1.1" + tlsAuth: true + tlsAuthWithCACert: true + # json object of data that will be encrypted. + secureJsonData: + tlsCACert: "..." + tlsClientCert: "..." + tlsClientKey: "..." + version: 1 + # allow users to edit datasources from the UI. + editable: false +``` + +## Sidecar for notifiers + +If the parameter `sidecar.notifiers.enabled` is set, an init container is deployed in the grafana +pod. This container lists all secrets (or configmaps, though not recommended) in the cluster and +filters out the ones with a label as defined in `sidecar.notifiers.label`. The files defined in +those secrets are written to a folder and accessed by grafana on startup. Using these yaml files, +the notification channels in grafana can be imported. The secrets must be created before +`helm install` so that the notifiers init container can list the secrets. + +Secrets are recommended over configmaps for this usecase because alert notification channels usually contain +private data like SMTP usernames and passwords. Secrets are the more appropriate cluster resource to manage those. + +Example datasource config adapted from [Grafana](https://grafana.com/docs/grafana/latest/administration/provisioning/#alert-notification-channels): + +```yaml +notifiers: + - name: notification-channel-1 + type: slack + uid: notifier1 + # either + org_id: 2 + # or + org_name: Main Org. + is_default: true + send_reminder: true + frequency: 1h + disable_resolve_message: false + # See `Supported Settings` section for settings supporter for each + # alert notification type. + settings: + recipient: 'XXX' + token: 'xoxb' + uploadImage: true + url: https://slack.com + +delete_notifiers: + - name: notification-channel-1 + uid: notifier1 + org_id: 2 + - name: notification-channel-2 + # default org_id: 1 +``` + +## How to serve Grafana with a path prefix (/grafana) + +In order to serve Grafana with a prefix (e.g., ), add the following to your values.yaml. + +```yaml +ingress: + enabled: true + annotations: + kubernetes.io/ingress.class: "nginx" + nginx.ingress.kubernetes.io/rewrite-target: /$1 + nginx.ingress.kubernetes.io/use-regex: "true" + + path: /grafana/?(.*) + hosts: + - k8s.example.dev + +grafana.ini: + server: + root_url: http://localhost:3000/grafana # this host can be localhost +``` + +## How to securely reference secrets in grafana.ini + +This example uses Grafana [file providers](https://grafana.com/docs/grafana/latest/administration/configuration/#file-provider) for secret values and the `extraSecretMounts` configuration flag (Additional grafana server secret mounts) to mount the secrets. + +In grafana.ini: + +```yaml +grafana.ini: + [auth.generic_oauth] + enabled = true + client_id = $__file{/etc/secrets/auth_generic_oauth/client_id} + client_secret = $__file{/etc/secrets/auth_generic_oauth/client_secret} +``` + +Existing secret, or created along with helm: + +```yaml +--- +apiVersion: v1 +kind: Secret +metadata: + name: auth-generic-oauth-secret +type: Opaque +stringData: + client_id: + client_secret: +``` + +Include in the `extraSecretMounts` configuration flag: + +```yaml +- extraSecretMounts: + - name: auth-generic-oauth-secret-mount + secretName: auth-generic-oauth-secret + defaultMode: 0440 + mountPath: /etc/secrets/auth_generic_oauth + readOnly: true +``` + +### extraSecretMounts using a Container Storage Interface (CSI) provider + +This example uses a CSI driver e.g. retrieving secrets using [Azure Key Vault Provider](https://github.com/Azure/secrets-store-csi-driver-provider-azure) + +```yaml +- extraSecretMounts: + - name: secrets-store-inline + mountPath: /run/secrets + readOnly: true + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: "my-provider" + nodePublishSecretRef: + name: akv-creds +``` + +## Image Renderer Plug-In + +This chart supports enabling [remote image rendering](https://github.com/grafana/grafana-image-renderer/blob/master/README.md#run-in-docker) + +```yaml +imageRenderer: + enabled: true +``` + +### Image Renderer NetworkPolicy + +By default the image-renderer pods will have a network policy which only allows ingress traffic from the created grafana instance + +### High Availability for unified alerting + +If you want to run Grafana in a high availability cluster you need to enable +the headless service by setting `headlessService: true` in your `values.yaml` +file. + +As next step you have to setup the `grafana.ini` in your `values.yaml` in a way +that it will make use of the headless service to obtain all the IPs of the +cluster. You should replace ``{{ Name }}`` with the name of your helm deployment. + +```yaml +grafana.ini: + ... + unified_alerting: + enabled: true + ha_peers: {{ Name }}-headless:9094 + alerting: + enabled: false +``` diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/dashboards/custom-dashboard.json b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/dashboards/custom-dashboard.json new file mode 100644 index 0000000000..9e26dfeeb6 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/dashboards/custom-dashboard.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/NOTES.txt b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/NOTES.txt new file mode 100644 index 0000000000..1fc8436d95 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/NOTES.txt @@ -0,0 +1,54 @@ +1. Get your '{{ .Values.adminUser }}' user password by running: + + kubectl get secret --namespace {{ template "grafana.namespace" . }} {{ template "grafana.fullname" . }} -o jsonpath="{.data.admin-password}" | base64 --decode ; echo + +2. The Grafana server can be accessed via port {{ .Values.service.port }} on the following DNS name from within your cluster: + + {{ template "grafana.fullname" . }}.{{ template "grafana.namespace" . }}.svc.cluster.local +{{ if .Values.ingress.enabled }} + If you bind grafana to 80, please update values in values.yaml and reinstall: + ``` + securityContext: + runAsUser: 0 + runAsGroup: 0 + fsGroup: 0 + + command: + - "setcap" + - "'cap_net_bind_service=+ep'" + - "/usr/sbin/grafana-server &&" + - "sh" + - "/run.sh" + ``` + Details refer to https://grafana.com/docs/installation/configuration/#http-port. + Or grafana would always crash. + + From outside the cluster, the server URL(s) are: +{{- range .Values.ingress.hosts }} + http://{{ . }} +{{- end }} +{{ else }} + Get the Grafana URL to visit by running these commands in the same shell: +{{ if contains "NodePort" .Values.service.type -}} + export NODE_PORT=$(kubectl get --namespace {{ template "grafana.namespace" . }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ template "grafana.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ template "grafana.namespace" . }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{ else if contains "LoadBalancer" .Values.service.type -}} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get svc --namespace {{ template "grafana.namespace" . }} -w {{ template "grafana.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ template "grafana.namespace" . }} {{ template "grafana.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + http://$SERVICE_IP:{{ .Values.service.port -}} +{{ else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ template "grafana.namespace" . }} -l "app.kubernetes.io/name={{ template "grafana.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + kubectl --namespace {{ template "grafana.namespace" . }} port-forward $POD_NAME 3000 +{{- end }} +{{- end }} + +3. Login with the password from step 1 and the username: {{ .Values.adminUser }} + +{{- if not .Values.persistence.enabled }} +################################################################################# +###### WARNING: Persistence is disabled!!! You will lose your data when ##### +###### the Grafana pod is terminated. ##### +################################################################################# +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/_helpers.tpl b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/_helpers.tpl new file mode 100644 index 0000000000..e5e3b287dd --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/_helpers.tpl @@ -0,0 +1,214 @@ +# Rancher +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "grafana.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "grafana.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "grafana.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create the name of the service account +*/}} +{{- define "grafana.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "grafana.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{- define "grafana.serviceAccountNameTest" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (print (include "grafana.fullname" .) "-test") .Values.serviceAccount.nameTest }} +{{- else -}} + {{ default "default" .Values.serviceAccount.nameTest }} +{{- end -}} +{{- end -}} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "grafana.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "grafana.labels" -}} +helm.sh/chart: {{ include "grafana.chart" . }} +{{ include "grafana.selectorLabels" . }} +{{- if or .Chart.AppVersion .Values.image.tag }} +app.kubernetes.io/version: {{ .Values.image.tag | default .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- if .Values.extraLabels }} +{{ toYaml .Values.extraLabels }} +{{- end }} +{{- end -}} + +{{/* +Selector labels +*/}} +{{- define "grafana.selectorLabels" -}} +app.kubernetes.io/name: {{ include "grafana.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "grafana.imageRenderer.labels" -}} +helm.sh/chart: {{ include "grafana.chart" . }} +{{ include "grafana.imageRenderer.selectorLabels" . }} +{{- if or .Chart.AppVersion .Values.image.tag }} +app.kubernetes.io/version: {{ .Values.image.tag | default .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{/* +Selector labels ImageRenderer +*/}} +{{- define "grafana.imageRenderer.selectorLabels" -}} +app.kubernetes.io/name: {{ include "grafana.name" . }}-image-renderer +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} + +{{/* +Looks if there's an existing secret and reuse its password. If not it generates +new password and use it. +*/}} +{{- define "grafana.password" -}} +{{- $secret := (lookup "v1" "Secret" (include "grafana.namespace" .) (include "grafana.fullname" .) ) -}} + {{- if $secret -}} + {{- index $secret "data" "admin-password" -}} + {{- else -}} + {{- (randAlphaNum 40) | b64enc | quote -}} + {{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for rbac. +*/}} +{{- define "grafana.rbac.apiVersion" -}} + {{- if .Capabilities.APIVersions.Has "rbac.authorization.k8s.io/v1" }} + {{- print "rbac.authorization.k8s.io/v1" -}} + {{- else -}} + {{- print "rbac.authorization.k8s.io/v1beta1" -}} + {{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for HorizontalPodAutoscaler. +*/}} +{{- define "grafana.hpa.apiVersion" -}} + {{- if .Capabilities.APIVersions.Has "autoscaling/v2" }} + {{- print "autoscaling/v2" -}} + {{- else if .Capabilities.APIVersions.Has "autoscaling/v1" }} + {{- print "autoscaling/v1" -}} + {{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for ingress. +*/}} +{{- define "grafana.ingress.apiVersion" -}} + {{- if and (.Capabilities.APIVersions.Has "networking.k8s.io/v1") (semverCompare ">= 1.19-0" .Capabilities.KubeVersion.Version) -}} + {{- print "networking.k8s.io/v1" -}} + {{- else if .Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" -}} + {{- print "networking.k8s.io/v1beta1" -}} + {{- else -}} + {{- print "extensions/v1beta1" -}} + {{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for podDisruptionBudget. +*/}} +{{- define "grafana.podDisruptionBudget.apiVersion" -}} + {{- if $.Capabilities.APIVersions.Has "policy/v1/PodDisruptionBudget" -}} + {{- print "policy/v1" -}} + {{- else -}} + {{- print "policy/v1beta1" -}} + {{- end -}} +{{- end -}} + +{{/* +Return if ingress is stable. +*/}} +{{- define "grafana.ingress.isStable" -}} + {{- eq (include "grafana.ingress.apiVersion" .) "networking.k8s.io/v1" -}} +{{- end -}} + +{{/* +Return if ingress supports ingressClassName. +*/}} +{{- define "grafana.ingress.supportsIngressClassName" -}} + {{- or (eq (include "grafana.ingress.isStable" .) "true") (and (eq (include "grafana.ingress.apiVersion" .) "networking.k8s.io/v1beta1") (semverCompare ">= 1.18-0" .Capabilities.KubeVersion.Version)) -}} +{{- end -}} + +{{/* +Return if ingress supports pathType. +*/}} +{{- define "grafana.ingress.supportsPathType" -}} + {{- or (eq (include "grafana.ingress.isStable" .) "true") (and (eq (include "grafana.ingress.apiVersion" .) "networking.k8s.io/v1beta1") (semverCompare ">= 1.18-0" .Capabilities.KubeVersion.Version)) -}} +{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/_pod.tpl b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/_pod.tpl new file mode 100644 index 0000000000..b74d03181a --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/_pod.tpl @@ -0,0 +1,895 @@ +{{- define "grafana.pod" -}} +{{- if .Values.schedulerName }} +schedulerName: "{{ .Values.schedulerName }}" +{{- end }} +serviceAccountName: {{ template "grafana.serviceAccountName" . }} +automountServiceAccountToken: {{ .Values.serviceAccount.autoMount }} +{{- with .Values.securityContext }} +securityContext: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- with .Values.hostAliases }} +hostAliases: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- if .Values.priorityClassName }} +priorityClassName: {{ .Values.priorityClassName }} +{{- end }} +{{- if ( or .Values.persistence.enabled .Values.dashboards .Values.sidecar.notifiers.enabled .Values.extraInitContainers (and .Values.sidecar.datasources.enabled .Values.sidecar.datasources.initDatasources)) }} +initContainers: +{{- end }} +{{- if ( and .Values.persistence.enabled .Values.initChownData.enabled ) }} + - name: init-chown-data + {{- if .Values.initChownData.image.sha }} + image: "{{ template "system_default_registry" . }}{{ .Values.initChownData.image.repository }}:{{ .Values.initChownData.image.tag }}@sha256:{{ .Values.initChownData.image.sha }}" + {{- else }} + image: "{{ template "system_default_registry" . }}{{ .Values.initChownData.image.repository }}:{{ .Values.initChownData.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.initChownData.image.pullPolicy }} + securityContext: + runAsNonRoot: false + runAsUser: 0 + command: ["chown", "-R", "{{ .Values.securityContext.runAsUser }}:{{ .Values.securityContext.runAsGroup }}", "/var/lib/grafana"] + {{- with .Values.initChownData.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: storage + mountPath: "/var/lib/grafana" +{{- if .Values.persistence.subPath }} + subPath: {{ tpl .Values.persistence.subPath . }} +{{- end }} +{{- end }} +{{- if .Values.dashboards }} + - name: download-dashboards + {{- if .Values.downloadDashboardsImage.sha }} + image: "{{ template "system_default_registry" . }}{{ .Values.downloadDashboardsImage.repository }}:{{ .Values.downloadDashboardsImage.tag }}@sha256:{{ .Values.downloadDashboardsImage.sha }}" + {{- else }} + image: "{{ template "system_default_registry" . }}{{ .Values.downloadDashboardsImage.repository }}:{{ .Values.downloadDashboardsImage.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.downloadDashboardsImage.pullPolicy }} + command: ["/bin/sh"] + args: [ "-c", "mkdir -p /var/lib/grafana/dashboards/default && /bin/sh -x /etc/grafana/download_dashboards.sh" ] + {{- with .Values.downloadDashboards.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + env: +{{- range $key, $value := .Values.downloadDashboards.env }} + - name: "{{ $key }}" + value: "{{ $value }}" +{{- end }} + {{- with .Values.downloadDashboards.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} +{{- if .Values.downloadDashboards.envFromSecret }} + envFrom: + - secretRef: + name: {{ tpl .Values.downloadDashboards.envFromSecret . }} +{{- end }} + volumeMounts: + - name: config + mountPath: "/etc/grafana/download_dashboards.sh" + subPath: download_dashboards.sh + - name: storage + mountPath: "/var/lib/grafana" +{{- if .Values.persistence.subPath }} + subPath: {{ tpl .Values.persistence.subPath . }} +{{- end }} + {{- range .Values.extraSecretMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + readOnly: {{ .readOnly }} + {{- end }} +{{- end }} +{{- if and .Values.sidecar.datasources.enabled .Values.sidecar.datasources.initDatasources }} + - name: {{ template "grafana.name" . }}-init-sc-datasources + {{- if .Values.sidecar.image.sha }} + image: "{{ template "system_default_registry" . }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ template "system_default_registry" . }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.datasources.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.datasources.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: "LIST" + - name: LABEL + value: "{{ .Values.sidecar.datasources.label }}" + {{- if .Values.sidecar.datasources.labelValue }} + - name: LABEL_VALUE + value: {{ quote .Values.sidecar.datasources.labelValue }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.datasources.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.datasources.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/datasources" + - name: RESOURCE + value: {{ quote .Values.sidecar.datasources.resource }} + {{- if .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ .Values.sidecar.enableUniqueFilenames }}" + {{- end }} + {{- if .Values.sidecar.datasources.searchNamespace }} + - name: NAMESPACE + value: "{{ tpl (.Values.sidecar.datasources.searchNamespace | join ",") . }}" + {{- end }} + {{- if .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ .Values.sidecar.skipTlsVerify }}" + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-datasources-volume + mountPath: "/etc/grafana/provisioning/datasources" +{{- end }} +{{- if .Values.sidecar.notifiers.enabled }} + - name: {{ template "grafana.name" . }}-sc-notifiers + {{- if .Values.sidecar.image.sha }} + image: "{{ template "system_default_registry" . }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ template "system_default_registry" . }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.notifiers.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.notifiers.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: LIST + - name: LABEL + value: "{{ .Values.sidecar.notifiers.label }}" + {{- if .Values.sidecar.notifiers.labelValue }} + - name: LABEL_VALUE + value: {{ quote .Values.sidecar.notifiers.labelValue }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.notifiers.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.notifiers.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/notifiers" + - name: RESOURCE + value: {{ quote .Values.sidecar.notifiers.resource }} + {{- if .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ .Values.sidecar.enableUniqueFilenames }}" + {{- end }} + {{- if .Values.sidecar.notifiers.searchNamespace }} + - name: NAMESPACE + value: "{{ tpl (.Values.sidecar.notifiers.searchNamespace | join ",") . }}" + {{- end }} + {{- if .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ .Values.sidecar.skipTlsVerify }}" + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-notifiers-volume + mountPath: "/etc/grafana/provisioning/notifiers" +{{- end}} +{{- if .Values.extraInitContainers }} +{{ tpl (toYaml .Values.extraInitContainers) . | indent 2 }} +{{- end }} +{{- if .Values.image.pullSecrets }} +imagePullSecrets: +{{- $root := . }} +{{- range .Values.image.pullSecrets }} + - name: {{ tpl . $root }} +{{- end}} +{{- end }} +{{- if not .Values.enableKubeBackwardCompatibility }} +enableServiceLinks: {{ .Values.enableServiceLinks }} +{{- end }} +containers: +{{- if .Values.sidecar.dashboards.enabled }} + - name: {{ template "grafana.name" . }}-sc-dashboard + {{- if .Values.sidecar.image.sha }} + image: "{{ template "system_default_registry" . }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ template "system_default_registry" . }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.dashboards.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.dashboards.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: {{ .Values.sidecar.dashboards.watchMethod }} + - name: LABEL + value: "{{ .Values.sidecar.dashboards.label }}" + {{- if .Values.sidecar.dashboards.labelValue }} + - name: LABEL_VALUE + value: {{ quote .Values.sidecar.dashboards.labelValue }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.dashboards.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.dashboards.logLevel }} + {{- end }} + - name: FOLDER + value: "{{ .Values.sidecar.dashboards.folder }}{{- with .Values.sidecar.dashboards.defaultFolderName }}/{{ . }}{{- end }}" + - name: RESOURCE + value: {{ quote .Values.sidecar.dashboards.resource }} + {{- if .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ .Values.sidecar.enableUniqueFilenames }}" + {{- end }} + {{- if .Values.sidecar.dashboards.searchNamespace }} + - name: NAMESPACE + value: "{{ tpl (.Values.sidecar.dashboards.searchNamespace | join ",") . }}" + {{- end }} + {{- if .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ .Values.sidecar.skipTlsVerify }}" + {{- end }} + {{- if .Values.sidecar.dashboards.folderAnnotation }} + - name: FOLDER_ANNOTATION + value: "{{ .Values.sidecar.dashboards.folderAnnotation }}" + {{- end }} + {{- if .Values.sidecar.dashboards.script }} + - name: SCRIPT + value: "{{ .Values.sidecar.dashboards.script }}" + {{- end }} + {{- if .Values.sidecar.dashboards.watchServerTimeout }} + {{- if ne .Values.sidecar.dashboards.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.dashboards.watchServerTimeout with .Values.sidecar.dashboards.watchMethod %s" .Values.sidecar.dashboards.watchMethod) }} + {{- end }} + - name: WATCH_SERVER_TIMEOUT + value: "{{ .Values.sidecar.dashboards.watchServerTimeout }}" + {{- end }} + {{- if .Values.sidecar.dashboards.watchClientTimeout }} + {{- if ne .Values.sidecar.dashboards.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.dashboards.watchClientTimeout with .Values.sidecar.dashboards.watchMethod %s" .Values.sidecar.dashboards.watchMethod) }} + {{- end }} + - name: WATCH_CLIENT_TIMEOUT + value: "{{ .Values.sidecar.dashboards.watchClientTimeout }}" + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-dashboard-volume + mountPath: {{ .Values.sidecar.dashboards.folder | quote }} + {{- if .Values.sidecar.dashboards.extraMounts }} + {{- toYaml .Values.sidecar.dashboards.extraMounts | trim | nindent 6}} + {{- end }} +{{- end}} +{{- if .Values.sidecar.datasources.enabled }} + - name: {{ template "grafana.name" . }}-sc-datasources + {{- if .Values.sidecar.image.sha }} + image: "{{ template "system_default_registry" . }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ template "system_default_registry" . }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.datasources.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.datasources.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: {{ .Values.sidecar.datasources.watchMethod }} + - name: LABEL + value: "{{ .Values.sidecar.datasources.label }}" + {{- if .Values.sidecar.datasources.labelValue }} + - name: LABEL_VALUE + value: {{ quote .Values.sidecar.datasources.labelValue }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.datasources.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.datasources.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/datasources" + - name: RESOURCE + value: {{ quote .Values.sidecar.datasources.resource }} + {{- if .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ .Values.sidecar.enableUniqueFilenames }}" + {{- end }} + {{- if .Values.sidecar.datasources.searchNamespace }} + - name: NAMESPACE + value: "{{ tpl (.Values.sidecar.datasources.searchNamespace | join ",") . }}" + {{- end }} + {{- if .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ .Values.sidecar.skipTlsVerify }}" + {{- end }} + {{- if .Values.sidecar.datasources.script }} + - name: SCRIPT + value: "{{ .Values.sidecar.datasources.script }}" + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_USERNAME + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.userKey | default "admin-user" }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_PASSWORD + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.passwordKey | default "admin-password" }} + {{- end }} + {{- if not .Values.sidecar.datasources.skipReload }} + - name: REQ_URL + value: {{ .Values.sidecar.datasources.reloadURL }} + - name: REQ_METHOD + value: POST + {{- end }} + {{- if .Values.sidecar.datasources.watchServerTimeout }} + {{- if ne .Values.sidecar.datasources.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.datasources.watchServerTimeout with .Values.sidecar.datasources.watchMethod %s" .Values.sidecar.datasources.watchMethod) }} + {{- end }} + - name: WATCH_SERVER_TIMEOUT + value: "{{ .Values.sidecar.datasources.watchServerTimeout }}" + {{- end }} + {{- if .Values.sidecar.datasources.watchClientTimeout }} + {{- if ne .Values.sidecar.datasources.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.datasources.watchClientTimeout with .Values.sidecar.datasources.watchMethod %s" .Values.sidecar.datasources.watchMethod) }} + {{- end }} + - name: WATCH_CLIENT_TIMEOUT + value: "{{ .Values.sidecar.datasources.watchClientTimeout }}" + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-datasources-volume + mountPath: "/etc/grafana/provisioning/datasources" +{{- end}} +{{- if .Values.sidecar.plugins.enabled }} + - name: {{ template "grafana.name" . }}-sc-plugins + {{- if .Values.sidecar.image.sha }} + image: "{{ template "system_default_registry" . }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ template "system_default_registry" . }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.plugins.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.plugins.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: {{ .Values.sidecar.plugins.watchMethod }} + - name: LABEL + value: "{{ .Values.sidecar.plugins.label }}" + {{- if .Values.sidecar.plugins.labelValue }} + - name: LABEL_VALUE + value: {{ quote .Values.sidecar.plugins.labelValue }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.plugins.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.plugins.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/plugins" + - name: RESOURCE + value: {{ quote .Values.sidecar.plugins.resource }} + {{- if .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ .Values.sidecar.enableUniqueFilenames }}" + {{- end }} + {{- if .Values.sidecar.plugins.searchNamespace }} + - name: NAMESPACE + value: "{{ tpl (.Values.sidecar.plugins.searchNamespace | join ",") . }}" + {{- end }} + {{- if .Values.sidecar.plugins.script }} + - name: SCRIPT + value: "{{ .Values.sidecar.plugins.script }}" + {{- end }} + {{- if .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ .Values.sidecar.skipTlsVerify }}" + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_USERNAME + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.userKey | default "admin-user" }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_PASSWORD + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.passwordKey | default "admin-password" }} + {{- end }} + {{- if not .Values.sidecar.plugins.skipReload }} + - name: REQ_URL + value: {{ .Values.sidecar.plugins.reloadURL }} + - name: REQ_METHOD + value: POST + {{- end }} + {{- if .Values.sidecar.plugins.watchServerTimeout }} + {{- if ne .Values.sidecar.plugins.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.plugins.watchServerTimeout with .Values.sidecar.plugins.watchMethod %s" .Values.sidecar.plugins.watchMethod) }} + {{- end }} + - name: WATCH_SERVER_TIMEOUT + value: "{{ .Values.sidecar.plugins.watchServerTimeout }}" + {{- end }} + {{- if .Values.sidecar.plugins.watchClientTimeout }} + {{- if ne .Values.sidecar.plugins.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.plugins.watchClientTimeout with .Values.sidecar.plugins.watchMethod %s" .Values.sidecar.plugins.watchMethod) }} + {{- end }} + - name: WATCH_CLIENT_TIMEOUT + value: "{{ .Values.sidecar.plugins.watchClientTimeout }}" + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-plugins-volume + mountPath: "/etc/grafana/provisioning/plugins" +{{- end}} + - name: {{ .Chart.Name }} + {{- if .Values.image.sha }} + image: "{{ template "system_default_registry" . }}{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}@sha256:{{ .Values.image.sha }}" + {{- else }} + image: "{{ template "system_default_registry" . }}{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + {{- end }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- if .Values.command }} + command: + {{- range .Values.command }} + - {{ . }} + {{- end }} + {{- end}} + {{- with .Values.containerSecurityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: config + mountPath: "/etc/grafana/grafana.ini" + subPath: grafana.ini + {{- if .Values.ldap.enabled }} + - name: ldap + mountPath: "/etc/grafana/ldap.toml" + subPath: ldap.toml + {{- end }} + {{- $root := . }} + {{- range .Values.extraConfigmapMounts }} + - name: {{ tpl .name $root }} + mountPath: {{ tpl .mountPath $root }} + subPath: {{ (tpl .subPath $root) | default "" }} + readOnly: {{ .readOnly }} + {{- end }} + - name: storage + mountPath: "/var/lib/grafana" +{{- if .Values.persistence.subPath }} + subPath: {{ tpl .Values.persistence.subPath . }} +{{- end }} +{{- if .Values.dashboards }} +{{- range $provider, $dashboards := .Values.dashboards }} +{{- range $key, $value := $dashboards }} +{{- if (or (hasKey $value "json") (hasKey $value "file")) }} + - name: dashboards-{{ $provider }} + mountPath: "/var/lib/grafana/dashboards/{{ $provider }}/{{ $key }}.json" + subPath: "{{ $key }}.json" +{{- end }} +{{- end }} +{{- end }} +{{- end -}} +{{- if .Values.dashboardsConfigMaps }} +{{- range (keys .Values.dashboardsConfigMaps | sortAlpha) }} + - name: dashboards-{{ . }} + mountPath: "/var/lib/grafana/dashboards/{{ . }}" +{{- end }} +{{- end }} +{{- if .Values.datasources }} +{{- range (keys .Values.datasources | sortAlpha) }} + - name: config + mountPath: "/etc/grafana/provisioning/datasources/{{ . }}" + subPath: {{ . | quote }} +{{- end }} +{{- end }} +{{- if .Values.notifiers }} +{{- range (keys .Values.notifiers | sortAlpha) }} + - name: config + mountPath: "/etc/grafana/provisioning/notifiers/{{ . }}" + subPath: {{ . | quote }} +{{- end }} +{{- end }} +{{- if .Values.alerting }} +{{- range (keys .Values.alerting | sortAlpha) }} + - name: config + mountPath: "/etc/grafana/provisioning/alerting/{{ . }}" + subPath: {{ . | quote }} +{{- end }} +{{- end }} +{{- if .Values.dashboardProviders }} +{{- range (keys .Values.dashboardProviders | sortAlpha) }} + - name: config + mountPath: "/etc/grafana/provisioning/dashboards/{{ . }}" + subPath: {{ . | quote }} +{{- end }} +{{- end }} +{{- if .Values.sidecar.dashboards.enabled }} + - name: sc-dashboard-volume + mountPath: {{ .Values.sidecar.dashboards.folder | quote }} +{{ if .Values.sidecar.dashboards.SCProvider }} + - name: sc-dashboard-provider + mountPath: "/etc/grafana/provisioning/dashboards/sc-dashboardproviders.yaml" + subPath: provider.yaml +{{- end}} +{{- end}} +{{- if .Values.sidecar.datasources.enabled }} + - name: sc-datasources-volume + mountPath: "/etc/grafana/provisioning/datasources" +{{- end}} +{{- if .Values.sidecar.plugins.enabled }} + - name: sc-plugins-volume + mountPath: "/etc/grafana/provisioning/plugins" +{{- end}} +{{- if .Values.sidecar.notifiers.enabled }} + - name: sc-notifiers-volume + mountPath: "/etc/grafana/provisioning/notifiers" +{{- end}} + {{- range .Values.extraSecretMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + readOnly: {{ .readOnly }} + subPath: {{ .subPath | default "" }} + {{- end }} + {{- range .Values.extraVolumeMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + subPath: {{ .subPath | default "" }} + readOnly: {{ .readOnly }} + {{- end }} + {{- range .Values.extraEmptyDirMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + {{- end }} + ports: + - name: {{ .Values.podPortName }} + containerPort: {{ .Values.service.targetPort }} + protocol: TCP + env: + {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: GF_SECURITY_ADMIN_USER + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.userKey | default "admin-user" }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: GF_SECURITY_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.passwordKey | default "admin-password" }} + {{- end }} + {{- if .Values.plugins }} + - name: GF_INSTALL_PLUGINS + valueFrom: + configMapKeyRef: + name: {{ template "grafana.fullname" . }} + key: plugins + {{- end }} + {{- if .Values.smtp.existingSecret }} + - name: GF_SMTP_USER + valueFrom: + secretKeyRef: + name: {{ .Values.smtp.existingSecret }} + key: {{ .Values.smtp.userKey | default "user" }} + - name: GF_SMTP_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.smtp.existingSecret }} + key: {{ .Values.smtp.passwordKey | default "password" }} + {{- end }} + {{- if .Values.imageRenderer.enabled }} + - name: GF_RENDERING_SERVER_URL + value: http://{{ template "grafana.fullname" . }}-image-renderer.{{ template "grafana.namespace" . }}:{{ .Values.imageRenderer.service.port }}/render + - name: GF_RENDERING_CALLBACK_URL + value: {{ .Values.imageRenderer.grafanaProtocol }}://{{ template "grafana.fullname" . }}.{{ template "grafana.namespace" . }}:{{ .Values.service.port }}/{{ .Values.imageRenderer.grafanaSubPath }} + {{- end }} + - name: GF_PATHS_DATA + value: {{ (get .Values "grafana.ini").paths.data }} + - name: GF_PATHS_LOGS + value: {{ (get .Values "grafana.ini").paths.logs }} + - name: GF_PATHS_PLUGINS + value: {{ (get .Values "grafana.ini").paths.plugins }} + - name: GF_PATHS_PROVISIONING + value: {{ (get .Values "grafana.ini").paths.provisioning }} + {{- range $key, $value := .Values.envValueFrom }} + - name: {{ $key | quote }} + valueFrom: +{{ tpl (toYaml $value) $ | indent 10 }} + {{- end }} +{{- range $key, $value := .Values.env }} + - name: "{{ tpl $key $ }}" + value: "{{ tpl (print $value) $ }}" +{{- end }} + {{- if or .Values.envFromSecret (or .Values.envRenderSecret .Values.envFromSecrets) .Values.envFromConfigMaps }} + envFrom: + {{- if .Values.envFromSecret }} + - secretRef: + name: {{ tpl .Values.envFromSecret . }} + {{- end }} + {{- if .Values.envRenderSecret }} + - secretRef: + name: {{ template "grafana.fullname" . }}-env + {{- end }} + {{- range .Values.envFromSecrets }} + - secretRef: + name: {{ tpl .name $ }} + optional: {{ .optional | default false }} + {{- end }} + {{- range .Values.envFromConfigMaps }} + - configMapRef: + name: {{ tpl .name $ }} + optional: {{ .optional | default false }} + {{- end }} + {{- end }} + {{- with .Values.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} +{{- if .Values.lifecycleHooks }} + lifecycle: {{ tpl (.Values.lifecycleHooks | toYaml) . | nindent 6 }} +{{- end }} + {{- with .Values.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} +{{- with .Values.extraContainers }} +{{ tpl . $ | indent 2 }} +{{- end }} +nodeSelector: {{ include "linux-node-selector" . | nindent 2 }} +{{- if .Values.nodeSelector }} +{{ toYaml .Values.nodeSelector | indent 2 }} +{{- end }} +{{- $root := . }} +{{- with .Values.affinity }} +affinity: +{{ tpl (toYaml .) $root | indent 2 }} +{{- end }} +{{- with .Values.topologySpreadConstraints }} +topologySpreadConstraints: + {{- toYaml . | nindent 2 }} +{{- end }} +tolerations: {{ include "linux-node-tolerations" . | nindent 2 }} +{{- if .Values.tolerations }} +{{ toYaml .Values.tolerations | indent 2 }} +{{- end }} +volumes: + - name: config + configMap: + name: {{ template "grafana.fullname" . }} +{{- $root := . }} +{{- range .Values.extraConfigmapMounts }} + - name: {{ tpl .name $root }} + configMap: + name: {{ tpl .configMap $root }} + {{- if .items }} + items: {{ toYaml .items | nindent 6 }} + {{- end }} +{{- end }} + {{- if .Values.dashboards }} + {{- range (keys .Values.dashboards | sortAlpha) }} + - name: dashboards-{{ . }} + configMap: + name: {{ template "grafana.fullname" $ }}-dashboards-{{ . }} + {{- end }} + {{- end }} + {{- if .Values.dashboardsConfigMaps }} + {{ $root := . }} + {{- range $provider, $name := .Values.dashboardsConfigMaps }} + - name: dashboards-{{ $provider }} + configMap: + name: {{ tpl $name $root }} + {{- end }} + {{- end }} + {{- if .Values.ldap.enabled }} + - name: ldap + secret: + {{- if .Values.ldap.existingSecret }} + secretName: {{ .Values.ldap.existingSecret }} + {{- else }} + secretName: {{ template "grafana.fullname" . }} + {{- end }} + items: + - key: ldap-toml + path: ldap.toml + {{- end }} +{{- if and .Values.persistence.enabled (eq .Values.persistence.type "pvc") }} + - name: storage + persistentVolumeClaim: + claimName: {{ tpl (.Values.persistence.existingClaim | default (include "grafana.fullname" .)) . }} +{{- else if and .Values.persistence.enabled (eq .Values.persistence.type "statefulset") }} +# nothing +{{- else }} + - name: storage +{{- if .Values.persistence.inMemory.enabled }} + emptyDir: + medium: Memory +{{- if .Values.persistence.inMemory.sizeLimit }} + sizeLimit: {{ .Values.persistence.inMemory.sizeLimit }} +{{- end -}} +{{- else }} + emptyDir: {} +{{- end -}} +{{- end -}} +{{- if .Values.sidecar.dashboards.enabled }} + - name: sc-dashboard-volume +{{- if .Values.sidecar.dashboards.sizeLimit }} + emptyDir: + sizeLimit: {{ .Values.sidecar.dashboards.sizeLimit }} +{{- else }} + emptyDir: {} +{{- end -}} +{{- if .Values.sidecar.dashboards.SCProvider }} + - name: sc-dashboard-provider + configMap: + name: {{ template "grafana.fullname" . }}-config-dashboards +{{- end }} +{{- end }} +{{- if .Values.sidecar.datasources.enabled }} + - name: sc-datasources-volume +{{- if .Values.sidecar.datasources.sizeLimit }} + emptyDir: + sizeLimit: {{ .Values.sidecar.datasources.sizeLimit }} +{{- else }} + emptyDir: {} +{{- end -}} +{{- end -}} +{{- if .Values.sidecar.plugins.enabled }} + - name: sc-plugins-volume +{{- if .Values.sidecar.plugins.sizeLimit }} + emptyDir: + sizeLimit: {{ .Values.sidecar.plugins.sizeLimit }} +{{- else }} + emptyDir: {} +{{- end -}} +{{- end -}} +{{- if .Values.sidecar.notifiers.enabled }} + - name: sc-notifiers-volume +{{- if .Values.sidecar.notifiers.sizeLimit }} + emptyDir: + sizeLimit: {{ .Values.sidecar.notifiers.sizeLimit }} +{{- else }} + emptyDir: {} +{{- end -}} +{{- end -}} +{{- range .Values.extraSecretMounts }} +{{- if .secretName }} + - name: {{ .name }} + secret: + secretName: {{ .secretName }} + defaultMode: {{ .defaultMode }} + {{- if .items }} + items: {{ toYaml .items | nindent 6 }} + {{- end }} +{{- else if .projected }} + - name: {{ .name }} + projected: {{- toYaml .projected | nindent 6 }} +{{- else if .csi }} + - name: {{ .name }} + csi: {{- toYaml .csi | nindent 6 }} +{{- end }} +{{- end }} +{{- range .Values.extraVolumeMounts }} + - name: {{ .name }} + {{- if .existingClaim }} + persistentVolumeClaim: + claimName: {{ .existingClaim }} + {{- else if .hostPath }} + hostPath: + path: {{ .hostPath }} + {{- else if .csi }} + csi: + data: + {{ toYaml .data | nindent 6 }} + {{- else }} + emptyDir: {} + {{- end }} +{{- end }} +{{- range .Values.extraEmptyDirMounts }} + - name: {{ .name }} + emptyDir: {} +{{- end -}} +{{- if .Values.extraContainerVolumes }} +{{ tpl (toYaml .Values.extraContainerVolumes) . | indent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/clusterrole.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/clusterrole.yaml new file mode 100644 index 0000000000..154658b51c --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/clusterrole.yaml @@ -0,0 +1,25 @@ +{{- if and .Values.rbac.create (not .Values.rbac.namespaced) (not .Values.rbac.useExistingRole) }} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + labels: + {{- include "grafana.labels" . | nindent 4 }} +{{- with .Values.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} + name: {{ template "grafana.fullname" . }}-clusterrole +{{- if or .Values.sidecar.dashboards.enabled (or .Values.rbac.extraClusterRoleRules (or .Values.sidecar.datasources.enabled .Values.sidecar.plugins.enabled)) }} +rules: +{{- if or .Values.sidecar.dashboards.enabled (or .Values.sidecar.datasources.enabled .Values.sidecar.plugins.enabled) }} +- apiGroups: [""] # "" indicates the core API group + resources: ["configmaps", "secrets"] + verbs: ["get", "watch", "list"] +{{- end}} +{{- with .Values.rbac.extraClusterRoleRules }} +{{ toYaml . | indent 0 }} +{{- end}} +{{- else }} +rules: [] +{{- end}} +{{- end}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/clusterrolebinding.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/clusterrolebinding.yaml new file mode 100644 index 0000000000..4accbfac04 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/clusterrolebinding.yaml @@ -0,0 +1,24 @@ +{{- if and .Values.rbac.create (not .Values.rbac.namespaced) }} +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "grafana.fullname" . }}-clusterrolebinding + labels: + {{- include "grafana.labels" . | nindent 4 }} +{{- with .Values.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} +subjects: + - kind: ServiceAccount + name: {{ template "grafana.serviceAccountName" . }} + namespace: {{ template "grafana.namespace" . }} +roleRef: + kind: ClusterRole +{{- if (not .Values.rbac.useExistingRole) }} + name: {{ template "grafana.fullname" . }}-clusterrole +{{- else }} + name: {{ .Values.rbac.useExistingRole }} +{{- end }} + apiGroup: rbac.authorization.k8s.io +{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/configmap-dashboard-provider.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/configmap-dashboard-provider.yaml new file mode 100644 index 0000000000..65d73858e0 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/configmap-dashboard-provider.yaml @@ -0,0 +1,29 @@ +{{- if .Values.sidecar.dashboards.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + {{- include "grafana.labels" . | nindent 4 }} +{{- with .Values.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} + name: {{ template "grafana.fullname" . }}-config-dashboards + namespace: {{ template "grafana.namespace" . }} +data: + provider.yaml: |- + apiVersion: 1 + providers: + - name: '{{ .Values.sidecar.dashboards.provider.name }}' + orgId: {{ .Values.sidecar.dashboards.provider.orgid }} + {{- if not .Values.sidecar.dashboards.provider.foldersFromFilesStructure }} + folder: '{{ .Values.sidecar.dashboards.provider.folder }}' + {{- end}} + type: {{ .Values.sidecar.dashboards.provider.type }} + disableDeletion: {{ .Values.sidecar.dashboards.provider.disableDelete }} + allowUiUpdates: {{ .Values.sidecar.dashboards.provider.allowUiUpdates }} + updateIntervalSeconds: {{ .Values.sidecar.dashboards.provider.updateIntervalSeconds | default 30 }} + options: + foldersFromFilesStructure: {{ .Values.sidecar.dashboards.provider.foldersFromFilesStructure }} + path: {{ .Values.sidecar.dashboards.folder }}{{- with .Values.sidecar.dashboards.defaultFolderName }}/{{ . }}{{- end }} +{{- end}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/configmap.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/configmap.yaml new file mode 100644 index 0000000000..87460cd36d --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/configmap.yaml @@ -0,0 +1,117 @@ +{{- if .Values.createConfigmap }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "grafana.fullname" . }} + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +{{- with .Values.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} +data: +{{- if .Values.plugins }} + plugins: {{ join "," .Values.plugins }} +{{- end }} + grafana.ini: | +{{- range $elem, $elemVal := index .Values "grafana.ini" }} + {{- if not (kindIs "map" $elemVal) }} + {{- if kindIs "invalid" $elemVal }} + {{ $elem }} = + {{- else if kindIs "string" $elemVal }} + {{ $elem }} = {{ tpl $elemVal $ }} + {{- else }} + {{ $elem }} = {{ $elemVal }} + {{- end }} + {{- end }} +{{- end }} +{{- range $key, $value := index .Values "grafana.ini" }} + {{- if kindIs "map" $value }} + [{{ $key }}] + {{- range $elem, $elemVal := $value }} + {{- if kindIs "invalid" $elemVal }} + {{ $elem }} = + {{- else if kindIs "string" $elemVal }} + {{ $elem }} = {{ tpl $elemVal $ }} + {{- else }} + {{ $elem }} = {{ $elemVal }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} + +{{- if .Values.datasources }} +{{ $root := . }} + {{- range $key, $value := .Values.datasources }} + {{ $key }}: | +{{ tpl (toYaml $value | indent 4) $root }} + {{- end -}} +{{- end -}} + +{{- if .Values.notifiers }} + {{- range $key, $value := .Values.notifiers }} + {{ $key }}: | +{{ toYaml $value | indent 4 }} + {{- end -}} +{{- end -}} + +{{- if .Values.alerting }} +{{ $root := . }} + {{- range $key, $value := .Values.alerting }} + {{ $key }}: | +{{ tpl (toYaml $value | indent 4) $root }} + {{- end -}} +{{- end -}} + +{{- if .Values.dashboardProviders }} + {{- range $key, $value := .Values.dashboardProviders }} + {{ $key }}: | +{{ toYaml $value | indent 4 }} + {{- end -}} +{{- end -}} + +{{- if .Values.dashboards }} + download_dashboards.sh: | + #!/usr/bin/env sh + set -euf + {{- if .Values.dashboardProviders }} + {{- range $key, $value := .Values.dashboardProviders }} + {{- range $value.providers }} + mkdir -p {{ .options.path }} + {{- end }} + {{- end }} + {{- end }} + {{ $dashboardProviders := .Values.dashboardProviders }} + {{- range $provider, $dashboards := .Values.dashboards }} + {{- range $key, $value := $dashboards }} + {{- if (or (hasKey $value "gnetId") (hasKey $value "url")) }} + curl -skf \ + --connect-timeout 60 \ + --max-time 60 \ + {{- if not $value.b64content }} + -H "Accept: application/json" \ + {{- if $value.token }} + -H "Authorization: token {{ $value.token }}" \ + {{- end }} + {{- if $value.bearerToken }} + -H "Authorization: Bearer {{ $value.bearerToken }}" \ + {{- end }} + {{- if $value.gitlabToken }} + -H "PRIVATE-TOKEN: {{ $value.gitlabToken }}" \ + {{- end }} + -H "Content-Type: application/json;charset=UTF-8" \ + {{ end }} + {{- $dpPath := "" -}} + {{- range $kd := (index $dashboardProviders "dashboardproviders.yaml").providers -}} + {{- if eq $kd.name $provider -}} + {{- $dpPath = $kd.options.path -}} + {{- end -}} + {{- end -}} + {{- if $value.url -}}"{{ $value.url }}"{{- else -}}"https://grafana.com/api/dashboards/{{ $value.gnetId }}/revisions/{{- if $value.revision -}}{{ $value.revision }}{{- else -}}1{{- end -}}/download"{{- end -}}{{ if $value.datasource }} | sed '/-- .* --/! s/"datasource":.*,/"datasource": "{{ $value.datasource }}",/g'{{ end }}{{- if $value.b64content -}} | base64 -d {{- end -}} \ + > "{{- if $dpPath -}}{{ $dpPath }}{{- else -}}/var/lib/grafana/dashboards/{{ $provider }}{{- end -}}/{{ $key }}.json" + {{- end }} + {{- end -}} + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/dashboards-json-configmap.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/dashboards-json-configmap.yaml new file mode 100644 index 0000000000..59e0be6415 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/dashboards-json-configmap.yaml @@ -0,0 +1,35 @@ +{{- if .Values.dashboards }} +{{ $files := .Files }} +{{- range $provider, $dashboards := .Values.dashboards }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "grafana.fullname" $ }}-dashboards-{{ $provider }} + namespace: {{ template "grafana.namespace" $ }} + labels: + {{- include "grafana.labels" $ | nindent 4 }} + dashboard-provider: {{ $provider }} +{{- if $dashboards }} +data: +{{- $dashboardFound := false }} +{{- range $key, $value := $dashboards }} +{{- if (or (hasKey $value "json") (hasKey $value "file")) }} +{{- $dashboardFound = true }} +{{ print $key | indent 2 }}.json: +{{- if hasKey $value "json" }} + |- +{{ $value.json | indent 6 }} +{{- end }} +{{- if hasKey $value "file" }} +{{ toYaml ( $files.Get $value.file ) | indent 4}} +{{- end }} +{{- end }} +{{- end }} +{{- if not $dashboardFound }} + {} +{{- end }} +{{- end }} +--- +{{- end }} + +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/deployment.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/deployment.yaml new file mode 100644 index 0000000000..fee9c335a5 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/deployment.yaml @@ -0,0 +1,50 @@ +{{ if (and (not .Values.useStatefulSet) (or (not .Values.persistence.enabled) (eq .Values.persistence.type "pvc"))) }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "grafana.fullname" . }} + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +{{- if .Values.labels }} +{{ toYaml .Values.labels | indent 4 }} +{{- end }} +{{- with .Values.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} +spec: + {{- if and (not .Values.autoscaling.enabled) (.Values.replicas) }} + replicas: {{ .Values.replicas }} + {{- end }} + revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} + selector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 6 }} +{{- with .Values.deploymentStrategy }} + strategy: +{{ toYaml . | trim | indent 4 }} +{{- end }} + template: + metadata: + labels: + {{- include "grafana.selectorLabels" . | nindent 8 }} +{{- with .Values.podLabels }} +{{ toYaml . | indent 8 }} +{{- end }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + checksum/dashboards-json-config: {{ include (print $.Template.BasePath "/dashboards-json-configmap.yaml") . | sha256sum }} + checksum/sc-dashboard-provider-config: {{ include (print $.Template.BasePath "/configmap-dashboard-provider.yaml") . | sha256sum }} +{{- if and (or (and (not .Values.admin.existingSecret) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD)) (and .Values.ldap.enabled (not .Values.ldap.existingSecret))) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} +{{- end }} +{{- if .Values.envRenderSecret }} + checksum/secret-env: {{ include (print $.Template.BasePath "/secret-env.yaml") . | sha256sum }} +{{- end }} +{{- with .Values.podAnnotations }} +{{ toYaml . | indent 8 }} +{{- end }} + spec: + {{- include "grafana.pod" . | indent 6 }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/extra-manifests.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/extra-manifests.yaml new file mode 100644 index 0000000000..a9bb3b6ba8 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/extra-manifests.yaml @@ -0,0 +1,4 @@ +{{ range .Values.extraObjects }} +--- +{{ tpl (toYaml .) $ }} +{{ end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/headless-service.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/headless-service.yaml new file mode 100644 index 0000000000..b5faddcfce --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/headless-service.yaml @@ -0,0 +1,22 @@ +{{- if or .Values.headlessService (and .Values.persistence.enabled (not .Values.persistence.existingClaim) (eq .Values.persistence.type "statefulset"))}} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "grafana.fullname" . }}-headless + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +{{- with .Values.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} +spec: + clusterIP: None + selector: + {{- include "grafana.selectorLabels" . | nindent 4 }} + type: ClusterIP + ports: + - protocol: TCP + port: 3000 + targetPort: {{ .Values.service.targetPort }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/hpa.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/hpa.yaml new file mode 100644 index 0000000000..236a06d658 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/hpa.yaml @@ -0,0 +1,21 @@ +{{- if .Values.autoscaling.enabled }} +apiVersion: {{ template "grafana.hpa.apiVersion" . }} +kind: HorizontalPodAutoscaler +metadata: + name: {{ template "grafana.fullname" . }} + namespace: {{ template "grafana.namespace" . }} + labels: + app.kubernetes.io/name: {{ template "grafana.name" . }} + helm.sh/chart: {{ template "grafana.chart" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ template "grafana.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: +{{ toYaml .Values.autoscaling.metrics | indent 4 }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/image-renderer-deployment.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/image-renderer-deployment.yaml new file mode 100644 index 0000000000..97a8675b22 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/image-renderer-deployment.yaml @@ -0,0 +1,123 @@ +{{ if .Values.imageRenderer.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "grafana.fullname" . }}-image-renderer + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.imageRenderer.labels" . | nindent 4 }} +{{- if .Values.imageRenderer.labels }} +{{ toYaml .Values.imageRenderer.labels | indent 4 }} +{{- end }} +{{- with .Values.imageRenderer.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} +spec: + replicas: {{ .Values.imageRenderer.replicas }} + revisionHistoryLimit: {{ .Values.imageRenderer.revisionHistoryLimit }} + selector: + matchLabels: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 6 }} +{{- with .Values.imageRenderer.deploymentStrategy }} + strategy: +{{ toYaml . | trim | indent 4 }} +{{- end }} + template: + metadata: + labels: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 8 }} +{{- with .Values.imageRenderer.podLabels }} +{{ toYaml . | indent 8 }} +{{- end }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} +{{- with .Values.imageRenderer.podAnnotations }} +{{ toYaml . | indent 8 }} +{{- end }} + spec: + + {{- if .Values.imageRenderer.schedulerName }} + schedulerName: "{{ .Values.imageRenderer.schedulerName }}" + {{- end }} + {{- if .Values.imageRenderer.serviceAccountName }} + serviceAccountName: "{{ .Values.imageRenderer.serviceAccountName }}" + {{- else }} + serviceAccountName: {{ template "grafana.serviceAccountName" . }} + {{- end }} + {{- if .Values.imageRenderer.securityContext }} + securityContext: + {{- toYaml .Values.imageRenderer.securityContext | nindent 8 }} + {{- end }} + {{- if .Values.imageRenderer.hostAliases }} + hostAliases: + {{- toYaml .Values.imageRenderer.hostAliases | nindent 8 }} + {{- end }} + {{- if .Values.imageRenderer.priorityClassName }} + priorityClassName: {{ .Values.imageRenderer.priorityClassName }} + {{- end }} + {{- if .Values.imageRenderer.image.pullSecrets }} + imagePullSecrets: + {{- $root := . }} + {{- range .Values.imageRenderer.image.pullSecrets }} + - name: {{ tpl . $root }} + {{- end}} + {{- end }} + containers: + - name: {{ .Chart.Name }}-image-renderer + {{- if .Values.imageRenderer.image.sha }} + image: "{{ template "system_default_registry" . }}{{ .Values.imageRenderer.image.repository }}:{{ .Values.imageRenderer.image.tag }}@sha256:{{ .Values.imageRenderer.image.sha }}" + {{- else }} + image: "{{ template "system_default_registry" . }}{{ .Values.imageRenderer.image.repository }}:{{ .Values.imageRenderer.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.imageRenderer.image.pullPolicy }} + {{- if .Values.imageRenderer.command }} + command: + {{- range .Values.imageRenderer.command }} + - {{ . }} + {{- end }} + {{- end}} + ports: + - name: {{ .Values.imageRenderer.service.portName }} + containerPort: {{ .Values.imageRenderer.service.targetPort }} + protocol: TCP + livenessProbe: + httpGet: + path: / + port: {{ .Values.imageRenderer.service.portName }} + env: + - name: HTTP_PORT + value: {{ .Values.imageRenderer.service.targetPort | quote }} + {{- range $key, $value := .Values.imageRenderer.env }} + - name: {{ $key | quote }} + value: {{ $value | quote }} + {{- end }} + securityContext: + capabilities: + drop: ['all'] + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /tmp + name: image-renderer-tmpfs + {{- with .Values.imageRenderer.resources }} + resources: +{{ toYaml . | indent 12 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} + {{- if .Values.imageRenderer.nodeSelector }} +{{ toYaml . | indent 8 }} + {{- end }} + {{- $root := . }} + {{- with .Values.imageRenderer.affinity }} + affinity: +{{ tpl (toYaml .) $root | indent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} + {{- if .Values.imageRenderer.tolerations }} +{{ toYaml . | indent 8 }} + {{- end }} + volumes: + - name: image-renderer-tmpfs + emptyDir: {} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/image-renderer-network-policy.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/image-renderer-network-policy.yaml new file mode 100644 index 0000000000..0d9bdfe4d0 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/image-renderer-network-policy.yaml @@ -0,0 +1,73 @@ +{{- if and (.Values.imageRenderer.enabled) (.Values.imageRenderer.networkPolicy.limitIngress) }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ template "grafana.fullname" . }}-image-renderer-ingress + namespace: {{ template "grafana.namespace" . }} + annotations: + comment: Limit image-renderer ingress traffic from grafana +spec: + podSelector: + matchLabels: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 6 }} + {{- if .Values.imageRenderer.podLabels }} + {{ toYaml .Values.imageRenderer.podLabels | nindent 6 }} + {{- end }} + + policyTypes: + - Ingress + ingress: + - ports: + - port: {{ .Values.imageRenderer.service.targetPort }} + protocol: TCP + from: + - namespaceSelector: + matchLabels: + name: {{ template "grafana.namespace" . }} + podSelector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 14 }} + {{- if .Values.podLabels }} + {{ toYaml .Values.podLabels | nindent 14 }} + {{- end }} +{{ end }} + +{{- if and (.Values.imageRenderer.enabled) (.Values.imageRenderer.networkPolicy.limitEgress) }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ template "grafana.fullname" . }}-image-renderer-egress + namespace: {{ template "grafana.namespace" . }} + annotations: + comment: Limit image-renderer egress traffic to grafana +spec: + podSelector: + matchLabels: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 6 }} + {{- if .Values.imageRenderer.podLabels }} + {{ toYaml .Values.imageRenderer.podLabels | nindent 6 }} + {{- end }} + + policyTypes: + - Egress + egress: + # allow dns resolution + - ports: + - port: 53 + protocol: UDP + - port: 53 + protocol: TCP + # talk only to grafana + - ports: + - port: {{ .Values.service.port }} + protocol: TCP + to: + - podSelector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 14 }} + {{- if .Values.podLabels }} + {{ toYaml .Values.podLabels | nindent 14 }} + {{- end }} +{{ end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/image-renderer-service.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/image-renderer-service.yaml new file mode 100644 index 0000000000..fcf707a3f7 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/image-renderer-service.yaml @@ -0,0 +1,33 @@ +{{ if .Values.imageRenderer.enabled }} +{{ if .Values.imageRenderer.service.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "grafana.fullname" . }}-image-renderer + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.imageRenderer.labels" . | nindent 4 }} +{{- if .Values.imageRenderer.service.labels }} +{{ toYaml .Values.imageRenderer.service.labels | indent 4 }} +{{- end }} +{{- with .Values.imageRenderer.service.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} +spec: + type: ClusterIP + {{- if .Values.imageRenderer.service.clusterIP }} + clusterIP: {{ .Values.imageRenderer.service.clusterIP }} + {{end}} + ports: + - name: {{ .Values.imageRenderer.service.portName }} + port: {{ .Values.imageRenderer.service.port }} + protocol: TCP + targetPort: {{ .Values.imageRenderer.service.targetPort }} + {{- if .Values.imageRenderer.appProtocol }} + appProtocol: {{ .Values.imageRenderer.appProtocol }} + {{- end }} + selector: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 4 }} +{{ end }} +{{ end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/ingress.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/ingress.yaml new file mode 100644 index 0000000000..7699cecaa8 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/ingress.yaml @@ -0,0 +1,78 @@ +{{- if .Values.ingress.enabled -}} +{{- $ingressApiIsStable := eq (include "grafana.ingress.isStable" .) "true" -}} +{{- $ingressSupportsIngressClassName := eq (include "grafana.ingress.supportsIngressClassName" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "grafana.ingress.supportsPathType" .) "true" -}} +{{- $fullName := include "grafana.fullname" . -}} +{{- $servicePort := .Values.service.port -}} +{{- $ingressPath := .Values.ingress.path -}} +{{- $ingressPathType := .Values.ingress.pathType -}} +{{- $extraPaths := .Values.ingress.extraPaths -}} +apiVersion: {{ include "grafana.ingress.apiVersion" . }} +kind: Ingress +metadata: + name: {{ $fullName }} + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +{{- if .Values.ingress.labels }} +{{ toYaml .Values.ingress.labels | indent 4 }} +{{- end }} + {{- if .Values.ingress.annotations }} + annotations: + {{- range $key, $value := .Values.ingress.annotations }} + {{ $key }}: {{ tpl $value $ | quote }} + {{- end }} + {{- end }} +spec: + {{- if and $ingressSupportsIngressClassName .Values.ingress.ingressClassName }} + ingressClassName: {{ .Values.ingress.ingressClassName }} + {{- end -}} +{{- if .Values.ingress.tls }} + tls: +{{ tpl (toYaml .Values.ingress.tls) $ | indent 4 }} +{{- end }} + rules: + {{- if .Values.ingress.hosts }} + {{- range .Values.ingress.hosts }} + - host: {{ tpl . $}} + http: + paths: +{{- if $extraPaths }} +{{ toYaml $extraPaths | indent 10 }} +{{- end }} + - path: {{ $ingressPath }} + {{- if $ingressSupportsPathType }} + pathType: {{ $ingressPathType }} + {{- end }} + backend: + {{- if $ingressApiIsStable }} + service: + name: {{ $fullName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end }} + {{- else }} + - http: + paths: + - backend: + {{- if $ingressApiIsStable }} + service: + name: {{ $fullName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- if $ingressPath }} + path: {{ $ingressPath }} + {{- end }} + {{- if $ingressSupportsPathType }} + pathType: {{ $ingressPathType }} + {{- end }} + {{- end -}} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/networkpolicy.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/networkpolicy.yaml new file mode 100644 index 0000000000..b751d9436f --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/networkpolicy.yaml @@ -0,0 +1,52 @@ +{{- if .Values.networkPolicy.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ template "grafana.fullname" . }} + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.labels }} + {{ toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + policyTypes: + {{- if .Values.networkPolicy.ingress }} + - Ingress + {{- end }} + {{- if .Values.networkPolicy.egress.enabled }} + - Egress + {{- end }} + podSelector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 6 }} + + {{- if .Values.networkPolicy.egress.enabled }} + egress: + - ports: + {{ .Values.networkPolicy.egress.ports | toJson }} + {{- end }} + {{- if .Values.networkPolicy.ingress }} + ingress: + - ports: + - port: {{ .Values.service.targetPort }} + {{- if not .Values.networkPolicy.allowExternal }} + from: + - podSelector: + matchLabels: + {{ template "grafana.fullname" . }}-client: "true" + {{- with .Values.networkPolicy.explicitNamespacesSelector }} + - namespaceSelector: + {{- toYaml . | nindent 12 }} + {{- end }} + - podSelector: + matchLabels: + {{- include "grafana.labels" . | nindent 14 }} + role: read + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/nginx-config.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/nginx-config.yaml new file mode 100644 index 0000000000..557471f6ff --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/nginx-config.yaml @@ -0,0 +1,94 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-nginx-proxy-config + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +data: + nginx.conf: |- + worker_processes auto; + error_log /dev/stdout warn; + pid /var/cache/nginx/nginx.pid; + + events { + worker_connections 1024; + } + + http { + include /etc/nginx/mime.types; + log_format main '[$time_local - $status] $remote_addr - $remote_user $request ($http_referer)'; + + proxy_connect_timeout 10; + proxy_read_timeout 180; + proxy_send_timeout 5; + proxy_buffering off; + proxy_cache_path /var/cache/nginx/cache levels=1:2 keys_zone=my_zone:100m inactive=1d max_size=10g; + + map $http_upgrade $connection_upgrade { + default upgrade; + '' close; + } + + server { + listen 8080; + access_log off; + + gzip on; + gzip_min_length 1k; + gzip_comp_level 2; + gzip_types text/plain application/javascript application/x-javascript text/css application/xml text/javascript image/jpeg image/gif image/png; + gzip_vary on; + gzip_disable "MSIE [1-6]\."; + + proxy_set_header Host $host; + + location /api/dashboards { + proxy_pass http://localhost:3000; + } + + location /api/search { + proxy_pass http://localhost:3000; + + sub_filter_types application/json; + sub_filter_once off; + } + + location /api/live/ { + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_set_header Host $http_host; + proxy_pass http://localhost:3000; + } + + location / { + proxy_cache my_zone; + proxy_cache_valid 200 302 1d; + proxy_cache_valid 301 30d; + proxy_cache_valid any 5m; + proxy_cache_bypass $http_cache_control; + add_header X-Proxy-Cache $upstream_cache_status; + add_header Cache-Control "public"; + + proxy_pass http://localhost:3000/; + + sub_filter_once off; + + {{- if eq .Values.global.cattle.clusterId "local" -}} + sub_filter '"appSubUrl":""' '"appSubUrl":"/api/v1/namespaces/{{ template "grafana.namespace" . }}/services/http:{{ template "grafana.fullname" . }}:{{ .Values.service.port }}/proxy"'; + {{- else -}} + sub_filter '"appSubUrl":""' '"appSubUrl":"/k8s/clusters/{{ .Values.global.cattle.clusterId }}/api/v1/namespaces/{{ template "grafana.namespace" . }}/services/http:{{ template "grafana.fullname" . }}:{{ .Values.service.port }}/proxy"'; + {{- end -}} + + sub_filter ':"/avatar/' ':"avatar/'; + + if ($request_filename ~ .*\.(?:js|css|jpg|jpeg|gif|png|ico|cur|gz|svg|svgz|mp4|ogg|ogv|webm)$) { + expires 90d; + } + + rewrite ^/k8s/clusters/.*/proxy(.*) /$1 break; + + } + } + } diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/poddisruptionbudget.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/poddisruptionbudget.yaml new file mode 100644 index 0000000000..70901b70c4 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/poddisruptionbudget.yaml @@ -0,0 +1,22 @@ +{{- if .Values.podDisruptionBudget }} +apiVersion: {{ include "grafana.podDisruptionBudget.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ template "grafana.fullname" . }} + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +{{- if .Values.labels }} +{{ toYaml .Values.labels | indent 4 }} +{{- end }} +spec: +{{- if .Values.podDisruptionBudget.minAvailable }} + minAvailable: {{ .Values.podDisruptionBudget.minAvailable }} +{{- end }} +{{- if .Values.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ .Values.podDisruptionBudget.maxUnavailable }} +{{- end }} + selector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 6 }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/podsecuritypolicy.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/podsecuritypolicy.yaml new file mode 100644 index 0000000000..82d295ad13 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/podsecuritypolicy.yaml @@ -0,0 +1,45 @@ +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "grafana.fullname" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +{{- if .Values.rbac.pspAnnotations }} + annotations: {{ toYaml .Values.rbac.pspAnnotations | nindent 4 }} +{{- end }} +spec: + privileged: false + allowPrivilegeEscalation: false + requiredDropCapabilities: + # Default set from Docker, with DAC_OVERRIDE and CHOWN + - ALL + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'csi' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/pvc.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/pvc.yaml new file mode 100644 index 0000000000..8a3ee12226 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/pvc.yaml @@ -0,0 +1,35 @@ +{{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) (eq .Values.persistence.type "pvc")}} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ template "grafana.fullname" . }} + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.persistence.annotations }} + annotations: +{{ toYaml . | indent 4 }} + {{- end }} + {{- with .Values.persistence.finalizers }} + finalizers: +{{ toYaml . | indent 4 }} + {{- end }} +spec: + accessModes: +{{- $_ := required "Must provide at least one access mode for persistent volumes used by Grafana" .Values.persistence.accessModes }} +{{- $_ := required "Must provide at least one access mode for persistent volumes used by Grafana" (first .Values.persistence.accessModes) }} + {{- range .Values.persistence.accessModes }} + - {{ . | quote }} + {{- end }} + resources: + requests: + storage: {{ required "Must provide size for persistent volumes used by Grafana" .Values.persistence.size | quote }} + {{- if .Values.persistence.storageClassName }} + storageClassName: {{ .Values.persistence.storageClassName }} + {{- end -}} + {{- with .Values.persistence.selectorLabels }} + selector: + matchLabels: +{{ toYaml . | indent 6 }} + {{- end }} +{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/role.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/role.yaml new file mode 100644 index 0000000000..80e2c596ae --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/role.yaml @@ -0,0 +1,32 @@ +{{- if and .Values.rbac.create (not .Values.rbac.useExistingRole) -}} +apiVersion: {{ template "grafana.rbac.apiVersion" . }} +kind: Role +metadata: + name: {{ template "grafana.fullname" . }} + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +{{- with .Values.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} +{{- if or .Values.global.cattle.psp.enabled (and .Values.rbac.namespaced (or .Values.sidecar.dashboards.enabled (or .Values.sidecar.datasources.enabled (or .Values.sidecar.plugins.enabled .Values.rbac.extraRoleRules)))) }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['extensions'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: [{{ template "grafana.fullname" . }}] +{{- end }} +{{- if and .Values.rbac.namespaced (or .Values.sidecar.dashboards.enabled (or .Values.sidecar.datasources.enabled .Values.sidecar.plugins.enabled)) }} +- apiGroups: [""] # "" indicates the core API group + resources: ["configmaps", "secrets"] + verbs: ["get", "watch", "list"] +{{- end }} +{{- with .Values.rbac.extraRoleRules }} +{{ toYaml . | indent 0 }} +{{- end}} +{{- else }} +rules: [] +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/rolebinding.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/rolebinding.yaml new file mode 100644 index 0000000000..e0107255ee --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/rolebinding.yaml @@ -0,0 +1,25 @@ +{{- if .Values.rbac.create -}} +apiVersion: {{ template "grafana.rbac.apiVersion" . }} +kind: RoleBinding +metadata: + name: {{ template "grafana.fullname" . }} + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +{{- with .Values.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role +{{- if (not .Values.rbac.useExistingRole) }} + name: {{ template "grafana.fullname" . }} +{{- else }} + name: {{ .Values.rbac.useExistingRole }} +{{- end }} +subjects: +- kind: ServiceAccount + name: {{ template "grafana.serviceAccountName" . }} + namespace: {{ template "grafana.namespace" . }} +{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/secret-env.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/secret-env.yaml new file mode 100644 index 0000000000..5c09313e66 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/secret-env.yaml @@ -0,0 +1,14 @@ +{{- if .Values.envRenderSecret }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "grafana.fullname" . }}-env + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +type: Opaque +data: +{{- range $key, $val := .Values.envRenderSecret }} + {{ $key }}: {{ $val | b64enc | quote }} +{{- end -}} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/secret.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/secret.yaml new file mode 100644 index 0000000000..c8aa750acb --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/secret.yaml @@ -0,0 +1,26 @@ +{{- if or (and (not .Values.admin.existingSecret) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION)) (and .Values.ldap.enabled (not .Values.ldap.existingSecret)) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "grafana.fullname" . }} + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +{{- with .Values.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} +type: Opaque +data: + {{- if and (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) (not .Values.admin.existingSecret) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) }} + admin-user: {{ .Values.adminUser | b64enc | quote }} + {{- if .Values.adminPassword }} + admin-password: {{ .Values.adminPassword | b64enc | quote }} + {{- else }} + admin-password: {{ template "grafana.password" . }} + {{- end }} + {{- end }} + {{- if not .Values.ldap.existingSecret }} + ldap-toml: {{ tpl .Values.ldap.config $ | b64enc | quote }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/service.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/service.yaml new file mode 100644 index 0000000000..d0a1756c69 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/service.yaml @@ -0,0 +1,55 @@ +{{ if .Values.service.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "grafana.fullname" . }} + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +{{- if .Values.service.labels }} +{{ toYaml .Values.service.labels | indent 4 }} +{{- end }} +{{- $root := . }} +{{- with .Values.service.annotations }} + annotations: +{{ tpl (toYaml . | indent 4) $root }} +{{- end }} +spec: +{{- if (or (eq .Values.service.type "ClusterIP") (empty .Values.service.type)) }} + type: ClusterIP + {{- if .Values.service.clusterIP }} + clusterIP: {{ .Values.service.clusterIP }} + {{end}} +{{- else if eq .Values.service.type "LoadBalancer" }} + type: {{ .Values.service.type }} + {{- if .Values.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.service.loadBalancerIP }} + {{- end }} + {{- if .Values.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: +{{ toYaml .Values.service.loadBalancerSourceRanges | indent 4 }} + {{- end -}} +{{- else }} + type: {{ .Values.service.type }} +{{- end }} +{{- if .Values.service.externalIPs }} + externalIPs: +{{ toYaml .Values.service.externalIPs | indent 4 }} +{{- end }} + ports: + - name: {{ .Values.service.portName }} + port: {{ .Values.service.port }} + protocol: TCP + targetPort: {{ .Values.service.targetPort }} + {{- if .Values.service.appProtocol }} + appProtocol: {{ .Values.service.appProtocol }} + {{- end }} + {{- if (and (eq .Values.service.type "NodePort") (not (empty .Values.service.nodePort))) }} + nodePort: {{.Values.service.nodePort}} + {{ end }} + {{- if .Values.extraExposePorts }} + {{- tpl (toYaml .Values.extraExposePorts) . | nindent 4 }} + {{- end }} + selector: + {{- include "grafana.selectorLabels" . | nindent 4 }} +{{ end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/serviceaccount.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/serviceaccount.yaml new file mode 100644 index 0000000000..4ccee15ed4 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/serviceaccount.yaml @@ -0,0 +1,14 @@ +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + {{- include "grafana.labels" . | nindent 4 }} +{{- $root := . }} +{{- with .Values.serviceAccount.annotations }} + annotations: +{{ tpl (toYaml . | indent 4) $root }} +{{- end }} + name: {{ template "grafana.serviceAccountName" . }} + namespace: {{ template "grafana.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/servicemonitor.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/servicemonitor.yaml new file mode 100644 index 0000000000..31ab6b8894 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/servicemonitor.yaml @@ -0,0 +1,58 @@ +{{- if .Values.serviceMonitor.enabled }} +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "grafana.fullname" . }} + {{- if .Values.serviceMonitor.namespace }} + namespace: {{ tpl .Values.serviceMonitor.namespace . }} + {{- else }} + namespace: {{ template "grafana.namespace" . }} + {{- end }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- if .Values.serviceMonitor.labels }} + {{- toYaml .Values.serviceMonitor.labels | nindent 4 }} + {{- end }} +spec: + endpoints: + - port: {{ .Values.service.portName }} + {{- with .Values.serviceMonitor.interval }} + interval: {{ . }} + {{- end }} + {{- with .Values.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + honorLabels: true + path: {{ .Values.serviceMonitor.path }} + scheme: {{ .Values.serviceMonitor.scheme }} + {{- if .Values.serviceMonitor.tlsConfig }} + tlsConfig: + {{- toYaml .Values.serviceMonitor.tlsConfig | nindent 6 }} + {{- end }} + metricRelabelings: + {{- if .Values.serviceMonitor.metricRelabelings }} + {{- toYaml .Values.serviceMonitor.metricRelabelings | nindent 6 }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName }} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + {{- if .Values.serviceMonitor.relabelings }} + relabelings: + {{- toYaml .Values.serviceMonitor.relabelings | nindent 4 }} + {{- end }} + jobLabel: "{{ .Release.Name }}" + selector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 8 }} + namespaceSelector: + matchNames: + - {{ template "grafana.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/statefulset.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/statefulset.yaml new file mode 100644 index 0000000000..aa6f305e22 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/statefulset.yaml @@ -0,0 +1,56 @@ +{{- if (or (.Values.useStatefulSet) (and .Values.persistence.enabled (not .Values.persistence.existingClaim) (eq .Values.persistence.type "statefulset")))}} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ template "grafana.fullname" . }} + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +{{- with .Values.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} +spec: + replicas: {{ .Values.replicas }} + selector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 6 }} + serviceName: {{ template "grafana.fullname" . }}-headless + template: + metadata: + labels: + {{- include "grafana.selectorLabels" . | nindent 8 }} +{{- with .Values.podLabels }} +{{ toYaml . | indent 8 }} +{{- end }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + checksum/dashboards-json-config: {{ include (print $.Template.BasePath "/dashboards-json-configmap.yaml") . | sha256sum }} + checksum/sc-dashboard-provider-config: {{ include (print $.Template.BasePath "/configmap-dashboard-provider.yaml") . | sha256sum }} + {{- if and (or (and (not .Values.admin.existingSecret) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD)) (and .Values.ldap.enabled (not .Values.ldap.existingSecret))) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} +{{- end }} +{{- with .Values.podAnnotations }} +{{ toYaml . | indent 8 }} +{{- end }} + spec: + {{- include "grafana.pod" . | nindent 6 }} + {{- if .Values.persistence.enabled}} + volumeClaimTemplates: + - metadata: + name: storage + spec: +{{- $_ := required "Must provide at least one access mode for persistent volumes used by Grafana" .Values.persistence.accessModes }} +{{- $_ := required "Must provide at least one access mode for persistent volumes used by Grafana" (first .Values.persistence.accessModes) }} + accessModes: {{ .Values.persistence.accessModes }} + storageClassName: {{ .Values.persistence.storageClassName }} + resources: + requests: + storage: {{ required "Must provide size for persistent volumes used by Grafana" .Values.persistence.size }} + {{- with .Values.persistence.selectorLabels }} + selector: + matchLabels: +{{ toYaml . | indent 10 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/tests/test-configmap.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/tests/test-configmap.yaml new file mode 100644 index 0000000000..ff53aaf1b3 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/tests/test-configmap.yaml @@ -0,0 +1,17 @@ +{{- if .Values.testFramework.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "grafana.fullname" . }}-test + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +data: + run.sh: |- + @test "Test Health" { + url="http://{{ template "grafana.fullname" . }}/api/health" + + code=$(wget --server-response --spider --timeout 10 --tries 1 ${url} 2>&1 | awk '/^ HTTP/{print $2}') + [ "$code" == "200" ] + } +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/tests/test-podsecuritypolicy.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/tests/test-podsecuritypolicy.yaml new file mode 100644 index 0000000000..5dd736efcb --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/tests/test-podsecuritypolicy.yaml @@ -0,0 +1,29 @@ +{{- if and .Values.testFramework.enabled .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "grafana.fullname" . }}-test + labels: + {{- include "grafana.labels" . | nindent 4 }} +spec: + allowPrivilegeEscalation: true + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + fsGroup: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + runAsUser: + rule: RunAsAny + volumes: + - configMap + - downwardAPI + - emptyDir + - projected + - csi + - secret +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/tests/test-role.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/tests/test-role.yaml new file mode 100644 index 0000000000..ea2f8c6b7c --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/tests/test-role.yaml @@ -0,0 +1,14 @@ +{{- if and .Values.testFramework.enabled .Values.global.cattle.psp.enabled -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ template "grafana.fullname" . }}-test + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +rules: +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: [{{ template "grafana.fullname" . }}-test] +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/tests/test-rolebinding.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/tests/test-rolebinding.yaml new file mode 100644 index 0000000000..7eda26512b --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/tests/test-rolebinding.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.testFramework.enabled .Values.global.cattle.psp.enabled -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ template "grafana.fullname" . }}-test + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ template "grafana.fullname" . }}-test +subjects: +- kind: ServiceAccount + name: {{ template "grafana.serviceAccountNameTest" . }} + namespace: {{ template "grafana.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/tests/test-serviceaccount.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/tests/test-serviceaccount.yaml new file mode 100644 index 0000000000..5c33507337 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/tests/test-serviceaccount.yaml @@ -0,0 +1,9 @@ +{{- if and .Values.testFramework.enabled .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + {{- include "grafana.labels" . | nindent 4 }} + name: {{ template "grafana.serviceAccountNameTest" . }} + namespace: {{ template "grafana.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/tests/test.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/tests/test.yaml new file mode 100644 index 0000000000..3a84fbe000 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/templates/tests/test.yaml @@ -0,0 +1,51 @@ +{{- if .Values.testFramework.enabled }} +apiVersion: v1 +kind: Pod +metadata: + name: {{ template "grafana.fullname" . }}-test + labels: + {{- include "grafana.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test-success + "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" + namespace: {{ template "grafana.namespace" . }} +spec: + serviceAccountName: {{ template "grafana.serviceAccountNameTest" . }} + {{- if .Values.testFramework.securityContext }} + securityContext: {{ toYaml .Values.testFramework.securityContext | nindent 4 }} + {{- end }} + {{- $root := . }} + {{- if .Values.image.pullSecrets }} + imagePullSecrets: + {{- range .Values.image.pullSecrets }} + - name: {{ tpl . $root }} + {{- end}} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 4 }} + {{- if .Values.nodeSelector }} +{{ toYaml .Values.nodeSelector | indent 4 }} + {{- end }} + {{- $root := . }} + {{- with .Values.affinity }} + affinity: +{{ tpl (toYaml .) $root | indent 4 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 4 }} +{{- if .Values.tolerations }} +{{ toYaml .Values.tolerations | indent 4 }} +{{- end }} + containers: + - name: {{ .Release.Name }}-test + image: "{{ template "system_default_registry" . }}{{ .Values.testFramework.image}}:{{ .Values.testFramework.tag }}" + imagePullPolicy: "{{ .Values.testFramework.imagePullPolicy}}" + command: ["/opt/bats/bin/bats", "-t", "/tests/run.sh"] + volumeMounts: + - mountPath: /tests + name: tests + readOnly: true + volumes: + - name: tests + configMap: + name: {{ template "grafana.fullname" . }}-test + restartPolicy: Never +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/values.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/values.yaml new file mode 100644 index 0000000000..aa0a8aef86 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/grafana/values.yaml @@ -0,0 +1,1088 @@ +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + +rbac: + create: true + ## Use an existing ClusterRole/Role (depending on rbac.namespaced false/true) + # useExistingRole: name-of-some-(cluster)role + pspAnnotations: {} + ## Specify pod annotations + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#apparmor + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#seccomp + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#sysctl + ## + # seccomp.security.alpha.kubernetes.io/allowedProfileNames: 'docker/default,runtime/default' + # seccomp.security.alpha.kubernetes.io/defaultProfileName: 'docker/default' + # apparmor.security.beta.kubernetes.io/allowedProfileNames: 'runtime/default' + # apparmor.security.beta.kubernetes.io/defaultProfileName: 'runtime/default' + + namespaced: false + extraRoleRules: [] + # - apiGroups: [] + # resources: [] + # verbs: [] + extraClusterRoleRules: [] + # - apiGroups: [] + # resources: [] + # verbs: [] +serviceAccount: + create: true + name: + nameTest: +## Service account annotations. Can be templated. +# annotations: +# eks.amazonaws.com/role-arn: arn:aws:iam::123456789000:role/iam-role-name-here + autoMount: true + +replicas: 1 + +## Create a headless service for the deployment +headlessService: false + +## Create HorizontalPodAutoscaler object for deployment type +# +autoscaling: + enabled: false +# minReplicas: 1 +# maxReplicas: 10 +# metrics: +# - type: Resource +# resource: +# name: cpu +# targetAverageUtilization: 60 +# - type: Resource +# resource: +# name: memory +# targetAverageUtilization: 60 + +## See `kubectl explain poddisruptionbudget.spec` for more +## ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/ +podDisruptionBudget: {} +# minAvailable: 1 +# maxUnavailable: 1 + +## See `kubectl explain deployment.spec.strategy` for more +## ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy +deploymentStrategy: + type: RollingUpdate + +readinessProbe: + httpGet: + path: /api/health + port: 3000 + +livenessProbe: + httpGet: + path: /api/health + port: 3000 + initialDelaySeconds: 60 + timeoutSeconds: 30 + failureThreshold: 10 + +## Use an alternate scheduler, e.g. "stork". +## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/ +## +# schedulerName: "default-scheduler" + +image: + repository: rancher/mirrored-grafana-grafana + # Overrides the Grafana image tag whose default is the chart appVersion + tag: 9.1.5 + sha: "" + pullPolicy: IfNotPresent + + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## Can be templated. + ## + # pullSecrets: + # - myRegistrKeySecretName + +testFramework: + enabled: false + image: "rancher/mirrored-bats-bats" + tag: "v1.4.1" + imagePullPolicy: IfNotPresent + securityContext: + runAsNonRoot: true + runAsUser: 1000 + +securityContext: + runAsNonRoot: true + runAsUser: 472 + runAsGroup: 472 + fsGroup: 472 + +containerSecurityContext: + {} + +# Enable creating the grafana configmap +createConfigmap: true + +# Extra configmaps to mount in grafana pods +# Values are templated. +extraConfigmapMounts: [] + # - name: certs-configmap + # mountPath: /etc/grafana/ssl/ + # subPath: certificates.crt # (optional) + # configMap: certs-configmap + # readOnly: true + + +extraEmptyDirMounts: [] + # - name: provisioning-notifiers + # mountPath: /etc/grafana/provisioning/notifiers + + +# Apply extra labels to common labels. +extraLabels: {} + +## Assign a PriorityClassName to pods if set +# priorityClassName: + +downloadDashboardsImage: + repository: rancher/mirrored-curlimages-curl + tag: 7.85.0 + sha: "" + pullPolicy: IfNotPresent + +downloadDashboards: + env: {} + envFromSecret: "" + resources: {} + securityContext: {} + +## Pod Annotations +# podAnnotations: {} + +## Pod Labels +# podLabels: {} + +podPortName: grafana + +## Deployment annotations +# annotations: {} + +## Expose the grafana service to be accessed from outside the cluster (LoadBalancer service). +## or access it from within the cluster (ClusterIP service). Set the service type and the port to serve it. +## ref: http://kubernetes.io/docs/user-guide/services/ +## +service: + enabled: true + type: ClusterIP + port: 80 + targetPort: 3000 + # targetPort: 4181 To be used with a proxy extraContainer + ## Service annotations. Can be templated. + annotations: {} + labels: {} + portName: service + # Adds the appProtocol field to the service. This allows to work with istio protocol selection. Ex: "http" or "tcp" + appProtocol: "" + +serviceMonitor: + ## If true, a ServiceMonitor CRD is created for a prometheus operator + ## https://github.com/coreos/prometheus-operator + ## + enabled: false + path: /metrics + # namespace: monitoring (defaults to use the namespace this chart is deployed to) + labels: {} + interval: 1m + scheme: http + tlsConfig: {} + scrapeTimeout: 30s + relabelings: [] + +extraExposePorts: [] + # - name: keycloak + # port: 8080 + # targetPort: 8080 + # type: ClusterIP + +# overrides pod.spec.hostAliases in the grafana deployment's pods +hostAliases: [] + # - ip: "1.2.3.4" + # hostnames: + # - "my.host.com" + +ingress: + enabled: false + # For Kubernetes >= 1.18 you should specify the ingress-controller via the field ingressClassName + # See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#specifying-the-class-of-an-ingress + # ingressClassName: nginx + # Values can be templated + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + labels: {} + path: / + + # pathType is only for k8s >= 1.18 + pathType: Prefix + + hosts: + - chart-example.local + ## Extra paths to prepend to every host configuration. This is useful when working with annotation based services. + extraPaths: [] + # - path: /* + # backend: + # serviceName: ssl-redirect + # servicePort: use-annotation + ## Or for k8s > 1.19 + # - path: /* + # pathType: Prefix + # backend: + # service: + # name: ssl-redirect + # port: + # name: use-annotation + + + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: {} +# limits: +# cpu: 100m +# memory: 128Mi +# requests: +# cpu: 100m +# memory: 128Mi + +## Node labels for pod assignment +## ref: https://kubernetes.io/docs/user-guide/node-selection/ +# +nodeSelector: {} + +## Tolerations for pod assignment +## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +## +tolerations: [] + +## Affinity for pod assignment (evaluated as template) +## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity +## +affinity: {} + +## Topology Spread Constraints +## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ +## +topologySpreadConstraints: [] + +## Additional init containers (evaluated as template) +## ref: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ +## +extraInitContainers: [] + +## Enable an Specify container in extraContainers. This is meant to allow adding an authentication proxy to a grafana pod +extraContainers: "" +# extraContainers: | +# - name: proxy +# image: quay.io/gambol99/keycloak-proxy:latest +# args: +# - -provider=github +# - -client-id= +# - -client-secret= +# - -github-org= +# - -email-domain=* +# - -cookie-secret= +# - -http-address=http://0.0.0.0:4181 +# - -upstream-url=http://127.0.0.1:3000 +# ports: +# - name: proxy-web +# containerPort: 4181 + +## Volumes that can be used in init containers that will not be mounted to deployment pods +extraContainerVolumes: [] +# - name: volume-from-secret +# secret: +# secretName: secret-to-mount +# - name: empty-dir-volume +# emptyDir: {} + +## Enable persistence using Persistent Volume Claims +## ref: http://kubernetes.io/docs/user-guide/persistent-volumes/ +## +persistence: + type: pvc + enabled: false + # storageClassName: default + accessModes: + - ReadWriteOnce + size: 10Gi + # annotations: {} + finalizers: + - kubernetes.io/pvc-protection + # selectorLabels: {} + ## Sub-directory of the PV to mount. Can be templated. + # subPath: "" + ## Name of an existing PVC. Can be templated. + # existingClaim: + + ## If persistence is not enabled, this allows to mount the + ## local storage in-memory to improve performance + ## + inMemory: + enabled: false + ## The maximum usage on memory medium EmptyDir would be + ## the minimum value between the SizeLimit specified + ## here and the sum of memory limits of all containers in a pod + ## + # sizeLimit: 300Mi + +initChownData: + ## If false, data ownership will not be reset at startup + ## This allows the prometheus-server to be run with an arbitrary user + ## + enabled: true + + ## initChownData container image + ## + image: + repository: rancher/mirrored-library-busybox + tag: "1.31.1" + sha: "" + pullPolicy: IfNotPresent + + ## initChownData resource requests and limits + ## Ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + +# Administrator credentials when not using an existing secret (see below) +adminUser: admin +# adminPassword: strongpassword + +# Use an existing secret for the admin user. +admin: + ## Name of the secret. Can be templated. + existingSecret: "" + userKey: admin-user + passwordKey: admin-password + +## Define command to be executed at startup by grafana container +## Needed if using `vault-env` to manage secrets (ref: https://banzaicloud.com/blog/inject-secrets-into-pods-vault/) +## Default is "run.sh" as defined in grafana's Dockerfile +# command: +# - "sh" +# - "/run.sh" + +## Use an alternate scheduler, e.g. "stork". +## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/ +## +# schedulerName: + +## Extra environment variables that will be pass onto deployment pods +## +## to provide grafana with access to CloudWatch on AWS EKS: +## 1. create an iam role of type "Web identity" with provider oidc.eks.* (note the provider for later) +## 2. edit the "Trust relationships" of the role, add a line inside the StringEquals clause using the +## same oidc eks provider as noted before (same as the existing line) +## also, replace NAMESPACE and prometheus-operator-grafana with the service account namespace and name +## +## "oidc.eks.us-east-1.amazonaws.com/id/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX:sub": "system:serviceaccount:NAMESPACE:prometheus-operator-grafana", +## +## 3. attach a policy to the role, you can use a built in policy called CloudWatchReadOnlyAccess +## 4. use the following env: (replace 123456789000 and iam-role-name-here with your aws account number and role name) +## +## env: +## AWS_ROLE_ARN: arn:aws:iam::123456789000:role/iam-role-name-here +## AWS_WEB_IDENTITY_TOKEN_FILE: /var/run/secrets/eks.amazonaws.com/serviceaccount/token +## AWS_REGION: us-east-1 +## +## 5. uncomment the EKS section in extraSecretMounts: below +## 6. uncomment the annotation section in the serviceAccount: above +## make sure to replace arn:aws:iam::123456789000:role/iam-role-name-here with your role arn + +env: {} + +## "valueFrom" environment variable references that will be added to deployment pods. Name is templated. +## ref: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#envvarsource-v1-core +## Renders in container spec as: +## env: +## ... +## - name: +## valueFrom: +## +envValueFrom: {} + # ENV_NAME: + # configMapKeyRef: + # name: configmap-name + # key: value_key + +## The name of a secret in the same kubernetes namespace which contain values to be added to the environment +## This can be useful for auth tokens, etc. Value is templated. +envFromSecret: "" + +## Sensible environment variables that will be rendered as new secret object +## This can be useful for auth tokens, etc +envRenderSecret: {} + +## The names of secrets in the same kubernetes namespace which contain values to be added to the environment +## Each entry should contain a name key, and can optionally specify whether the secret must be defined with an optional key. +## Name is templated. +envFromSecrets: [] +## - name: secret-name +## optional: true + +## The names of conifgmaps in the same kubernetes namespace which contain values to be added to the environment +## Each entry should contain a name key, and can optionally specify whether the configmap must be defined with an optional key. +## Name is templated. +## ref: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.23/#configmapenvsource-v1-core +envFromConfigMaps: [] +## - name: configmap-name +## optional: true + +# Inject Kubernetes services as environment variables. +# See https://kubernetes.io/docs/concepts/services-networking/connect-applications-service/#environment-variables +enableServiceLinks: true + +## Additional grafana server secret mounts +# Defines additional mounts with secrets. Secrets must be manually created in the namespace. +extraSecretMounts: [] + # - name: secret-files + # mountPath: /etc/secrets + # secretName: grafana-secret-files + # readOnly: true + # subPath: "" + # + # for AWS EKS (cloudwatch) use the following (see also instruction in env: above) + # - name: aws-iam-token + # mountPath: /var/run/secrets/eks.amazonaws.com/serviceaccount + # readOnly: true + # projected: + # defaultMode: 420 + # sources: + # - serviceAccountToken: + # audience: sts.amazonaws.com + # expirationSeconds: 86400 + # path: token + # + # for CSI e.g. Azure Key Vault use the following + # - name: secrets-store-inline + # mountPath: /run/secrets + # readOnly: true + # csi: + # driver: secrets-store.csi.k8s.io + # readOnly: true + # volumeAttributes: + # secretProviderClass: "akv-grafana-spc" + # nodePublishSecretRef: # Only required when using service principal mode + # name: grafana-akv-creds # Only required when using service principal mode + +## Additional grafana server volume mounts +# Defines additional volume mounts. +extraVolumeMounts: [] + # - name: extra-volume-0 + # mountPath: /mnt/volume0 + # readOnly: true + # existingClaim: volume-claim + # - name: extra-volume-1 + # mountPath: /mnt/volume1 + # readOnly: true + # hostPath: /usr/shared/ + # - name: grafana-secrets + # csi: true + # data: + # driver: secrets-store.csi.k8s.io + # readOnly: true + # volumeAttributes: + # secretProviderClass: "grafana-env-spc" + +## Container Lifecycle Hooks. Execute a specific bash command or make an HTTP request +lifecycleHooks: {} + # postStart: + # exec: + # command: [] + +## Pass the plugins you want installed as a list. +## +plugins: [] + # - digrich-bubblechart-panel + # - grafana-clock-panel + +## Configure grafana datasources +## ref: http://docs.grafana.org/administration/provisioning/#datasources +## +datasources: {} +# datasources.yaml: +# apiVersion: 1 +# datasources: +# - name: Prometheus +# type: prometheus +# url: http://prometheus-prometheus-server +# access: proxy +# isDefault: true +# - name: CloudWatch +# type: cloudwatch +# access: proxy +# uid: cloudwatch +# editable: false +# jsonData: +# authType: default +# defaultRegion: us-east-1 + +## Configure grafana alerting (can be templated) +## ref: http://docs.grafana.org/administration/provisioning/#alerting +## +alerting: {} + # rules.yaml: + # apiVersion: 1 + # groups: + # - orgId: 1 + # name: '{{ .Chart.Name }}_my_rule_group' + # folder: my_first_folder + # interval: 60s + # rules: + # - uid: my_id_1 + # title: my_first_rule + # condition: A + # data: + # - refId: A + # datasourceUid: '-100' + # model: + # conditions: + # - evaluator: + # params: + # - 3 + # type: gt + # operator: + # type: and + # query: + # params: + # - A + # reducer: + # type: last + # type: query + # datasource: + # type: __expr__ + # uid: '-100' + # expression: 1==0 + # intervalMs: 1000 + # maxDataPoints: 43200 + # refId: A + # type: math + # dashboardUid: my_dashboard + # panelId: 123 + # noDataState: Alerting + # for: 60s + # annotations: + # some_key: some_value + # labels: + # team: sre_team_1 + # contactpoints.yaml: + # apiVersion: 1 + # contactPoints: + # - orgId: 1 + # name: cp_1 + # receivers: + # - uid: first_uid + # type: pagerduty + # settings: + # integrationKey: XXX + # severity: critical + # class: ping failure + # component: Grafana + # group: app-stack + # summary: | + # {{ `{{ template "default.message" . }}` }} + +## Configure notifiers +## ref: http://docs.grafana.org/administration/provisioning/#alert-notification-channels +## +notifiers: {} +# notifiers.yaml: +# notifiers: +# - name: email-notifier +# type: email +# uid: email1 +# # either: +# org_id: 1 +# # or +# org_name: Main Org. +# is_default: true +# settings: +# addresses: an_email_address@example.com +# delete_notifiers: + +## Configure grafana dashboard providers +## ref: http://docs.grafana.org/administration/provisioning/#dashboards +## +## `path` must be /var/lib/grafana/dashboards/ +## +dashboardProviders: {} +# dashboardproviders.yaml: +# apiVersion: 1 +# providers: +# - name: 'default' +# orgId: 1 +# folder: '' +# type: file +# disableDeletion: false +# editable: true +# options: +# path: /var/lib/grafana/dashboards/default + +## Configure grafana dashboard to import +## NOTE: To use dashboards you must also enable/configure dashboardProviders +## ref: https://grafana.com/dashboards +## +## dashboards per provider, use provider name as key. +## +dashboards: {} + # default: + # some-dashboard: + # json: | + # $RAW_JSON + # custom-dashboard: + # file: dashboards/custom-dashboard.json + # prometheus-stats: + # gnetId: 2 + # revision: 2 + # datasource: Prometheus + # local-dashboard: + # url: https://example.com/repository/test.json + # token: '' + # local-dashboard-base64: + # url: https://example.com/repository/test-b64.json + # token: '' + # b64content: true + # local-dashboard-gitlab: + # url: https://example.com/repository/test-gitlab.json + # gitlabToken: '' + # local-dashboard-bitbucket: + # url: https://example.com/repository/test-bitbucket.json + # bearerToken: '' + +## Reference to external ConfigMap per provider. Use provider name as key and ConfigMap name as value. +## A provider dashboards must be defined either by external ConfigMaps or in values.yaml, not in both. +## ConfigMap data example: +## +## data: +## example-dashboard.json: | +## RAW_JSON +## +dashboardsConfigMaps: {} +# default: "" + +## Grafana's primary configuration +## NOTE: values in map will be converted to ini format +## ref: http://docs.grafana.org/installation/configuration/ +## +grafana.ini: + paths: + data: /var/lib/grafana/ + logs: /var/log/grafana + plugins: /var/lib/grafana/plugins + provisioning: /etc/grafana/provisioning + analytics: + check_for_updates: true + log: + mode: console + grafana_net: + url: https://grafana.net + server: + domain: "{{ if (and .Values.ingress.enabled .Values.ingress.hosts) }}{{ .Values.ingress.hosts | first }}{{ end }}" +## grafana Authentication can be enabled with the following values on grafana.ini + # server: + # The full public facing url you use in browser, used for redirects and emails + # root_url: + # https://grafana.com/docs/grafana/latest/auth/github/#enable-github-in-grafana + # auth.github: + # enabled: false + # allow_sign_up: false + # scopes: user:email,read:org + # auth_url: https://github.com/login/oauth/authorize + # token_url: https://github.com/login/oauth/access_token + # api_url: https://api.github.com/user + # team_ids: + # allowed_organizations: + # client_id: + # client_secret: +## LDAP Authentication can be enabled with the following values on grafana.ini +## NOTE: Grafana will fail to start if the value for ldap.toml is invalid + # auth.ldap: + # enabled: true + # allow_sign_up: true + # config_file: /etc/grafana/ldap.toml + +## Grafana's LDAP configuration +## Templated by the template in _helpers.tpl +## NOTE: To enable the grafana.ini must be configured with auth.ldap.enabled +## ref: http://docs.grafana.org/installation/configuration/#auth-ldap +## ref: http://docs.grafana.org/installation/ldap/#configuration +ldap: + enabled: false + # `existingSecret` is a reference to an existing secret containing the ldap configuration + # for Grafana in a key `ldap-toml`. + existingSecret: "" + # `config` is the content of `ldap.toml` that will be stored in the created secret + config: "" + # config: |- + # verbose_logging = true + + # [[servers]] + # host = "my-ldap-server" + # port = 636 + # use_ssl = true + # start_tls = false + # ssl_skip_verify = false + # bind_dn = "uid=%s,ou=users,dc=myorg,dc=com" + +## Grafana's SMTP configuration +## NOTE: To enable, grafana.ini must be configured with smtp.enabled +## ref: http://docs.grafana.org/installation/configuration/#smtp +smtp: + # `existingSecret` is a reference to an existing secret containing the smtp configuration + # for Grafana. + existingSecret: "" + userKey: "user" + passwordKey: "password" + +## Sidecars that collect the configmaps with specified label and stores the included files them into the respective folders +## Requires at least Grafana 5 to work and can't be used together with parameters dashboardProviders, datasources and dashboards +sidecar: + image: + repository: rancher/mirrored-kiwigrid-k8s-sidecar + tag: 1.19.2 + sha: "" + imagePullPolicy: IfNotPresent + resources: {} +# limits: +# cpu: 100m +# memory: 100Mi +# requests: +# cpu: 50m +# memory: 50Mi + securityContext: {} + # skipTlsVerify Set to true to skip tls verification for kube api calls + # skipTlsVerify: true + enableUniqueFilenames: false + readinessProbe: {} + livenessProbe: {} + # Log level default for all sidecars. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. Defaults to INFO + # logLevel: INFO + dashboards: + enabled: false + # Additional environment variables for the dashboards sidecar + env: {} + # Do not reprocess already processed unchanged resources on k8s API reconnect. + # ignoreAlreadyProcessed: true + SCProvider: true + # label that the configmaps with dashboards are marked with + label: grafana_dashboard + # value of label that the configmaps with dashboards are set to + labelValue: "" + # Log level. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. + # logLevel: INFO + # folder in the pod that should hold the collected dashboards (unless `defaultFolderName` is set) + folder: /tmp/dashboards + # The default folder name, it will create a subfolder under the `folder` and put dashboards in there instead + defaultFolderName: null + # Namespaces list. If specified, the sidecar will search for config-maps/secrets inside these namespaces. + # Otherwise the namespace in which the sidecar is running will be used. + # It's also possible to specify ALL to search in all namespaces. + searchNamespace: null + # Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. + watchMethod: WATCH + # search in configmap, secret or both + resource: both + # If specified, the sidecar will look for annotation with this name to create folder and put graph here. + # You can use this parameter together with `provider.foldersFromFilesStructure`to annotate configmaps and create folder structure. + folderAnnotation: null + # Absolute path to shell script to execute after a configmap got reloaded + script: null + # watchServerTimeout: request to the server, asking it to cleanly close the connection after that. + # defaults to 60sec; much higher values like 3600 seconds (1h) are feasible for non-Azure K8S + # watchServerTimeout: 3600 + # + # watchClientTimeout: is a client-side timeout, configuring your local socket. + # If you have a network outage dropping all packets with no RST/FIN, + # this is how long your client waits before realizing & dropping the connection. + # defaults to 66sec (sic!) + # watchClientTimeout: 60 + # + # provider configuration that lets grafana manage the dashboards + provider: + # name of the provider, should be unique + name: sidecarProvider + # orgid as configured in grafana + orgid: 1 + # folder in which the dashboards should be imported in grafana + folder: '' + # type of the provider + type: file + # disableDelete to activate a import-only behaviour + disableDelete: false + # allow updating provisioned dashboards from the UI + allowUiUpdates: false + # allow Grafana to replicate dashboard structure from filesystem + foldersFromFilesStructure: false + # Additional dashboard sidecar volume mounts + extraMounts: [] + # Sets the size limit of the dashboard sidecar emptyDir volume + sizeLimit: {} + datasources: + enabled: false + # Additional environment variables for the datasourcessidecar + env: {} + # Do not reprocess already processed unchanged resources on k8s API reconnect. + # ignoreAlreadyProcessed: true + # label that the configmaps with datasources are marked with + label: grafana_datasource + # value of label that the configmaps with datasources are set to + labelValue: "" + # Log level. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. + # logLevel: INFO + # If specified, the sidecar will search for datasource config-maps inside this namespace. + # Otherwise the namespace in which the sidecar is running will be used. + # It's also possible to specify ALL to search in all namespaces + searchNamespace: null + # Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. + watchMethod: WATCH + # search in configmap, secret or both + resource: both + # watchServerTimeout: request to the server, asking it to cleanly close the connection after that. + # defaults to 60sec; much higher values like 3600 seconds (1h) are feasible for non-Azure K8S + # watchServerTimeout: 3600 + # + # watchClientTimeout: is a client-side timeout, configuring your local socket. + # If you have a network outage dropping all packets with no RST/FIN, + # this is how long your client waits before realizing & dropping the connection. + # defaults to 66sec (sic!) + # watchClientTimeout: 60 + # + # Endpoint to send request to reload datasources + reloadURL: "http://localhost:3000/api/admin/provisioning/datasources/reload" + # Absolute path to shell script to execute after a datasource got reloaded + script: null + skipReload: true + # Deploy the datasource sidecar as an initContainer in addition to a container. + # This is needed if skipReload is true, to load any datasources defined at startup time. + initDatasources: true + # Sets the size limit of the datasource sidecar emptyDir volume + sizeLimit: {} + plugins: + enabled: false + # Additional environment variables for the plugins sidecar + env: {} + # Do not reprocess already processed unchanged resources on k8s API reconnect. + # ignoreAlreadyProcessed: true + # label that the configmaps with plugins are marked with + label: grafana_plugin + # value of label that the configmaps with plugins are set to + labelValue: "" + # Log level. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. + # logLevel: INFO + # If specified, the sidecar will search for plugin config-maps inside this namespace. + # Otherwise the namespace in which the sidecar is running will be used. + # It's also possible to specify ALL to search in all namespaces + searchNamespace: null + # Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. + watchMethod: WATCH + # search in configmap, secret or both + resource: both + # watchServerTimeout: request to the server, asking it to cleanly close the connection after that. + # defaults to 60sec; much higher values like 3600 seconds (1h) are feasible for non-Azure K8S + # watchServerTimeout: 3600 + # + # watchClientTimeout: is a client-side timeout, configuring your local socket. + # If you have a network outage dropping all packets with no RST/FIN, + # this is how long your client waits before realizing & dropping the connection. + # defaults to 66sec (sic!) + # watchClientTimeout: 60 + # + # Endpoint to send request to reload plugins + reloadURL: "http://localhost:3000/api/admin/provisioning/plugins/reload" + # Absolute path to shell script to execute after a plugin got reloaded + script: null + skipReload: false + # Deploy the datasource sidecar as an initContainer in addition to a container. + # This is needed if skipReload is true, to load any plugins defined at startup time. + initPlugins: false + # Sets the size limit of the plugin sidecar emptyDir volume + sizeLimit: {} + notifiers: + enabled: false + # Additional environment variables for the notifierssidecar + env: {} + # Do not reprocess already processed unchanged resources on k8s API reconnect. + # ignoreAlreadyProcessed: true + # label that the configmaps with notifiers are marked with + label: grafana_notifier + # value of label that the configmaps with notifiers are set to + labelValue: "" + # Log level. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. + # logLevel: INFO + # If specified, the sidecar will search for notifier config-maps inside this namespace. + # Otherwise the namespace in which the sidecar is running will be used. + # It's also possible to specify ALL to search in all namespaces + searchNamespace: null + # search in configmap, secret or both + resource: both + # Sets the size limit of the notifier sidecar emptyDir volume + sizeLimit: {} + +## Override the deployment namespace +## +namespaceOverride: "" + +## Number of old ReplicaSets to retain +## +revisionHistoryLimit: 10 + +## Add a seperate remote image renderer deployment/service +imageRenderer: + # Enable the image-renderer deployment & service + enabled: false + replicas: 1 + image: + # image-renderer Image repository + repository: rancher/mirrored-grafana-grafana-image-renderer + # image-renderer Image tag + tag: 3.0.1 + # image-renderer Image sha (optional) + sha: "" + # image-renderer ImagePullPolicy + pullPolicy: Always + # extra environment variables + env: + HTTP_HOST: "0.0.0.0" + # RENDERING_ARGS: --no-sandbox,--disable-gpu,--window-size=1280x758 + # RENDERING_MODE: clustered + # IGNORE_HTTPS_ERRORS: true + # image-renderer deployment serviceAccount + serviceAccountName: "" + # image-renderer deployment securityContext + securityContext: {} + # image-renderer deployment Host Aliases + hostAliases: [] + # image-renderer deployment priority class + priorityClassName: '' + service: + # Enable the image-renderer service + enabled: true + # image-renderer service port name + portName: 'http' + # image-renderer service port used by both service and deployment + port: 8081 + targetPort: 8081 + # Adds the appProtocol field to the image-renderer service. This allows to work with istio protocol selection. Ex: "http" or "tcp" + appProtocol: "" + # If https is enabled in Grafana, this needs to be set as 'https' to correctly configure the callback used in Grafana + grafanaProtocol: http + # In case a sub_path is used this needs to be added to the image renderer callback + grafanaSubPath: "" + # name of the image-renderer port on the pod + podPortName: http + # number of image-renderer replica sets to keep + revisionHistoryLimit: 10 + networkPolicy: + # Enable a NetworkPolicy to limit inbound traffic to only the created grafana pods + limitIngress: true + # Enable a NetworkPolicy to limit outbound traffic to only the created grafana pods + limitEgress: false + resources: {} +# limits: +# cpu: 100m +# memory: 100Mi +# requests: +# cpu: 50m +# memory: 50Mi + ## Node labels for pod assignment + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + # + nodeSelector: {} + + ## Tolerations for pod assignment + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + + ## Affinity for pod assignment (evaluated as template) + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity + ## + affinity: {} + +networkPolicy: + ## @param networkPolicy.enabled Enable creation of NetworkPolicy resources. Only Ingress traffic is filtered for now. + ## + enabled: false + ## @param networkPolicy.allowExternal Don't require client label for connections + ## The Policy model to apply. When set to false, only pods with the correct + ## client label will have network access to grafana port defined. + ## When true, grafana will accept connections from any source + ## (with the correct destination port). + ## + ingress: true + ## @param networkPolicy.ingress When true enables the creation + ## an ingress network policy + ## + allowExternal: true + ## @param networkPolicy.explicitNamespacesSelector A Kubernetes LabelSelector to explicitly select namespaces from which traffic could be allowed + ## If explicitNamespacesSelector is missing or set to {}, only client Pods that are in the networkPolicy's namespace + ## and that match other criteria, the ones that have the good label, can reach the grafana. + ## But sometimes, we want the grafana to be accessible to clients from other namespaces, in this case, we can use this + ## LabelSelector to select these namespaces, note that the networkPolicy's namespace should also be explicitly added. + ## + ## Example: + ## explicitNamespacesSelector: + ## matchLabels: + ## role: frontend + ## matchExpressions: + ## - {key: role, operator: In, values: [frontend]} + ## + explicitNamespacesSelector: {} + ## + ## + ## + ## + ## + ## + egress: + ## @param networkPolicy.egress.enabled When enabled, an egress network policy will be + ## created allowing grafana to connect to external data sources from kubernetes cluster. + enabled: false + ## + ## @param networkPolicy.egress.ports Add individual ports to be allowed by the egress + ports: [] + ## Add ports to the egress by specifying - port: + ## E.X. + ## ports: + ## - port: 80 + ## - port: 443 + ## + ## + ## + ## + ## + ## + +# Enable backward compatibility of kubernetes where version below 1.13 doesn't have the enableServiceLinks option +enableKubeBackwardCompatibility: false +useStatefulSet: false +# Create a dynamic manifests via values: +extraObjects: [] + # - apiVersion: "kubernetes-client.io/v1" + # kind: ExternalSecret + # metadata: + # name: grafana-secrets + # spec: + # backendType: gcpSecretsManager + # data: + # - key: grafana-admin-password + # name: adminPassword diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedKubelet/.helmignore b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedKubelet/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedKubelet/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedKubelet/Chart.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedKubelet/Chart.yaml new file mode 100644 index 0000000000..03c7b37a86 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedKubelet/Chart.yaml @@ -0,0 +1,14 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.16.0-0 < 1.28.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +name: hardenedKubelet +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedKubelet/README.md b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedKubelet/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedKubelet/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedKubelet/templates/_helpers.tpl b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedKubelet/templates/_helpers.tpl new file mode 100644 index 0000000000..8e651dccfd --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedKubelet/templates/_helpers.tpl @@ -0,0 +1,166 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $forceHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- end }} +{{- $metricRelabelings := gt (len (keys $clusterNameRelabel)) 0 | ternary (append ($metricRelabelings) ($clusterNameRelabel)) ($metricRelabelings) }} +{{- $metricRelabelings := gt (len (keys $clusterIdRelabel)) 0 | ternary (append ($metricRelabelings) ($clusterIdRelabel)) ($metricRelabelings) }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $forceHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedKubelet/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedKubelet/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedKubelet/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedKubelet/templates/pushprox-clients.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedKubelet/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedKubelet/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedKubelet/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedKubelet/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedKubelet/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedKubelet/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedKubelet/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedKubelet/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedKubelet/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedKubelet/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedKubelet/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedKubelet/templates/validate-install-crd.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedKubelet/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedKubelet/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedKubelet/templates/validate-psp-install.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedKubelet/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedKubelet/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedKubelet/values.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedKubelet/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedKubelet/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedNodeExporter/.helmignore b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedNodeExporter/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedNodeExporter/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedNodeExporter/Chart.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedNodeExporter/Chart.yaml new file mode 100644 index 0000000000..dfa8b78d89 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedNodeExporter/Chart.yaml @@ -0,0 +1,14 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.16.0-0 < 1.28.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +name: hardenedNodeExporter +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedNodeExporter/README.md b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedNodeExporter/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedNodeExporter/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedNodeExporter/templates/_helpers.tpl b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedNodeExporter/templates/_helpers.tpl new file mode 100644 index 0000000000..8e651dccfd --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedNodeExporter/templates/_helpers.tpl @@ -0,0 +1,166 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $forceHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- end }} +{{- $metricRelabelings := gt (len (keys $clusterNameRelabel)) 0 | ternary (append ($metricRelabelings) ($clusterNameRelabel)) ($metricRelabelings) }} +{{- $metricRelabelings := gt (len (keys $clusterIdRelabel)) 0 | ternary (append ($metricRelabelings) ($clusterIdRelabel)) ($metricRelabelings) }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $forceHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedNodeExporter/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedNodeExporter/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedNodeExporter/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedNodeExporter/templates/pushprox-clients.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedNodeExporter/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedNodeExporter/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedNodeExporter/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedNodeExporter/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedNodeExporter/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedNodeExporter/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedNodeExporter/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedNodeExporter/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedNodeExporter/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedNodeExporter/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedNodeExporter/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedNodeExporter/templates/validate-install-crd.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedNodeExporter/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedNodeExporter/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedNodeExporter/templates/validate-psp-install.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedNodeExporter/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedNodeExporter/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedNodeExporter/values.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedNodeExporter/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/hardenedNodeExporter/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/k3sServer/.helmignore b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/k3sServer/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/k3sServer/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/k3sServer/Chart.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/k3sServer/Chart.yaml new file mode 100644 index 0000000000..3e4dd7d9c6 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/k3sServer/Chart.yaml @@ -0,0 +1,14 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.16.0-0 < 1.28.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +name: k3sServer +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/k3sServer/README.md b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/k3sServer/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/k3sServer/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/k3sServer/templates/_helpers.tpl b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/k3sServer/templates/_helpers.tpl new file mode 100644 index 0000000000..8e651dccfd --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/k3sServer/templates/_helpers.tpl @@ -0,0 +1,166 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $forceHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- end }} +{{- $metricRelabelings := gt (len (keys $clusterNameRelabel)) 0 | ternary (append ($metricRelabelings) ($clusterNameRelabel)) ($metricRelabelings) }} +{{- $metricRelabelings := gt (len (keys $clusterIdRelabel)) 0 | ternary (append ($metricRelabelings) ($clusterIdRelabel)) ($metricRelabelings) }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $forceHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/k3sServer/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/k3sServer/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/k3sServer/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/k3sServer/templates/pushprox-clients.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/k3sServer/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/k3sServer/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/k3sServer/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/k3sServer/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/k3sServer/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/k3sServer/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/k3sServer/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/k3sServer/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/k3sServer/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/k3sServer/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/k3sServer/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/k3sServer/templates/validate-install-crd.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/k3sServer/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/k3sServer/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/k3sServer/templates/validate-psp-install.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/k3sServer/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/k3sServer/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/k3sServer/values.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/k3sServer/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/k3sServer/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/.helmignore b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/.helmignore new file mode 100644 index 0000000000..f0c1319444 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/Chart.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/Chart.yaml new file mode 100644 index 0000000000..31c5f21199 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/Chart.yaml @@ -0,0 +1,28 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.16.0-0 < 1.27.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-kube-state-metrics +apiVersion: v2 +appVersion: 2.6.0 +description: Install kube-state-metrics to generate and expose cluster-level metrics +home: https://github.com/kubernetes/kube-state-metrics/ +keywords: +- metric +- monitoring +- prometheus +- kubernetes +maintainers: +- email: tariq.ibrahim@mulesoft.com + name: tariq1890 +- email: manuel@rueg.eu + name: mrueg +- email: davidcalvertfr@gmail.com + name: dotdc +name: kube-state-metrics +sources: +- https://github.com/kubernetes/kube-state-metrics/ +type: application +version: 4.18.0 diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/README.md b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/README.md new file mode 100644 index 0000000000..7c2e16918f --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/README.md @@ -0,0 +1,68 @@ +# kube-state-metrics Helm Chart + +Installs the [kube-state-metrics agent](https://github.com/kubernetes/kube-state-metrics). + +## Get Repo Info + +```console +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update +``` + +_See [helm repo](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + +## Install Chart + +```console +helm install [RELEASE_NAME] prometheus-community/kube-state-metrics [flags] +``` + +_See [configuration](#configuration) below._ + +_See [helm install](https://helm.sh/docs/helm/helm_install/) for command documentation._ + +## Uninstall Chart + +```console +helm uninstall [RELEASE_NAME] +``` + +This removes all the Kubernetes components associated with the chart and deletes the release. + +_See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall/) for command documentation._ + +## Upgrading Chart + +```console +helm upgrade [RELEASE_NAME] prometheus-community/kube-state-metrics [flags] +``` + +_See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documentation._ + +### Migrating from stable/kube-state-metrics and kubernetes/kube-state-metrics + +You can upgrade in-place: + +1. [get repo info](#get-repo-info) +1. [upgrade](#upgrading-chart) your existing release name using the new chart repo + + +## Upgrading to v3.0.0 + +v3.0.0 includes kube-state-metrics v2.0, see the [changelog](https://github.com/kubernetes/kube-state-metrics/blob/release-2.0/CHANGELOG.md) for major changes on the application-side. + +The upgraded chart now the following changes: +* Dropped support for helm v2 (helm v3 or later is required) +* collectors key was renamed to resources +* namespace key was renamed to namespaces + + +## Configuration + +See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To see all configurable options with detailed comments: + +```console +helm show values prometheus-community/kube-state-metrics +``` + +You may also run `helm show values` on this chart's [dependencies](#dependencies) for additional options. diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/NOTES.txt b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/NOTES.txt new file mode 100644 index 0000000000..5a646e0cca --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/NOTES.txt @@ -0,0 +1,10 @@ +kube-state-metrics is a simple service that listens to the Kubernetes API server and generates metrics about the state of the objects. +The exposed metrics can be found here: +https://github.com/kubernetes/kube-state-metrics/blob/master/docs/README.md#exposed-metrics + +The metrics are exported on the HTTP endpoint /metrics on the listening port. +In your case, {{ template "kube-state-metrics.fullname" . }}.{{ template "kube-state-metrics.namespace" . }}.svc.cluster.local:{{ .Values.service.port }}/metrics + +They are served either as plaintext or protobuf depending on the Accept header. +They are designed to be consumed either by Prometheus itself or by a scraper that is compatible with scraping a Prometheus client endpoint. + diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/_helpers.tpl b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/_helpers.tpl new file mode 100644 index 0000000000..4de75db2a9 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/_helpers.tpl @@ -0,0 +1,111 @@ +# Rancher +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "kube-state-metrics.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "kube-state-metrics.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "kube-state-metrics.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "kube-state-metrics.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "kube-state-metrics.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "kube-state-metrics.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Generate basic labels +*/}} +{{- define "kube-state-metrics.labels" }} +helm.sh/chart: {{ template "kube-state-metrics.chart" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/component: metrics +app.kubernetes.io/part-of: {{ template "kube-state-metrics.name" . }} +{{- include "kube-state-metrics.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels }} +{{- end }} +{{- if .Values.releaseLabel }} +release: {{ .Release.Name }} +{{- end }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "kube-state-metrics.selectorLabels" }} +app.kubernetes.io/name: {{ include "kube-state-metrics.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/clusterrolebinding.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/clusterrolebinding.yaml new file mode 100644 index 0000000000..cf9f628d04 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/clusterrolebinding.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.rbac.create .Values.rbac.useClusterRole -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + name: {{ template "kube-state-metrics.fullname" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole +{{- if .Values.rbac.useExistingRole }} + name: {{ .Values.rbac.useExistingRole }} +{{- else }} + name: {{ template "kube-state-metrics.fullname" . }} +{{- end }} +subjects: +- kind: ServiceAccount + name: {{ template "kube-state-metrics.serviceAccountName" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} +{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/deployment.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/deployment.yaml new file mode 100644 index 0000000000..a3c8f96b21 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/deployment.yaml @@ -0,0 +1,172 @@ +apiVersion: apps/v1 +{{- if .Values.autosharding.enabled }} +kind: StatefulSet +{{- else }} +kind: Deployment +{{- end }} +metadata: + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + {{- if .Values.annotations }} + annotations: +{{ toYaml .Values.annotations | indent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "kube-state-metrics.selectorLabels" . | indent 6 }} + replicas: {{ .Values.replicas }} + {{- if .Values.autosharding.enabled }} + serviceName: {{ template "kube-state-metrics.fullname" . }} + volumeClaimTemplates: [] + {{- end }} + template: + metadata: + labels: + {{- include "kube-state-metrics.labels" . | indent 8 }} + {{- if .Values.podAnnotations }} + annotations: +{{ toYaml .Values.podAnnotations | indent 8 }} + {{- end }} + spec: + hostNetwork: {{ .Values.hostNetwork }} + serviceAccountName: {{ template "kube-state-metrics.serviceAccountName" . }} + {{- if .Values.securityContext.enabled }} + securityContext: {{- omit .Values.securityContext "enabled" | toYaml | nindent 8 }} + {{- end }} + {{- if .Values.priorityClassName }} + priorityClassName: {{ .Values.priorityClassName }} + {{- end }} + containers: + - name: {{ template "kube-state-metrics.name" . }} + {{- if .Values.autosharding.enabled }} + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- end }} + args: + {{- if .Values.extraArgs }} + {{- .Values.extraArgs | toYaml | nindent 8 }} + {{- end }} + {{- if .Values.service.port }} + - --port={{ .Values.service.port | default 8080}} + {{- end }} + {{- if .Values.collectors }} + - --resources={{ .Values.collectors | join "," }} + {{- end }} + {{- if .Values.metricLabelsAllowlist }} + - --metric-labels-allowlist={{ .Values.metricLabelsAllowlist | join "," }} + {{- end }} + {{- if .Values.metricAnnotationsAllowList }} + - --metric-annotations-allowlist={{ .Values.metricAnnotationsAllowList | join "," }} + {{- end }} + {{- if .Values.metricAllowlist }} + - --metric-allowlist={{ .Values.metricAllowlist | join "," }} + {{- end }} + {{- if .Values.metricDenylist }} + - --metric-denylist={{ .Values.metricDenylist | join "," }} + {{- end }} + {{- if .Values.releaseNamespace }} + - --namespaces={{ template "kube-state-metrics.namespace" . }} + {{- else if .Values.namespaces }} + - --namespaces={{ tpl (.Values.namespaces | join ",") $ }} + {{- end }} + {{- if .Values.namespacesDenylist }} + - --namespaces-denylist={{ tpl (.Values.namespacesDenylist | join ",") $ }} + {{- end }} + {{- if .Values.autosharding.enabled }} + - --pod=$(POD_NAME) + - --pod-namespace=$(POD_NAMESPACE) + {{- end }} + {{- if .Values.kubeconfig.enabled }} + - --kubeconfig=/opt/k8s/.kube/config + {{- end }} + {{- if .Values.selfMonitor.telemetryHost }} + - --telemetry-host={{ .Values.selfMonitor.telemetryHost }} + {{- end }} + {{- if .Values.selfMonitor.telemetryPort }} + - --telemetry-port={{ .Values.selfMonitor.telemetryPort | default 8081 }} + {{- end }} + {{- if or (.Values.kubeconfig.enabled) (.Values.volumeMounts) }} + volumeMounts: + {{- if .Values.kubeconfig.enabled }} + - name: kubeconfig + mountPath: /opt/k8s/.kube/ + readOnly: true + {{- end }} + {{- if .Values.volumeMounts }} +{{ toYaml .Values.volumeMounts | indent 8 }} + {{- end }} + {{- end }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- if .Values.image.sha }} + image: "{{ template "system_default_registry" . }}{{ .Values.image.repository }}:{{ .Values.image.tag }}@sha256:{{ .Values.image.sha }}" + {{- else }} + image: "{{ template "system_default_registry" . }}{{ .Values.image.repository }}:{{ .Values.image.tag }}" + {{- end }} + ports: + - containerPort: {{ .Values.service.port | default 8080}} + name: "http" + {{- if .Values.selfMonitor.enabled }} + - containerPort: {{ .Values.selfMonitor.telemetryPort | default 8081 }} + name: "metrics" + {{- end }} + livenessProbe: + httpGet: + path: /healthz + port: {{ .Values.service.port | default 8080}} + initialDelaySeconds: 5 + timeoutSeconds: 5 + readinessProbe: + httpGet: + path: / + port: {{ .Values.service.port | default 8080}} + initialDelaySeconds: 5 + timeoutSeconds: 5 + {{- if .Values.resources }} + resources: +{{ toYaml .Values.resources | indent 10 }} +{{- end }} +{{- if .Values.containerSecurityContext }} + securityContext: +{{ toYaml .Values.containerSecurityContext | indent 10 }} +{{- end }} +{{- if .Values.imagePullSecrets }} + imagePullSecrets: +{{ toYaml .Values.imagePullSecrets | indent 8 }} + {{- end }} + {{- if .Values.affinity }} + affinity: +{{ toYaml .Values.affinity | indent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} + {{- if .Values.nodeSelector }} +{{ toYaml .Values.nodeSelector | indent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} + {{- if .Values.tolerations }} +{{ toYaml .Values.tolerations | indent 8 }} + {{- end }} + {{- if .Values.topologySpreadConstraints }} + topologySpreadConstraints: +{{ toYaml .Values.topologySpreadConstraints | indent 8 }} + {{- end }} + {{- if or (.Values.kubeconfig.enabled) (.Values.volumes) }} + volumes: + {{- if .Values.kubeconfig.enabled}} + - name: kubeconfig + secret: + secretName: {{ template "kube-state-metrics.fullname" . }}-kubeconfig + {{- end }} + {{- if .Values.volumes }} +{{ toYaml .Values.volumes | indent 8 }} + {{- end }} + {{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/kubeconfig-secret.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/kubeconfig-secret.yaml new file mode 100644 index 0000000000..6af0084502 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/kubeconfig-secret.yaml @@ -0,0 +1,12 @@ +{{- if .Values.kubeconfig.enabled -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "kube-state-metrics.fullname" . }}-kubeconfig + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} +type: Opaque +data: + config: '{{ .Values.kubeconfig.secret }}' +{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/pdb.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/pdb.yaml new file mode 100644 index 0000000000..3771b511de --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/pdb.yaml @@ -0,0 +1,18 @@ +{{- if .Values.podDisruptionBudget -}} +{{ if $.Capabilities.APIVersions.Has "policy/v1/PodDisruptionBudget" -}} +apiVersion: policy/v1 +{{- else -}} +apiVersion: policy/v1beta1 +{{- end }} +kind: PodDisruptionBudget +metadata: + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} +spec: + selector: + matchLabels: + app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }} +{{ toYaml .Values.podDisruptionBudget | indent 2 }} +{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/podsecuritypolicy.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/podsecuritypolicy.yaml new file mode 100644 index 0000000000..72872cf893 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/podsecuritypolicy.yaml @@ -0,0 +1,39 @@ +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "kube-state-metrics.fullname" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} +{{- if .Values.podSecurityPolicy.annotations }} + annotations: +{{ toYaml .Values.podSecurityPolicy.annotations | indent 4 }} +{{- end }} +spec: + privileged: false + volumes: + - 'secret' +{{- if .Values.podSecurityPolicy.additionalVolumes }} +{{ toYaml .Values.podSecurityPolicy.additionalVolumes | indent 4 }} +{{- end }} + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/psp-clusterrole.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/psp-clusterrole.yaml new file mode 100644 index 0000000000..9814623c55 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/psp-clusterrole.yaml @@ -0,0 +1,19 @@ +{{- if and .Values.global.cattle.psp.enabled .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + name: psp-{{ template "kube-state-metrics.fullname" . }} +rules: +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if semverCompare "> 1.15.0-0" $kubeTargetVersion }} +- apiGroups: ['policy'] +{{- else }} +- apiGroups: ['extensions'] +{{- end }} + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "kube-state-metrics.fullname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/psp-clusterrolebinding.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/psp-clusterrolebinding.yaml new file mode 100644 index 0000000000..60f8a72d95 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/psp-clusterrolebinding.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.global.cattle.psp.enabled .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + name: psp-{{ template "kube-state-metrics.fullname" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp-{{ template "kube-state-metrics.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ template "kube-state-metrics.serviceAccountName" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/role.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/role.yaml new file mode 100644 index 0000000000..5fbd43c6f9 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/role.yaml @@ -0,0 +1,193 @@ +{{- if not (kindIs "slice" .Values.collectors) }} +{{- fail "Collectors need to be a List since kube-state-metrics chart 3.2.2. Please check README for more information."}} +{{- end }} +{{- if and (eq .Values.rbac.create true) (not .Values.rbac.useExistingRole) -}} +{{- range (ternary (split "," .Values.namespaces) (list "") (eq $.Values.rbac.useClusterRole false)) }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +{{- if eq $.Values.rbac.useClusterRole false }} +kind: Role +{{- else }} +kind: ClusterRole +{{- end }} +metadata: + labels: + {{- include "kube-state-metrics.labels" $ | indent 4 }} + name: {{ template "kube-state-metrics.fullname" $ }} +{{- if eq $.Values.rbac.useClusterRole false }} + namespace: {{ . }} +{{- end }} +rules: +{{ if has "certificatesigningrequests" $.Values.collectors }} +- apiGroups: ["certificates.k8s.io"] + resources: + - certificatesigningrequests + verbs: ["list", "watch"] +{{ end -}} +{{ if has "configmaps" $.Values.collectors }} +- apiGroups: [""] + resources: + - configmaps + verbs: ["list", "watch"] +{{ end -}} +{{ if has "cronjobs" $.Values.collectors }} +- apiGroups: ["batch"] + resources: + - cronjobs + verbs: ["list", "watch"] +{{ end -}} +{{ if has "daemonsets" $.Values.collectors }} +- apiGroups: ["extensions", "apps"] + resources: + - daemonsets + verbs: ["list", "watch"] +{{ end -}} +{{ if has "deployments" $.Values.collectors }} +- apiGroups: ["extensions", "apps"] + resources: + - deployments + verbs: ["list", "watch"] +{{ end -}} +{{ if has "endpoints" $.Values.collectors }} +- apiGroups: [""] + resources: + - endpoints + verbs: ["list", "watch"] +{{ end -}} +{{ if has "horizontalpodautoscalers" $.Values.collectors }} +- apiGroups: ["autoscaling"] + resources: + - horizontalpodautoscalers + verbs: ["list", "watch"] +{{ end -}} +{{ if has "ingresses" $.Values.collectors }} +- apiGroups: ["extensions", "networking.k8s.io"] + resources: + - ingresses + verbs: ["list", "watch"] +{{ end -}} +{{ if has "jobs" $.Values.collectors }} +- apiGroups: ["batch"] + resources: + - jobs + verbs: ["list", "watch"] +{{ end -}} +{{ if has "limitranges" $.Values.collectors }} +- apiGroups: [""] + resources: + - limitranges + verbs: ["list", "watch"] +{{ end -}} +{{ if has "mutatingwebhookconfigurations" $.Values.collectors }} +- apiGroups: ["admissionregistration.k8s.io"] + resources: + - mutatingwebhookconfigurations + verbs: ["list", "watch"] +{{ end -}} +{{ if has "namespaces" $.Values.collectors }} +- apiGroups: [""] + resources: + - namespaces + verbs: ["list", "watch"] +{{ end -}} +{{ if has "networkpolicies" $.Values.collectors }} +- apiGroups: ["networking.k8s.io"] + resources: + - networkpolicies + verbs: ["list", "watch"] +{{ end -}} +{{ if has "nodes" $.Values.collectors }} +- apiGroups: [""] + resources: + - nodes + verbs: ["list", "watch"] +{{ end -}} +{{ if has "persistentvolumeclaims" $.Values.collectors }} +- apiGroups: [""] + resources: + - persistentvolumeclaims + verbs: ["list", "watch"] +{{ end -}} +{{ if has "persistentvolumes" $.Values.collectors }} +- apiGroups: [""] + resources: + - persistentvolumes + verbs: ["list", "watch"] +{{ end -}} +{{ if has "poddisruptionbudgets" $.Values.collectors }} +- apiGroups: ["policy"] + resources: + - poddisruptionbudgets + verbs: ["list", "watch"] +{{ end -}} +{{ if has "pods" $.Values.collectors }} +- apiGroups: [""] + resources: + - pods + verbs: ["list", "watch"] +{{ end -}} +{{ if has "replicasets" $.Values.collectors }} +- apiGroups: ["extensions", "apps"] + resources: + - replicasets + verbs: ["list", "watch"] +{{ end -}} +{{ if has "replicationcontrollers" $.Values.collectors }} +- apiGroups: [""] + resources: + - replicationcontrollers + verbs: ["list", "watch"] +{{ end -}} +{{ if has "resourcequotas" $.Values.collectors }} +- apiGroups: [""] + resources: + - resourcequotas + verbs: ["list", "watch"] +{{ end -}} +{{ if has "secrets" $.Values.collectors }} +- apiGroups: [""] + resources: + - secrets + verbs: ["list", "watch"] +{{ end -}} +{{ if has "services" $.Values.collectors }} +- apiGroups: [""] + resources: + - services + verbs: ["list", "watch"] +{{ end -}} +{{ if has "statefulsets" $.Values.collectors }} +- apiGroups: ["apps"] + resources: + - statefulsets + verbs: ["list", "watch"] +{{ end -}} +{{ if has "storageclasses" $.Values.collectors }} +- apiGroups: ["storage.k8s.io"] + resources: + - storageclasses + verbs: ["list", "watch"] +{{ end -}} +{{ if has "validatingwebhookconfigurations" $.Values.collectors }} +- apiGroups: ["admissionregistration.k8s.io"] + resources: + - validatingwebhookconfigurations + verbs: ["list", "watch"] +{{ end -}} +{{ if has "volumeattachments" $.Values.collectors }} +- apiGroups: ["storage.k8s.io"] + resources: + - volumeattachments + verbs: ["list", "watch"] +{{ end -}} +{{ if has "verticalpodautoscalers" $.Values.collectors }} +- apiGroups: ["autoscaling.k8s.io"] + resources: + - verticalpodautoscalers + verbs: ["list", "watch"] +{{ end -}} +{{ if $.Values.rbac.extraRules }} +{{ toYaml $.Values.rbac.extraRules }} +{{ end }} +{{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/rolebinding.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/rolebinding.yaml new file mode 100644 index 0000000000..135094f7bc --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/rolebinding.yaml @@ -0,0 +1,24 @@ +{{- if and (eq .Values.rbac.create true) (eq .Values.rbac.useClusterRole false) -}} +{{- range (split "," $.Values.namespaces) }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + {{- include "kube-state-metrics.labels" $ | indent 4 }} + name: {{ template "kube-state-metrics.fullname" $ }} + namespace: {{ . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role +{{- if (not $.Values.rbac.useExistingRole) }} + name: {{ template "kube-state-metrics.fullname" $ }} +{{- else }} + name: {{ $.Values.rbac.useExistingRole }} +{{- end }} +subjects: +- kind: ServiceAccount + name: {{ template "kube-state-metrics.serviceAccountName" $ }} + namespace: {{ template "kube-state-metrics.namespace" $ }} +{{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/service.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/service.yaml new file mode 100644 index 0000000000..92c6d4fcad --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/service.yaml @@ -0,0 +1,41 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + annotations: + {{- if .Values.prometheusScrape }} + prometheus.io/scrape: '{{ .Values.prometheusScrape }}' + {{- end }} + {{- if .Values.service.annotations }} + {{- toYaml .Values.service.annotations | nindent 4 }} + {{- end }} +spec: + type: "{{ .Values.service.type }}" + ports: + - name: "http" + protocol: TCP + port: {{ .Values.service.port | default 8080}} + {{- if .Values.service.nodePort }} + nodePort: {{ .Values.service.nodePort }} + {{- end }} + targetPort: {{ .Values.service.port | default 8080}} + {{ if .Values.selfMonitor.enabled }} + - name: "metrics" + protocol: TCP + port: {{ .Values.selfMonitor.telemetryPort | default 8081 }} + targetPort: {{ .Values.selfMonitor.telemetryPort | default 8081 }} + {{- if .Values.selfMonitor.telemetryNodePort }} + nodePort: {{ .Values.selfMonitor.telemetryNodePort }} + {{- end }} + {{ end }} +{{- if .Values.service.loadBalancerIP }} + loadBalancerIP: "{{ .Values.service.loadBalancerIP }}" +{{- end }} +{{- if .Values.service.clusterIP }} + clusterIP: "{{ .Values.service.clusterIP }}" +{{- end }} + selector: + {{- include "kube-state-metrics.selectorLabels" . | indent 4 }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/serviceaccount.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/serviceaccount.yaml new file mode 100644 index 0000000000..e1229eb95e --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/serviceaccount.yaml @@ -0,0 +1,15 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + name: {{ template "kube-state-metrics.serviceAccountName" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} +{{- if .Values.serviceAccount.annotations }} + annotations: +{{ toYaml .Values.serviceAccount.annotations | indent 4 }} +{{- end }} +imagePullSecrets: +{{ toYaml .Values.serviceAccount.imagePullSecrets | indent 2 }} +{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/servicemonitor.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/servicemonitor.yaml new file mode 100644 index 0000000000..41da4142be --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/servicemonitor.yaml @@ -0,0 +1,86 @@ +{{- if .Values.prometheus.monitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + {{- with .Values.prometheus.monitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + jobLabel: {{ default "app.kubernetes.io/name" .Values.prometheus.monitor.jobLabel }} + selector: + matchLabels: + {{- if .Values.prometheus.monitor.selectorOverride -}} + {{ toYaml .Values.prometheus.monitor.selectorOverride | nindent 6 }} + {{ else }} + {{- include "kube-state-metrics.selectorLabels" . | indent 6 }} + {{- end }} + endpoints: + - port: http + {{- if .Values.prometheus.monitor.interval }} + interval: {{ .Values.prometheus.monitor.interval }} + {{- end }} + {{- if .Values.prometheus.monitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.prometheus.monitor.scrapeTimeout }} + {{- end }} + {{- if .Values.prometheus.monitor.proxyUrl }} + proxyUrl: {{ .Values.prometheus.monitor.proxyUrl}} + {{- end }} + {{- if .Values.prometheus.monitor.honorLabels }} + honorLabels: true + {{- end }} + metricRelabelings: + {{- if .Values.prometheus.monitor.metricRelabelings }} + {{- toYaml .Values.prometheus.monitor.metricRelabelings | nindent 8 }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName }} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + {{- if .Values.prometheus.monitor.relabelings }} + relabelings: + {{- toYaml .Values.prometheus.monitor.relabelings | nindent 8 }} + {{- end }} + {{- if .Values.prometheus.monitor.scheme }} + scheme: {{ .Values.prometheus.monitor.scheme }} + {{- end }} + {{- if .Values.prometheus.monitor.tlsConfig }} + tlsConfig: + {{- toYaml .Values.prometheus.monitor.tlsConfig | nindent 8 }} + {{- end }} + {{- if .Values.selfMonitor.enabled }} + - port: metrics + {{- if .Values.prometheus.monitor.interval }} + interval: {{ .Values.prometheus.monitor.interval }} + {{- end }} + {{- if .Values.prometheus.monitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.prometheus.monitor.scrapeTimeout }} + {{- end }} + {{- if .Values.prometheus.monitor.proxyUrl }} + proxyUrl: {{ .Values.prometheus.monitor.proxyUrl}} + {{- end }} + {{- if .Values.prometheus.monitor.honorLabels }} + honorLabels: true + {{- end }} + {{- if .Values.prometheus.monitor.relabelings }} + relabelings: + {{- toYaml .Values.prometheus.monitor.relabelings | nindent 8 }} + {{- end }} + {{- if .Values.prometheus.monitor.scheme }} + scheme: {{ .Values.prometheus.monitor.scheme }} + {{- end }} + {{- if .Values.prometheus.monitor.tlsConfig }} + tlsConfig: + {{- toYaml .Values.prometheus.monitor.tlsConfig | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/stsdiscovery-role.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/stsdiscovery-role.yaml new file mode 100644 index 0000000000..489de147c1 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/stsdiscovery-role.yaml @@ -0,0 +1,26 @@ +{{- if and .Values.autosharding.enabled .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: stsdiscovery-{{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get +- apiGroups: + - apps + resourceNames: + - {{ template "kube-state-metrics.fullname" . }} + resources: + - statefulsets + verbs: + - get + - list + - watch +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/stsdiscovery-rolebinding.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/stsdiscovery-rolebinding.yaml new file mode 100644 index 0000000000..73b37a4f64 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/templates/stsdiscovery-rolebinding.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.autosharding.enabled .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: stsdiscovery-{{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: stsdiscovery-{{ template "kube-state-metrics.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ template "kube-state-metrics.serviceAccountName" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/values.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/values.yaml new file mode 100644 index 0000000000..d96131a035 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kube-state-metrics/values.yaml @@ -0,0 +1,271 @@ +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + +# Default values for kube-state-metrics. +prometheusScrape: true +image: + repository: rancher/mirrored-kube-state-metrics-kube-state-metrics + tag: v2.6.0 + sha: "" + pullPolicy: IfNotPresent + +imagePullSecrets: [] +# - name: "image-pull-secret" + +# If set to true, this will deploy kube-state-metrics as a StatefulSet and the data +# will be automatically sharded across <.Values.replicas> pods using the built-in +# autodiscovery feature: https://github.com/kubernetes/kube-state-metrics#automated-sharding +# This is an experimental feature and there are no stability guarantees. +autosharding: + enabled: false + +replicas: 1 + +# List of additional cli arguments to configure kube-state-metrics +# for example: --enable-gzip-encoding, --log-file, etc. +# all the possible args can be found here: https://github.com/kubernetes/kube-state-metrics/blob/master/docs/cli-arguments.md +extraArgs: [] + +service: + port: 8080 + # Default to clusterIP for backward compatibility + type: ClusterIP + nodePort: 0 + loadBalancerIP: "" + clusterIP: "" + annotations: {} + +## Additional labels to add to all resources +customLabels: {} + # app: kube-state-metrics + +## set to true to add the release label so scraping of the servicemonitor with kube-prometheus-stack works out of the box +releaseLabel: false + +hostNetwork: false + +rbac: + # If true, create & use RBAC resources + create: true + + # Set to a rolename to use existing role - skipping role creating - but still doing serviceaccount and rolebinding to it, rolename set here. + # useExistingRole: your-existing-role + + # If set to false - Run without Cluteradmin privs needed - ONLY works if namespace is also set (if useExistingRole is set this name is used as ClusterRole or Role to bind to) + useClusterRole: true + + # Add permissions for CustomResources' apiGroups in Role/ClusterRole. Should be used in conjunction with Custom Resource State Metrics configuration + # Example: + # - apiGroups: ["monitoring.coreos.com"] + # resources: ["prometheuses"] + # verbs: ["list", "watch"] + extraRules: [] + +serviceAccount: + # Specifies whether a ServiceAccount should be created, require rbac true + create: true + # The name of the ServiceAccount to use. + # If not set and create is true, a name is generated using the fullname template + name: + # Reference to one or more secrets to be used when pulling images + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + imagePullSecrets: [] + # ServiceAccount annotations. + # Use case: AWS EKS IAM roles for service accounts + # ref: https://docs.aws.amazon.com/eks/latest/userguide/specify-service-account-role.html + annotations: {} + +prometheus: + monitor: + enabled: false + additionalLabels: {} + namespace: "" + jobLabel: "" + interval: "" + scrapeTimeout: "" + proxyUrl: "" + selectorOverride: {} + honorLabels: false + metricRelabelings: [] + relabelings: [] + scheme: "" + tlsConfig: {} + +## Specify if a Pod Security Policy for kube-state-metrics must be created +## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/ +## +podSecurityPolicy: + annotations: {} + ## Specify pod annotations + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#apparmor + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#seccomp + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#sysctl + ## + # seccomp.security.alpha.kubernetes.io/allowedProfileNames: '*' + # seccomp.security.alpha.kubernetes.io/defaultProfileName: 'docker/default' + # apparmor.security.beta.kubernetes.io/defaultProfileName: 'runtime/default' + + additionalVolumes: [] + +securityContext: + enabled: true + runAsNonRoot: true + runAsGroup: 65534 + runAsUser: 65534 + fsGroup: 65534 + +## Specify security settings for a Container +## Allows overrides and additional options compared to (Pod) securityContext +## Ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container +containerSecurityContext: {} + +## Node labels for pod assignment +## Ref: https://kubernetes.io/docs/user-guide/node-selection/ +nodeSelector: {} + +## Affinity settings for pod assignment +## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ +affinity: {} + +## Tolerations for pod assignment +## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +tolerations: [] + +## Topology spread constraints for pod assignment +## Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ +topologySpreadConstraints: [] + +# Annotations to be added to the deployment/statefulset +annotations: {} + +# Annotations to be added to the pod +podAnnotations: {} + +## Assign a PriorityClassName to pods if set +# priorityClassName: "" + +# Ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/ +podDisruptionBudget: {} + +# Comma-separated list of metrics to be exposed. +# This list comprises of exact metric names and/or regex patterns. +# The allowlist and denylist are mutually exclusive. +metricAllowlist: [] + +# Comma-separated list of metrics not to be enabled. +# This list comprises of exact metric names and/or regex patterns. +# The allowlist and denylist are mutually exclusive. +metricDenylist: [] + +# Comma-separated list of additional Kubernetes label keys that will be used in the resource's +# labels metric. By default the metric contains only name and namespace labels. +# To include additional labels, provide a list of resource names in their plural form and Kubernetes +# label keys you would like to allow for them (Example: '=namespaces=[k8s-label-1,k8s-label-n,...],pods=[app],...)'. +# A single '*' can be provided per resource instead to allow any labels, but that has +# severe performance implications (Example: '=pods=[*]'). +metricLabelsAllowlist: [] + # - namespaces=[k8s-label-1,k8s-label-n] + +# Comma-separated list of Kubernetes annotations keys that will be used in the resource' +# labels metric. By default the metric contains only name and namespace labels. +# To include additional annotations provide a list of resource names in their plural form and Kubernetes +# annotation keys you would like to allow for them (Example: '=namespaces=[kubernetes.io/team,...],pods=[kubernetes.io/team],...)'. +# A single '*' can be provided per resource instead to allow any annotations, but that has +# severe performance implications (Example: '=pods=[*]'). +metricAnnotationsAllowList: [] + # - pods=[k8s-annotation-1,k8s-annotation-n] + +# Available collectors for kube-state-metrics. +# By default, all available resources are enabled, comment out to disable. +collectors: + - certificatesigningrequests + - configmaps + - cronjobs + - daemonsets + - deployments + - endpoints + - horizontalpodautoscalers + - ingresses + - jobs + - limitranges + - mutatingwebhookconfigurations + - namespaces + - networkpolicies + - nodes + - persistentvolumeclaims + - persistentvolumes + - poddisruptionbudgets + - pods + - replicasets + - replicationcontrollers + - resourcequotas + - secrets + - services + - statefulsets + - storageclasses + - validatingwebhookconfigurations + - volumeattachments + # - verticalpodautoscalers # not a default resource, see also: https://github.com/kubernetes/kube-state-metrics#enabling-verticalpodautoscalers + +# Enabling kubeconfig will pass the --kubeconfig argument to the container +kubeconfig: + enabled: false + # base64 encoded kube-config file + secret: + +# Enable only the release namespace for collecting resources. By default all namespaces are collected. +# If releaseNamespace and namespaces are both set only releaseNamespace will be used. +releaseNamespace: false + +# Comma-separated list of namespaces to be enabled for collecting resources. By default all namespaces are collected. +namespaces: "" + +# Comma-separated list of namespaces not to be enabled. If namespaces and namespaces-denylist are both set, +# only namespaces that are excluded in namespaces-denylist will be used. +namespacesDenylist: "" + +## Override the deployment namespace +## +namespaceOverride: "" + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 64Mi + # requests: + # cpu: 10m + # memory: 32Mi + +## Provide a k8s version to define apiGroups for podSecurityPolicy Cluster Role. +## For example: kubeTargetVersionOverride: 1.14.9 +## +kubeTargetVersionOverride: "" + +# Enable self metrics configuration for service and Service Monitor +# Default values for telemetry configuration can be overridden +# If you set telemetryNodePort, you must also set service.type to NodePort +selfMonitor: + enabled: false + # telemetryHost: 0.0.0.0 + # telemetryPort: 8081 + # telemetryNodePort: 0 + +# volumeMounts are used to add custom volume mounts to deployment. +# See example below +volumeMounts: [] +# - mountPath: /etc/config +# name: config-volume + +# volumes are used to add custom volumes to deployment +# See example below +volumes: [] +# - configMap: +# name: cm-for-volume +# name: config-volume diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmControllerManager/.helmignore b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmControllerManager/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmControllerManager/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmControllerManager/Chart.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmControllerManager/Chart.yaml new file mode 100644 index 0000000000..93062b7425 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmControllerManager/Chart.yaml @@ -0,0 +1,14 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.16.0-0 < 1.28.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +name: kubeAdmControllerManager +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmControllerManager/README.md b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmControllerManager/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmControllerManager/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmControllerManager/templates/_helpers.tpl b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmControllerManager/templates/_helpers.tpl new file mode 100644 index 0000000000..8e651dccfd --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmControllerManager/templates/_helpers.tpl @@ -0,0 +1,166 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $forceHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- end }} +{{- $metricRelabelings := gt (len (keys $clusterNameRelabel)) 0 | ternary (append ($metricRelabelings) ($clusterNameRelabel)) ($metricRelabelings) }} +{{- $metricRelabelings := gt (len (keys $clusterIdRelabel)) 0 | ternary (append ($metricRelabelings) ($clusterIdRelabel)) ($metricRelabelings) }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $forceHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmControllerManager/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmControllerManager/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmControllerManager/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmControllerManager/templates/pushprox-clients.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmControllerManager/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmControllerManager/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmControllerManager/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmControllerManager/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmControllerManager/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmControllerManager/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmControllerManager/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmControllerManager/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmControllerManager/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmControllerManager/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmControllerManager/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmControllerManager/templates/validate-install-crd.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmControllerManager/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmControllerManager/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmControllerManager/templates/validate-psp-install.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmControllerManager/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmControllerManager/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmControllerManager/values.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmControllerManager/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmControllerManager/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmEtcd/.helmignore b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmEtcd/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmEtcd/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmEtcd/Chart.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmEtcd/Chart.yaml new file mode 100644 index 0000000000..95459be0d3 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmEtcd/Chart.yaml @@ -0,0 +1,14 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.16.0-0 < 1.28.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +name: kubeAdmEtcd +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmEtcd/README.md b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmEtcd/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmEtcd/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmEtcd/templates/_helpers.tpl b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmEtcd/templates/_helpers.tpl new file mode 100644 index 0000000000..8e651dccfd --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmEtcd/templates/_helpers.tpl @@ -0,0 +1,166 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $forceHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- end }} +{{- $metricRelabelings := gt (len (keys $clusterNameRelabel)) 0 | ternary (append ($metricRelabelings) ($clusterNameRelabel)) ($metricRelabelings) }} +{{- $metricRelabelings := gt (len (keys $clusterIdRelabel)) 0 | ternary (append ($metricRelabelings) ($clusterIdRelabel)) ($metricRelabelings) }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $forceHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmEtcd/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmEtcd/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmEtcd/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmEtcd/templates/pushprox-clients.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmEtcd/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmEtcd/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmEtcd/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmEtcd/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmEtcd/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmEtcd/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmEtcd/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmEtcd/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmEtcd/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmEtcd/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmEtcd/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmEtcd/templates/validate-install-crd.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmEtcd/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmEtcd/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmEtcd/templates/validate-psp-install.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmEtcd/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmEtcd/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmEtcd/values.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmEtcd/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmEtcd/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmProxy/.helmignore b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmProxy/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmProxy/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmProxy/Chart.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmProxy/Chart.yaml new file mode 100644 index 0000000000..86570faeee --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmProxy/Chart.yaml @@ -0,0 +1,14 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.16.0-0 < 1.28.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +name: kubeAdmProxy +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmProxy/README.md b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmProxy/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmProxy/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmProxy/templates/_helpers.tpl b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmProxy/templates/_helpers.tpl new file mode 100644 index 0000000000..8e651dccfd --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmProxy/templates/_helpers.tpl @@ -0,0 +1,166 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $forceHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- end }} +{{- $metricRelabelings := gt (len (keys $clusterNameRelabel)) 0 | ternary (append ($metricRelabelings) ($clusterNameRelabel)) ($metricRelabelings) }} +{{- $metricRelabelings := gt (len (keys $clusterIdRelabel)) 0 | ternary (append ($metricRelabelings) ($clusterIdRelabel)) ($metricRelabelings) }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $forceHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmProxy/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmProxy/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmProxy/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmProxy/templates/pushprox-clients.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmProxy/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmProxy/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmProxy/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmProxy/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmProxy/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmProxy/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmProxy/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmProxy/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmProxy/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmProxy/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmProxy/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmProxy/templates/validate-install-crd.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmProxy/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmProxy/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmProxy/templates/validate-psp-install.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmProxy/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmProxy/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmProxy/values.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmProxy/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmProxy/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmScheduler/.helmignore b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmScheduler/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmScheduler/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmScheduler/Chart.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmScheduler/Chart.yaml new file mode 100644 index 0000000000..6ca5852598 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmScheduler/Chart.yaml @@ -0,0 +1,14 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.16.0-0 < 1.28.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +name: kubeAdmScheduler +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmScheduler/README.md b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmScheduler/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmScheduler/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmScheduler/templates/_helpers.tpl b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmScheduler/templates/_helpers.tpl new file mode 100644 index 0000000000..8e651dccfd --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmScheduler/templates/_helpers.tpl @@ -0,0 +1,166 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $forceHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- end }} +{{- $metricRelabelings := gt (len (keys $clusterNameRelabel)) 0 | ternary (append ($metricRelabelings) ($clusterNameRelabel)) ($metricRelabelings) }} +{{- $metricRelabelings := gt (len (keys $clusterIdRelabel)) 0 | ternary (append ($metricRelabelings) ($clusterIdRelabel)) ($metricRelabelings) }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $forceHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmScheduler/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmScheduler/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmScheduler/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmScheduler/templates/pushprox-clients.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmScheduler/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmScheduler/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmScheduler/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmScheduler/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmScheduler/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmScheduler/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmScheduler/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmScheduler/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmScheduler/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmScheduler/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmScheduler/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmScheduler/templates/validate-install-crd.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmScheduler/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmScheduler/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmScheduler/templates/validate-psp-install.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmScheduler/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmScheduler/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmScheduler/values.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmScheduler/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/kubeAdmScheduler/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/.helmignore b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/.helmignore new file mode 100644 index 0000000000..f0c1319444 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/Chart.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/Chart.yaml new file mode 100644 index 0000000000..49e3979445 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/Chart.yaml @@ -0,0 +1,27 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.16.0-0 < 1.27.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-prometheus-adapter +apiVersion: v1 +appVersion: v0.10.0 +description: A Helm chart for k8s prometheus adapter +home: https://github.com/kubernetes-sigs/prometheus-adapter +keywords: +- hpa +- metrics +- prometheus +- adapter +maintainers: +- email: mattias.gees@jetstack.io + name: mattiasgees +- name: steven-sheehy +- email: hfernandez@mesosphere.com + name: hectorj2f +name: prometheus-adapter +sources: +- https://github.com/kubernetes/charts +- https://github.com/kubernetes-sigs/prometheus-adapter +version: 3.4.0 diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/README.md b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/README.md new file mode 100644 index 0000000000..b834160682 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/README.md @@ -0,0 +1,138 @@ +# Prometheus Adapter + +Installs the [Prometheus Adapter](https://github.com/kubernetes-sigs/prometheus-adapter) for the Custom Metrics API. Custom metrics are used in Kubernetes by [Horizontal Pod Autoscalers](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/) to scale workloads based upon your own metric pulled from an external metrics provider like Prometheus. This chart complements the [metrics-server](https://github.com/helm/charts/tree/master/stable/metrics-server) chart that provides resource only metrics. + +## Prerequisites + +Kubernetes 1.14+ + +## Get Helm Repositories Info + +```console +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update +``` + +_See [`helm repo`](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + +## Install Helm Chart + +```console +helm install [RELEASE_NAME] prometheus-community/prometheus-adapter +``` + +_See [configuration](#configuration) below._ + +_See [helm install](https://helm.sh/docs/helm/helm_install/) for command documentation._ + +## Uninstall Helm Chart + +```console +helm uninstall [RELEASE_NAME] +``` + +This removes all the Kubernetes components associated with the chart and deletes the release. + +_See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall/) for command documentation._ + +## Upgrading Helm Chart + +```console +helm upgrade [RELEASE_NAME] [CHART] --install +``` + +_See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documentation._ + +### To 3.0.0 + +Due to a change in deployment labels, the upgrade requires `helm upgrade --force` in order to re-create the deployment. + +## Configuration + +See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To see all configurable options with detailed comments, visit the chart's [values.yaml](./values.yaml), or run these configuration commands: + +```console +helm show values prometheus-community/prometheus-adapter +``` + +### Prometheus Service Endpoint + +To use the chart, ensure the `prometheus.url` and `prometheus.port` are configured with the correct Prometheus service endpoint. If Prometheus is exposed under HTTPS the host's CA Bundle must be exposed to the container using `extraVolumes` and `extraVolumeMounts`. + +### Adapter Rules + +Additionally, the chart comes with a set of default rules out of the box but they may pull in too many metrics or not map them correctly for your needs. Therefore, it is recommended to populate `rules.custom` with a list of rules (see the [config document](https://github.com/kubernetes-sigs/prometheus-adapter/blob/master/docs/config.md) for the proper format). + +### Horizontal Pod Autoscaler Metrics + +Finally, to configure your Horizontal Pod Autoscaler to use the custom metric, see the custom metrics section of the [HPA walkthrough](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-multiple-metrics-and-custom-metrics). + +The Prometheus Adapter can serve three different [metrics APIs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-metrics-apis): + +### Custom Metrics + +Enabling this option will cause custom metrics to be served at `/apis/custom.metrics.k8s.io/v1beta1`. Enabled by default when `rules.default` is true, but can be customized by populating `rules.custom`: + +```yaml +rules: + custom: + - seriesQuery: '{__name__=~"^some_metric_count$"}' + resources: + template: <<.Resource>> + name: + matches: "" + as: "my_custom_metric" + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) +``` + +### External Metrics + +Enabling this option will cause external metrics to be served at `/apis/external.metrics.k8s.io/v1beta1`. Can be enabled by populating `rules.external`: + +```yaml +rules: + external: + - seriesQuery: '{__name__=~"^some_metric_count$"}' + resources: + template: <<.Resource>> + name: + matches: "" + as: "my_external_metric" + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) +``` + +### Resource Metrics + +Enabling this option will cause resource metrics to be served at `/apis/metrics.k8s.io/v1beta1`. Resource metrics will allow pod CPU and Memory metrics to be used in [Horizontal Pod Autoscalers](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/) as well as the `kubectl top` command. Can be enabled by populating `rules.resource`: + +```yaml +rules: + resource: + cpu: + containerQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>, container!=""}[3m])) by (<<.GroupBy>>) + nodeQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>, id='/'}[3m])) by (<<.GroupBy>>) + resources: + overrides: + node: + resource: node + namespace: + resource: namespace + pod: + resource: pod + containerLabel: container + memory: + containerQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>, container!=""}) by (<<.GroupBy>>) + nodeQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>,id='/'}) by (<<.GroupBy>>) + resources: + overrides: + node: + resource: node + namespace: + resource: namespace + pod: + resource: pod + containerLabel: container + window: 3m +``` + +**NOTE:** Setting a value for `rules.resource` will also deploy the resource metrics API service, providing the same functionality as [metrics-server](https://github.com/helm/charts/tree/master/stable/metrics-server). As such it is not possible to deploy them both in the same cluster. diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/NOTES.txt b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/NOTES.txt new file mode 100644 index 0000000000..b7b9b99322 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/NOTES.txt @@ -0,0 +1,9 @@ +{{ template "k8s-prometheus-adapter.fullname" . }} has been deployed. +In a few minutes you should be able to list metrics using the following command(s): +{{ if .Values.rules.resource }} + kubectl get --raw /apis/metrics.k8s.io/v1beta1 +{{- end }} + kubectl get --raw /apis/custom.metrics.k8s.io/v1beta1 +{{ if .Values.rules.external }} + kubectl get --raw /apis/external.metrics.k8s.io/v1beta1 +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/_helpers.tpl b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/_helpers.tpl new file mode 100644 index 0000000000..edbb829b2b --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/_helpers.tpl @@ -0,0 +1,113 @@ +# Rancher +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "k8s-prometheus-adapter.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "k8s-prometheus-adapter.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "k8s-prometheus-adapter.namespace" -}} +{{- default .Release.Namespace .Values.namespaceOverride -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "k8s-prometheus-adapter.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Generate basic labels +*/}} +{{- define "k8s-prometheus-adapter.labels" }} +helm.sh/chart: {{ include "k8s-prometheus-adapter.chart" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/component: metrics +app.kubernetes.io/part-of: {{ template "k8s-prometheus-adapter.name" . }} +{{- include "k8s-prometheus-adapter.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels }} +{{- end }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "k8s-prometheus-adapter.selectorLabels" }} +app.kubernetes.io/name: {{ include "k8s-prometheus-adapter.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "k8s-prometheus-adapter.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "k8s-prometheus-adapter.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* Get Policy API Version */}} +{{- define "k8s-prometheus-adapter.pdb.apiVersion" -}} +{{- if and (.Capabilities.APIVersions.Has "policy/v1") (semverCompare ">= 1.21-0" .Capabilities.KubeVersion.Version) -}} + {{- print "policy/v1" -}} +{{- else -}} + {{- print "policy/v1beta1" -}} +{{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/certmanager.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/certmanager.yaml new file mode 100644 index 0000000000..4e32c964c6 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/certmanager.yaml @@ -0,0 +1,76 @@ +{{- if .Values.certManager.enabled -}} +--- +# Create a selfsigned Issuer, in order to create a root CA certificate for +# signing webhook serving certificates +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }}-self-signed-issuer + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +spec: + selfSigned: {} +--- +# Generate a CA Certificate used to sign certificates for the webhook +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }}-root-cert + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +spec: + secretName: {{ template "k8s-prometheus-adapter.fullname" . }}-root-cert + duration: {{ .Values.certManager.caCertDuration }} + issuerRef: + name: {{ template "k8s-prometheus-adapter.fullname" . }}-self-signed-issuer + commonName: "ca.webhook.prometheus-adapter" + isCA: true +--- +# Create an Issuer that uses the above generated CA certificate to issue certs +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }}-root-issuer + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +spec: + ca: + secretName: {{ template "k8s-prometheus-adapter.fullname" . }}-root-cert +--- +# Finally, generate a serving certificate for the apiservices to use +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }}-cert + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +spec: + secretName: {{ template "k8s-prometheus-adapter.fullname" . }} + duration: {{ .Values.certManager.certDuration }} + issuerRef: + name: {{ template "k8s-prometheus-adapter.fullname" . }}-root-issuer + dnsNames: + - {{ template "k8s-prometheus-adapter.fullname" . }} + - {{ template "k8s-prometheus-adapter.fullname" . }}.{{ include "k8s-prometheus-adapter.namespace" . }} + - {{ template "k8s-prometheus-adapter.fullname" . }}.{{ include "k8s-prometheus-adapter.namespace" . }}.svc +{{- end -}} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/cluster-role-binding-auth-delegator.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/cluster-role-binding-auth-delegator.yaml new file mode 100644 index 0000000000..6701e6ba08 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/cluster-role-binding-auth-delegator.yaml @@ -0,0 +1,20 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-system-auth-delegator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} +{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/cluster-role-binding-resource-reader.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/cluster-role-binding-resource-reader.yaml new file mode 100644 index 0000000000..67efd2aa2f --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/cluster-role-binding-resource-reader.yaml @@ -0,0 +1,20 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-resource-reader +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "k8s-prometheus-adapter.name" . }}-resource-reader +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} +{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/cluster-role-resource-reader.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/cluster-role-resource-reader.yaml new file mode 100644 index 0000000000..2c690a03cc --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/cluster-role-resource-reader.yaml @@ -0,0 +1,24 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-resource-reader +rules: +- apiGroups: + - "" + resources: + - namespaces + - pods + - services + - configmaps + verbs: + - get + - list + - watch +{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/configmap.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/configmap.yaml new file mode 100644 index 0000000000..17f415d970 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/configmap.yaml @@ -0,0 +1,97 @@ +{{- if not .Values.rules.existing -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +data: + config.yaml: | +{{- if or .Values.rules.default .Values.rules.custom }} + rules: +{{- if .Values.rules.default }} + - seriesQuery: '{__name__=~"^container_.*",container!="POD",namespace!="",pod!=""}' + seriesFilters: [] + resources: + overrides: + namespace: + resource: namespace + pod: + resource: pod + name: + matches: ^container_(.*)_seconds_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>,container!="POD"}[5m])) + by (<<.GroupBy>>) + - seriesQuery: '{__name__=~"^container_.*",container!="POD",namespace!="",pod!=""}' + seriesFilters: + - isNot: ^container_.*_seconds_total$ + resources: + overrides: + namespace: + resource: namespace + pod: + resource: pod + name: + matches: ^container_(.*)_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>,container!="POD"}[5m])) + by (<<.GroupBy>>) + - seriesQuery: '{__name__=~"^container_.*",container!="POD",namespace!="",pod!=""}' + seriesFilters: + - isNot: ^container_.*_total$ + resources: + overrides: + namespace: + resource: namespace + pod: + resource: pod + name: + matches: ^container_(.*)$ + as: "" + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>,container!="POD"}) by (<<.GroupBy>>) + - seriesQuery: '{namespace!="",__name__!~"^container_.*"}' + seriesFilters: + - isNot: .*_total$ + resources: + template: <<.Resource>> + name: + matches: "" + as: "" + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) + - seriesQuery: '{namespace!="",__name__!~"^container_.*"}' + seriesFilters: + - isNot: .*_seconds_total + resources: + template: <<.Resource>> + name: + matches: ^(.*)_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>}[5m])) by (<<.GroupBy>>) + - seriesQuery: '{namespace!="",__name__!~"^container_.*"}' + seriesFilters: [] + resources: + template: <<.Resource>> + name: + matches: ^(.*)_seconds_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>}[5m])) by (<<.GroupBy>>) +{{- end -}} +{{- if .Values.rules.custom }} +{{ toYaml .Values.rules.custom | indent 4 }} +{{- end -}} +{{- end -}} +{{- if .Values.rules.external }} + externalRules: +{{ toYaml .Values.rules.external | indent 4 }} +{{- end -}} +{{- if .Values.rules.resource }} + resourceRules: +{{ toYaml .Values.rules.resource | indent 6 }} +{{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/custom-metrics-apiservice.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/custom-metrics-apiservice.yaml new file mode 100644 index 0000000000..d2dbf32f7c --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/custom-metrics-apiservice.yaml @@ -0,0 +1,32 @@ +{{- if or .Values.rules.default .Values.rules.custom }} +{{- if .Capabilities.APIVersions.Has "apiregistration.k8s.io/v1" }} +apiVersion: apiregistration.k8s.io/v1 +{{- else }} +apiVersion: apiregistration.k8s.io/v1beta1 +{{- end }} +kind: APIService +metadata: +{{- if or .Values.certManager.enabled .Values.customAnnotations }} + annotations: + certmanager.k8s.io/inject-ca-from: {{ printf "%s/%s-root-cert" (include "k8s-prometheus-adapter.namespace" .) (include "k8s-prometheus-adapter.fullname" .) | quote }} + cert-manager.io/inject-ca-from: {{ printf "%s/%s-root-cert" (include "k8s-prometheus-adapter.namespace" .) (include "k8s-prometheus-adapter.fullname" .) | quote }} + {{- if .Values.customAnnotations }} + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} +{{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: v1beta1.custom.metrics.k8s.io +spec: + service: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} + {{- if .Values.tls.enable }} + caBundle: {{ b64enc .Values.tls.ca }} + {{- end }} + group: custom.metrics.k8s.io + version: v1beta1 + insecureSkipTLSVerify: {{ if or .Values.tls.enable .Values.certManager.enabled }}false{{ else }}true{{ end }} + groupPriorityMinimum: 100 + versionPriority: 100 +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/custom-metrics-cluster-role-binding-hpa.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/custom-metrics-cluster-role-binding-hpa.yaml new file mode 100644 index 0000000000..0cc6920836 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/custom-metrics-cluster-role-binding-hpa.yaml @@ -0,0 +1,24 @@ +{{- /* +This if must be aligned with custom-metrics-cluster-role.yaml +as otherwise this binding will point to not existing role. +*/ -}} +{{- if and .Values.rbac.create (or .Values.rules.default .Values.rules.custom) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-hpa-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "k8s-prometheus-adapter.name" . }}-server-resources +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} +{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/custom-metrics-cluster-role.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/custom-metrics-cluster-role.yaml new file mode 100644 index 0000000000..4aa15ffe99 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/custom-metrics-cluster-role.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.rbac.create (or .Values.rules.default .Values.rules.custom) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-server-resources +rules: +- apiGroups: + - custom.metrics.k8s.io + resources: ["*"] + verbs: ["*"] +{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/deployment.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/deployment.yaml new file mode 100644 index 0000000000..64446e685b --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/deployment.yaml @@ -0,0 +1,147 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} +spec: + replicas: {{ .Values.replicas }} + strategy: {{ toYaml .Values.strategy | nindent 4 }} + selector: + matchLabels: + {{- include "k8s-prometheus-adapter.selectorLabels" . | indent 6 }} + template: + metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 8 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | trim | nindent 8 }} + {{- end }} + name: {{ template "k8s-prometheus-adapter.name" . }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + {{- with .Values.podAnnotations }} + {{- toYaml . | trim | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + {{- if .Values.hostNetwork.enabled }} + hostNetwork: true + {{- end }} + {{- if .Values.dnsPolicy }} + dnsPolicy: {{ .Values.dnsPolicy }} + {{- end}} + containers: + - name: {{ .Chart.Name }} + image: "{{ template "system_default_registry" . }}{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - /adapter + - --secure-port={{ .Values.listenPort }} + {{- if or .Values.tls.enable .Values.certManager.enabled }} + - --tls-cert-file=/var/run/serving-cert/tls.crt + - --tls-private-key-file=/var/run/serving-cert/tls.key + {{- end }} + - --cert-dir=/tmp/cert + - --logtostderr=true + - --prometheus-url={{ tpl .Values.prometheus.url . }}{{ if .Values.prometheus.port }}:{{ .Values.prometheus.port }}{{end}}{{ .Values.prometheus.path }} + - --metrics-relist-interval={{ .Values.metricsRelistInterval }} + - --v={{ .Values.logLevel }} + - --config=/etc/adapter/config.yaml + {{- if .Values.extraArguments }} + {{- toYaml .Values.extraArguments | trim | nindent 8 }} + {{- end }} + ports: + - containerPort: {{ .Values.listenPort }} + name: https + livenessProbe: + httpGet: + path: /healthz + port: https + scheme: HTTPS + initialDelaySeconds: 30 + timeoutSeconds: 5 + readinessProbe: + httpGet: + path: /healthz + port: https + scheme: HTTPS + initialDelaySeconds: 30 + timeoutSeconds: 5 + {{- if .Values.resources }} + resources: + {{- toYaml .Values.resources | nindent 10 }} + {{- end }} + {{- with .Values.dnsConfig }} + dnsConfig: + {{ toYaml . | indent 8 }} + {{- end }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["all"] + readOnlyRootFilesystem: true + runAsNonRoot: true + {{- if .Values.runAsUser }} + runAsUser: {{ .Values.runAsUser }} + {{- end }} + volumeMounts: + {{- if .Values.extraVolumeMounts }} + {{ toYaml .Values.extraVolumeMounts | trim | nindent 8 }} + {{ end }} + - mountPath: /etc/adapter/ + name: config + readOnly: true + - mountPath: /tmp + name: tmp + {{- if or .Values.tls.enable .Values.certManager.enabled }} + - mountPath: /var/run/serving-cert + name: volume-serving-cert + readOnly: true + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.nodeSelector }} +{{- toYaml .Values.nodeSelector | nindent 8 }} +{{- end }} + affinity: + {{- toYaml .Values.affinity | nindent 8 }} + topologySpreadConstraints: + {{- toYaml .Values.topologySpreadConstraints | nindent 8 }} + priorityClassName: {{ .Values.priorityClassName }} + {{- if .Values.podSecurityContext }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.tolerations }} +{{- toYaml .Values.tolerations | nindent 8 }} +{{- end }} + {{- if .Values.image.pullSecrets }} + imagePullSecrets: + {{- range .Values.image.pullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} + volumes: + {{- if .Values.extraVolumes }} + {{ toYaml .Values.extraVolumes | trim | nindent 6 }} + {{ end }} + - name: config + configMap: + name: {{ .Values.rules.existing | default (include "k8s-prometheus-adapter.fullname" . ) }} + - name: tmp + emptyDir: {} + {{- if or .Values.tls.enable .Values.certManager.enabled }} + - name: volume-serving-cert + secret: + secretName: {{ template "k8s-prometheus-adapter.fullname" . }} + {{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/external-metrics-apiservice.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/external-metrics-apiservice.yaml new file mode 100644 index 0000000000..7088af7a9e --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/external-metrics-apiservice.yaml @@ -0,0 +1,32 @@ +{{- if .Values.rules.external }} +{{- if .Capabilities.APIVersions.Has "apiregistration.k8s.io/v1" }} +apiVersion: apiregistration.k8s.io/v1 +{{- else }} +apiVersion: apiregistration.k8s.io/v1beta1 +{{- end }} +kind: APIService +metadata: +{{- if or .Values.certManager.enabled .Values.customAnnotations }} + annotations: + certmanager.k8s.io/inject-ca-from: {{ printf "%s/%s-root-cert" (include "k8s-prometheus-adapter.namespace" .) (include "k8s-prometheus-adapter.fullname" .) | quote }} + cert-manager.io/inject-ca-from: {{ printf "%s/%s-root-cert" (include "k8s-prometheus-adapter.namespace" .) (include "k8s-prometheus-adapter.fullname" .) | quote }} + {{- if .Values.customAnnotations }} + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} +{{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: v1beta1.external.metrics.k8s.io +spec: + service: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} + {{- if .Values.tls.enable }} + caBundle: {{ b64enc .Values.tls.ca }} + {{- end }} + group: external.metrics.k8s.io + version: v1beta1 + insecureSkipTLSVerify: {{ if or .Values.tls.enable .Values.certManager.enabled }}false{{ else }}true{{ end }} + groupPriorityMinimum: 100 + versionPriority: 100 +{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/external-metrics-cluster-role-binding-hpa.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/external-metrics-cluster-role-binding-hpa.yaml new file mode 100644 index 0000000000..05547bd323 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/external-metrics-cluster-role-binding-hpa.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.rbac.create .Values.rules.external -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-hpa-controller-external-metrics +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "k8s-prometheus-adapter.name" . }}-external-metrics +subjects: +- kind: ServiceAccount + name: horizontal-pod-autoscaler + namespace: kube-system +{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/external-metrics-cluster-role.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/external-metrics-cluster-role.yaml new file mode 100644 index 0000000000..212ea78b25 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/external-metrics-cluster-role.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.rbac.create .Values.rules.external -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-external-metrics +rules: +- apiGroups: + - "external.metrics.k8s.io" + resources: + - "*" + verbs: + - list + - get + - watch +{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/pdb.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/pdb.yaml new file mode 100644 index 0000000000..205761a9f1 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/pdb.yaml @@ -0,0 +1,23 @@ +{{- if .Values.podDisruptionBudget.enabled }} +apiVersion: {{ include "k8s-prometheus-adapter.pdb.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +spec: + {{- if .Values.podDisruptionBudget.minAvailable }} + minAvailable: {{ .Values.podDisruptionBudget.minAvailable }} + {{- end }} + {{- if .Values.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ .Values.podDisruptionBudget.maxUnavailable }} + {{- end }} + selector: + matchLabels: + {{- include "k8s-prometheus-adapter.selectorLabels" . | indent 6 }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/psp.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/psp.yaml new file mode 100644 index 0000000000..0620357950 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/psp.yaml @@ -0,0 +1,66 @@ +{{- if .Values.global.cattle.psp.enabled -}} +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +spec: + {{- if .Values.hostNetwork.enabled }} + hostNetwork: true + hostPorts: + - min: {{ .Values.listenPort }} + max: {{ .Values.listenPort }} + {{- end }} + fsGroup: + rule: RunAsAny + runAsGroup: + rule: RunAsAny + runAsUser: + rule: MustRunAs + ranges: + - min: 1024 + max: 65535 + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + volumes: + - secret + - emptyDir + - configMap +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-psp +rules: +- apiGroups: + - 'policy' + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "k8s-prometheus-adapter.fullname" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-psp +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "k8s-prometheus-adapter.name" . }}-psp +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} +{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/resource-metrics-apiservice.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/resource-metrics-apiservice.yaml new file mode 100644 index 0000000000..c86037fe86 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/resource-metrics-apiservice.yaml @@ -0,0 +1,32 @@ +{{- if .Values.rules.resource}} +{{- if .Capabilities.APIVersions.Has "apiregistration.k8s.io/v1" }} +apiVersion: apiregistration.k8s.io/v1 +{{- else }} +apiVersion: apiregistration.k8s.io/v1beta1 +{{- end }} +kind: APIService +metadata: +{{- if or .Values.certManager.enabled .Values.customAnnotations }} + annotations: + certmanager.k8s.io/inject-ca-from: {{ printf "%s/%s-root-cert" (include "k8s-prometheus-adapter.namespace" .) (include "k8s-prometheus-adapter.fullname" .) | quote }} + cert-manager.io/inject-ca-from: {{ printf "%s/%s-root-cert" (include "k8s-prometheus-adapter.namespace" .) (include "k8s-prometheus-adapter.fullname" .) | quote }} + {{- if .Values.customAnnotations }} + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} +{{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: v1beta1.metrics.k8s.io +spec: + service: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} + {{- if .Values.tls.enable }} + caBundle: {{ b64enc .Values.tls.ca }} + {{- end }} + group: metrics.k8s.io + version: v1beta1 + insecureSkipTLSVerify: {{ if or .Values.tls.enable .Values.certManager.enabled }}false{{ else }}true{{ end }} + groupPriorityMinimum: 100 + versionPriority: 100 +{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/resource-metrics-cluster-role-binding.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/resource-metrics-cluster-role-binding.yaml new file mode 100644 index 0000000000..3c247e48d2 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/resource-metrics-cluster-role-binding.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.rbac.create .Values.rules.resource -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-hpa-controller-metrics +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "k8s-prometheus-adapter.name" . }}-metrics +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} +{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/resource-metrics-cluster-role.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/resource-metrics-cluster-role.yaml new file mode 100644 index 0000000000..73d8953046 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/resource-metrics-cluster-role.yaml @@ -0,0 +1,23 @@ +{{- if and .Values.rbac.create .Values.rules.resource -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-metrics +rules: +- apiGroups: + - "" + resources: + - pods + - nodes + - nodes/stats + verbs: + - get + - list + - watch +{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/role-binding-auth-reader.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/role-binding-auth-reader.yaml new file mode 100644 index 0000000000..d3c77c1c65 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/role-binding-auth-reader.yaml @@ -0,0 +1,21 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-auth-reader + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} +{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/secret.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/secret.yaml new file mode 100644 index 0000000000..3e7e8887bd --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/secret.yaml @@ -0,0 +1,17 @@ +{{- if .Values.tls.enable -}} +apiVersion: v1 +kind: Secret +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} +type: kubernetes.io/tls +data: + tls.crt: {{ b64enc .Values.tls.certificate }} + tls.key: {{ b64enc .Values.tls.key }} +{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/service.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/service.yaml new file mode 100644 index 0000000000..ddac37cfa1 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/service.yaml @@ -0,0 +1,27 @@ +apiVersion: v1 +kind: Service +metadata: + {{- if or .Values.service.annotations .Values.customAnnotations }} + annotations: + {{- if .Values.service.annotations }} + {{ toYaml .Values.service.annotations | indent 4 }} + {{- end }} + {{- if .Values.customAnnotations }} + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} +spec: + ports: + - port: {{ .Values.service.port }} + protocol: TCP + targetPort: https + selector: + {{- include "k8s-prometheus-adapter.selectorLabels" . | indent 4 }} + type: {{ .Values.service.type }} + {{- if .Values.service.clusterIP }} + clusterIP: {{ .Values.service.clusterIP }} + {{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/serviceaccount.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/serviceaccount.yaml new file mode 100644 index 0000000000..30a169ae0e --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/templates/serviceaccount.yaml @@ -0,0 +1,18 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} +{{- if or .Values.serviceAccount.annotations .Values.customAnnotations }} + annotations: + {{- if .Values.serviceAccount.annotations }} + {{- toYaml .Values.serviceAccount.annotations | nindent 4 }} + {{- end }} + {{- if .Values.customAnnotations }} + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} +{{- end }} +{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/values.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/values.yaml new file mode 100644 index 0000000000..34f9e999dd --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-adapter/values.yaml @@ -0,0 +1,217 @@ +# Default values for k8s-prometheus-adapter.. +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + +affinity: {} + +topologySpreadConstraints: [] + +image: + repository: rancher/mirrored-prometheus-adapter-prometheus-adapter + tag: v0.10.0 + pullPolicy: IfNotPresent + +logLevel: 4 + +metricsRelistInterval: 1m + +listenPort: 6443 + +# User to run adapter container as +runAsUser: 10001 + +nodeSelector: {} + +priorityClassName: "" + +## Override the release namespace (for multi-namespace deployments in combined charts) +namespaceOverride: "" + +## Additional annotations to add to all resources +customAnnotations: {} + # role: custom-metrics + +## Additional labels to add to all resources +customLabels: {} + # monitoring: prometheus-adapter + +# Url to access prometheus +prometheus: + # Value is templated + url: http://prometheus.default.svc + port: 9090 + path: "" + +replicas: 1 + +# k8s 1.21 needs fsGroup to be set for non root deployments +# ref: https://github.com/kubernetes/kubernetes/issues/70679 +podSecurityContext: + fsGroup: 10001 + +rbac: + # Specifies whether RBAC resources should be created + create: true + +serviceAccount: + # Specifies whether a service account should be created + create: true + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: + # ServiceAccount annotations. + # Use case: AWS EKS IAM roles for service accounts + # ref: https://docs.aws.amazon.com/eks/latest/userguide/specify-service-account-role.html + annotations: {} + +# Custom DNS configuration to be added to prometheus-adapter pods +dnsConfig: {} + # nameservers: + # - 1.2.3.4 + # searches: + # - ns1.svc.cluster-domain.example + # - my.dns.search.suffix + # options: + # - name: ndots + # value: "2" + # - name: edns0 + +resources: {} + # requests: + # cpu: 100m + # memory: 128Mi + # limits: + # cpu: 100m + # memory: 128Mi + +rules: + default: true + + custom: [] + # - seriesQuery: '{__name__=~"^some_metric_count$"}' + # resources: + # template: <<.Resource>> + # name: + # matches: "" + # as: "my_custom_metric" + # metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) + + # Mounts a configMap with pre-generated rules for use. Overrides the + # default, custom, external and resource entries + existing: + + external: [] + # - seriesQuery: '{__name__=~"^some_metric_count$"}' + # resources: + # template: <<.Resource>> + # name: + # matches: "" + # as: "my_external_metric" + # metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) + + # resource: + # cpu: + # containerQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>, container!=""}[3m])) by (<<.GroupBy>>) + # nodeQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>, id='/'}[3m])) by (<<.GroupBy>>) + # resources: + # overrides: + # node: + # resource: node + # namespace: + # resource: namespace + # pod: + # resource: pod + # containerLabel: container + # memory: + # containerQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>, container!=""}) by (<<.GroupBy>>) + # nodeQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>,id='/'}) by (<<.GroupBy>>) + # resources: + # overrides: + # node: + # resource: node + # namespace: + # resource: namespace + # pod: + # resource: pod + # containerLabel: container + # window: 3m + +service: + annotations: {} + port: 443 + type: ClusterIP + # clusterIP: 1.2.3.4 + +tls: + enable: false + ca: |- + # Public CA file that signed the APIService + key: |- + # Private key of the APIService + certificate: |- + # Public key of the APIService + +# Any extra arguments +extraArguments: [] + # - --tls-private-key-file=/etc/tls/tls.key + # - --tls-cert-file=/etc/tls/tls.crt + +# Any extra volumes +extraVolumes: [] + # - name: example-name + # hostPath: + # path: /path/on/host + # type: DirectoryOrCreate + # - name: ssl-certs + # hostPath: + # path: /etc/ssl/certs/ca-bundle.crt + # type: File + +# Any extra volume mounts +extraVolumeMounts: [] + # - name: example-name + # mountPath: /path/in/container + # - name: ssl-certs + # mountPath: /etc/ssl/certs/ca-certificates.crt + # readOnly: true + +tolerations: [] + +# Labels added to the pod +podLabels: {} + +# Annotations added to the pod +podAnnotations: {} + +hostNetwork: + # Specifies if prometheus-adapter should be started in hostNetwork mode. + # + # You would require this enabled if you use alternate overlay networking for pods and + # API server unable to communicate with metrics-server. As an example, this is required + # if you use Weave network on EKS. See also dnsPolicy + enabled: false + +# When hostNetwork is enabled, you probably want to set this to ClusterFirstWithHostNet +# dnsPolicy: ClusterFirstWithHostNet + +# Deployment strategy type +strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 25% + maxSurge: 25% + +podDisruptionBudget: + # Specifies if PodDisruptionBudget should be enabled + # When enabled, minAvailable or maxUnavailable should also be defined. + enabled: false + minAvailable: + maxUnavailable: 1 + +certManager: + enabled: false + caCertDuration: 43800h + certDuration: 8760h diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-node-exporter/.helmignore b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-node-exporter/.helmignore new file mode 100644 index 0000000000..f0c1319444 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-node-exporter/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-node-exporter/Chart.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-node-exporter/Chart.yaml new file mode 100644 index 0000000000..ac00322eaa --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-node-exporter/Chart.yaml @@ -0,0 +1,25 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.16.0-0 < 1.27.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-node-exporter +apiVersion: v2 +appVersion: 1.3.1 +description: A Helm chart for prometheus node-exporter +home: https://github.com/prometheus/node_exporter/ +keywords: +- node-exporter +- prometheus +- exporter +maintainers: +- email: gianrubio@gmail.com + name: gianrubio +- email: zanhsieh@gmail.com + name: zanhsieh +name: prometheus-node-exporter +sources: +- https://github.com/prometheus/node_exporter/ +type: application +version: 4.2.0 diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-node-exporter/README.md b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-node-exporter/README.md new file mode 100644 index 0000000000..02de7b14c6 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-node-exporter/README.md @@ -0,0 +1,77 @@ +# Prometheus `Node Exporter` + +Prometheus exporter for hardware and OS metrics exposed by *NIX kernels, written in Go with pluggable metric collectors. + +This chart bootstraps a prometheus [`Node Exporter`](http://github.com/prometheus/node_exporter) daemonset on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. + +## Get Repository Info + +```console +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update +``` + +_See [`helm repo`](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + +## Install Chart + +```console +helm install [RELEASE_NAME] prometheus-community/prometheus-node-exporter +``` + +_See [configuration](#configuring) below._ + +_See [helm install](https://helm.sh/docs/helm/helm_install/) for command documentation._ + +## Uninstall Chart + +```console +helm uninstall [RELEASE_NAME] +``` + +This removes all the Kubernetes components associated with the chart and deletes the release. + +_See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall/) for command documentation._ + +## Upgrading Chart + +```console +helm upgrade [RELEASE_NAME] [CHART] --install +``` + +_See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documentation._ + +### 3.x to 4.x + +Starting from version 4.0.0, the `node exporter` chart is using the [Kubernetes recommended labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/common-labels/). Therefore you have to delete the daemonset before you upgrade. + +```console +kubectl delete daemonset -l app=prometheus-node-exporter +helm upgrade -i prometheus-node-exporter prometheus-community/prometheus-node-exporter +``` + +If you use your own custom [ServiceMonitor](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#servicemonitor) or [PodMonitor](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#podmonitor), please ensure to upgrade their `selector` fields accordingly to the new labels. + +### From 2.x to 3.x + +Change the following: + +```yaml +hostRootFsMount: true +``` + +to: + +```yaml +hostRootFsMount: + enabled: true + mountPropagation: HostToContainer +``` + +## Configuring + +See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To see all configurable options with detailed comments, visit the chart's [values.yaml](./values.yaml), or run these configuration commands: + +```console +helm show values prometheus-community/prometheus-node-exporter +``` diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-node-exporter/templates/NOTES.txt b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-node-exporter/templates/NOTES.txt new file mode 100644 index 0000000000..77d0b3c6fe --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-node-exporter/templates/NOTES.txt @@ -0,0 +1,15 @@ +1. Get the application URL by running these commands: +{{- if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ template "prometheus-node-exporter.namespace" . }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ template "prometheus-node-exporter.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ template "prometheus-node-exporter.namespace" . }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get svc -w {{ template "prometheus-node-exporter.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ template "prometheus-node-exporter.namespace" . }} {{ template "prometheus-node-exporter.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ template "prometheus-node-exporter.namespace" . }} -l "app.kubernetes.io/name={{ template "prometheus-node-exporter.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + echo "Visit http://127.0.0.1:{{ .Values.service.port }} to use your application" + kubectl port-forward --namespace {{ template "prometheus-node-exporter.namespace" . }} $POD_NAME {{ .Values.service.port }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-node-exporter/templates/_helpers.tpl b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-node-exporter/templates/_helpers.tpl new file mode 100644 index 0000000000..10e567d81e --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-node-exporter/templates/_helpers.tpl @@ -0,0 +1,136 @@ +# Rancher +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "prometheus-node-exporter.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "prometheus-node-exporter.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* Generate basic labels */}} +{{- define "prometheus-node-exporter.labels" }} +helm.sh/chart: {{ template "prometheus-node-exporter.chart" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/component: metrics +app.kubernetes.io/part-of: {{ template "prometheus-node-exporter.name" . }} +{{- include "prometheus-node-exporter.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +{{- if .Values.podLabels}} +{{ toYaml .Values.podLabels }} +{{- end }} +{{- if .Values.releaseLabel }} +release: {{ .Release.Name }} +{{- end }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "prometheus-node-exporter.selectorLabels" }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/name: {{ template "prometheus-node-exporter.name" . }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "prometheus-node-exporter.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + + +{{/* +Create the name of the service account to use +*/}} +{{- define "prometheus-node-exporter.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "prometheus-node-exporter.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* +The image to use +*/}} +{{- define "prometheus-node-exporter.image" -}} +{{- if .Values.image.sha -}} +{{- printf "%s:%s@%s" .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) .Values.image.sha }} +{{- else -}} +{{- printf "%s:%s" .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) }} +{{- end }} +{{- end }} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "prometheus-node-exporter.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* +Create the namespace name of the service monitor +*/}} +{{- define "prometheus-node-exporter.monitor-namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- if .Values.prometheus.monitor.namespace -}} + {{- .Values.prometheus.monitor.namespace -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} + {{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-node-exporter/templates/daemonset.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-node-exporter/templates/daemonset.yaml new file mode 100644 index 0000000000..28984a8dd0 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-node-exporter/templates/daemonset.yaml @@ -0,0 +1,234 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ template "prometheus-node-exporter.fullname" . }} + namespace: {{ template "prometheus-node-exporter.namespace" . }} + labels: {{ include "prometheus-node-exporter.labels" . | indent 4 }} +spec: + selector: + matchLabels: + {{- include "prometheus-node-exporter.selectorLabels" . | indent 6 }} + {{- if .Values.updateStrategy }} + updateStrategy: +{{ toYaml .Values.updateStrategy | indent 4 }} + {{- end }} + template: + metadata: + labels: {{ include "prometheus-node-exporter.labels" . | indent 8 }} + {{- if .Values.podAnnotations }} + annotations: + {{- toYaml .Values.podAnnotations | nindent 8 }} + {{- end }} + spec: + automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }} + serviceAccountName: {{ template "prometheus-node-exporter.serviceAccountName" . }} +{{- if .Values.securityContext }} + securityContext: +{{ toYaml .Values.securityContext | indent 8 }} +{{- end }} +{{- if .Values.priorityClassName }} + priorityClassName: {{ .Values.priorityClassName }} +{{- end }} + {{- if .Values.extraInitContainers }} + initContainers: + {{ toYaml .Values.extraInitContainers | nindent 6 }} + {{- end }} + containers: + - name: node-exporter + image: {{ template "system_default_registry" . }}{{ include "prometheus-node-exporter.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - --path.procfs=/host/proc + - --path.sysfs=/host/sys + {{- if .Values.hostRootFsMount.enabled }} + - --path.rootfs=/host/root + {{- end }} + - --web.listen-address=[$(HOST_IP)]:{{ .Values.service.port }} +{{- if .Values.extraArgs }} +{{ toYaml .Values.extraArgs | indent 12 }} +{{- end }} + {{- with .Values.containerSecurityContext }} + securityContext: {{ toYaml . | nindent 12 }} + {{- end }} + env: + - name: HOST_IP + {{- if .Values.service.listenOnAllInterfaces }} + value: 0.0.0.0 + {{- else }} + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.hostIP + {{- end }} + {{- range $key, $value := .Values.env }} + - name: {{ $key }} + value: {{ $value | quote }} + {{- end }} + ports: + - name: {{ .Values.service.portName }} + containerPort: {{ .Values.service.port }} + protocol: TCP + livenessProbe: + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + httpGet: + httpHeaders: + {{- range $_, $header := .Values.livenessProbe.httpGet.httpHeaders }} + - name: {{ $header.name }} + value: {{ $header.value }} + {{- end }} + path: / + port: {{ .Values.service.port }} + scheme: {{ upper .Values.livenessProbe.httpGet.scheme }} + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + readinessProbe: + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + httpGet: + httpHeaders: + {{- range $_, $header := .Values.readinessProbe.httpGet.httpHeaders }} + - name: {{ $header.name }} + value: {{ $header.value }} + {{- end }} + path: / + port: {{ .Values.service.port }} + scheme: {{ upper .Values.readinessProbe.httpGet.scheme }} + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + successThreshold: {{ .Values.readinessProbe.successThreshold }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + resources: +{{ toYaml .Values.resources | indent 12 }} + volumeMounts: + - name: proc + mountPath: /host/proc + readOnly: true + - name: sys + mountPath: /host/sys + readOnly: true + {{- if .Values.hostRootFsMount.enabled }} + - name: root + mountPath: /host/root + {{- with .Values.hostRootFsMount.mountPropagation }} + mountPropagation: {{ . }} + {{- end }} + readOnly: true + {{- end }} + {{- if .Values.extraHostVolumeMounts }} + {{- range $_, $mount := .Values.extraHostVolumeMounts }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: {{ $mount.readOnly }} + {{- if $mount.mountPropagation }} + mountPropagation: {{ $mount.mountPropagation }} + {{- end }} + {{- end }} + {{- end }} + {{- if .Values.sidecarVolumeMount }} + {{- range $_, $mount := .Values.sidecarVolumeMount }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: true + {{- end }} + {{- end }} + {{- if .Values.configmaps }} + {{- range $_, $mount := .Values.configmaps }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + {{- end }} + {{- if .Values.secrets }} + {{- range $_, $mount := .Values.secrets }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + {{- end }} + {{- end }} + {{- end }} +{{- if .Values.sidecars }} +{{ toYaml .Values.sidecars | indent 8 }} + {{- if or .Values.sidecarVolumeMount .Values.sidecarHostVolumeMounts }} + volumeMounts: + {{- range $_, $mount := .Values.sidecarVolumeMount }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: {{ $mount.readOnly }} + {{- end }} + {{- range $_, $mount := .Values.sidecarHostVolumeMounts }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: {{ $mount.readOnly }} + {{- if $mount.mountPropagation }} + mountPropagation: {{ $mount.mountPropagation }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} +{{- if .Values.imagePullSecrets }} + imagePullSecrets: +{{ toYaml .Values.imagePullSecrets | indent 8 }} + {{- end }} + hostNetwork: {{ .Values.hostNetwork }} + hostPID: {{ .Values.hostPID }} +{{- if .Values.affinity }} + affinity: +{{ toYaml .Values.affinity | indent 8 }} +{{- end }} +{{- with .Values.dnsConfig }} + dnsConfig: +{{ toYaml . | indent 8 }} +{{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.nodeSelector }} +{{- toYaml .Values.nodeSelector | nindent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.tolerations }} +{{- toYaml .Values.tolerations | nindent 8 }} +{{- end }} + volumes: + - name: proc + hostPath: + path: /proc + - name: sys + hostPath: + path: /sys + {{- if .Values.hostRootFsMount.enabled }} + - name: root + hostPath: + path: / + {{- end }} + {{- if .Values.extraHostVolumeMounts }} + {{- range $_, $mount := .Values.extraHostVolumeMounts }} + - name: {{ $mount.name }} + hostPath: + path: {{ $mount.hostPath }} + {{- end }} + {{- end }} + {{- if .Values.sidecarVolumeMount }} + {{- range $_, $mount := .Values.sidecarVolumeMount }} + - name: {{ $mount.name }} + emptyDir: + medium: Memory + {{- end }} + {{- end }} + {{- if .Values.sidecarHostVolumeMounts }} + {{- range $_, $mount := .Values.sidecarHostVolumeMounts }} + - name: {{ $mount.name }} + hostPath: + path: {{ $mount.hostPath }} + {{- end }} + {{- end }} + {{- if .Values.configmaps }} + {{- range $_, $mount := .Values.configmaps }} + - name: {{ $mount.name }} + configMap: + name: {{ $mount.name }} + {{- end }} + {{- end }} + {{- if .Values.secrets }} + {{- range $_, $mount := .Values.secrets }} + - name: {{ $mount.name }} + secret: + secretName: {{ $mount.name }} + {{- end }} + {{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-node-exporter/templates/endpoints.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-node-exporter/templates/endpoints.yaml new file mode 100644 index 0000000000..b638c954ac --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-node-exporter/templates/endpoints.yaml @@ -0,0 +1,17 @@ +{{- if .Values.endpoints }} +apiVersion: v1 +kind: Endpoints +metadata: + name: {{ template "prometheus-node-exporter.fullname" . }} + namespace: {{ template "prometheus-node-exporter.namespace" . }} + labels: {{ include "prometheus-node-exporter.labels" . | indent 4 }} +subsets: + - addresses: + {{- range .Values.endpoints }} + - ip: {{ . }} + {{- end }} + ports: + - name: {{ .Values.service.portName }} + port: {{ .Values.service.port }} + protocol: TCP +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-node-exporter/templates/psp-clusterrole.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-node-exporter/templates/psp-clusterrole.yaml new file mode 100644 index 0000000000..91514b313c --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-node-exporter/templates/psp-clusterrole.yaml @@ -0,0 +1,15 @@ +{{- if .Values.rbac.create }} +{{- if .Values.global.cattle.psp.enabled }} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: psp-{{ template "prometheus-node-exporter.fullname" . }} + labels: {{ include "prometheus-node-exporter.labels" . | indent 4 }} +rules: +- apiGroups: ['extensions'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "prometheus-node-exporter.fullname" . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-node-exporter/templates/psp-clusterrolebinding.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-node-exporter/templates/psp-clusterrolebinding.yaml new file mode 100644 index 0000000000..edca1d556c --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-node-exporter/templates/psp-clusterrolebinding.yaml @@ -0,0 +1,17 @@ +{{- if .Values.rbac.create }} +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: psp-{{ template "prometheus-node-exporter.fullname" . }} + labels: {{ include "prometheus-node-exporter.labels" . | indent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp-{{ template "prometheus-node-exporter.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ template "prometheus-node-exporter.fullname" . }} + namespace: {{ template "prometheus-node-exporter.namespace" . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-node-exporter/templates/psp.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-node-exporter/templates/psp.yaml new file mode 100644 index 0000000000..6ec4212ff6 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-node-exporter/templates/psp.yaml @@ -0,0 +1,50 @@ +{{- if .Values.rbac.create }} +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "prometheus-node-exporter.fullname" . }} + namespace: {{ template "prometheus-node-exporter.namespace" . }} + labels: {{ include "prometheus-node-exporter.labels" . | indent 4 }} +{{- if .Values.rbac.pspAnnotations }} + annotations: +{{ toYaml .Values.rbac.pspAnnotations | indent 4 }} +{{- end}} +spec: + privileged: false + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' + - 'hostPath' + hostNetwork: true + hostIPC: false + hostPID: true + hostPorts: + - min: 0 + max: 65535 + runAsUser: + # Permits the container to run with root privileges as well. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-node-exporter/templates/service.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-node-exporter/templates/service.yaml new file mode 100644 index 0000000000..fbed05ca04 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-node-exporter/templates/service.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "prometheus-node-exporter.fullname" . }} + namespace: {{ template "prometheus-node-exporter.namespace" . }} + labels: {{ include "prometheus-node-exporter.labels" . | indent 4 }} +{{- if .Values.service.annotations }} + annotations: +{{ toYaml .Values.service.annotations | indent 4 }} +{{- end }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + {{- if ( and (eq .Values.service.type "NodePort" ) (not (empty .Values.service.nodePort)) ) }} + nodePort: {{ .Values.service.nodePort }} + {{- end }} + targetPort: {{ .Values.service.targetPort }} + protocol: TCP + name: {{ .Values.service.portName }} + selector: + {{- include "prometheus-node-exporter.selectorLabels" . | indent 4 }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-node-exporter/templates/serviceaccount.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-node-exporter/templates/serviceaccount.yaml new file mode 100644 index 0000000000..dc3fee6ac6 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-node-exporter/templates/serviceaccount.yaml @@ -0,0 +1,14 @@ +{{- if .Values.rbac.create -}} +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "prometheus-node-exporter.serviceAccountName" . }} + namespace: {{ template "prometheus-node-exporter.namespace" . }} + labels: {{ include "prometheus-node-exporter.labels" . | indent 4 }} + annotations: +{{ toYaml .Values.serviceAccount.annotations | indent 4 }} +imagePullSecrets: +{{ toYaml .Values.serviceAccount.imagePullSecrets | indent 2 }} +{{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-node-exporter/templates/servicemonitor.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-node-exporter/templates/servicemonitor.yaml new file mode 100644 index 0000000000..04bb1807ce --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-node-exporter/templates/servicemonitor.yaml @@ -0,0 +1,61 @@ +{{- if .Values.prometheus.monitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "prometheus-node-exporter.fullname" . }} + namespace: {{ template "prometheus-node-exporter.monitor-namespace" . }} + labels: {{ include "prometheus-node-exporter.labels" . | indent 4 }} + {{- if .Values.prometheus.monitor.additionalLabels }} + {{- toYaml .Values.prometheus.monitor.additionalLabels | nindent 4 }} + {{- end }} +spec: + jobLabel: {{ default "app.kubernetes.io/name" .Values.prometheus.monitor.jobLabel }} + selector: + matchLabels: + {{- if .Values.prometheus.monitor.selectorOverride }} + {{ toYaml .Values.prometheus.monitor.selectorOverride | indent 6 }} + {{ else }} + {{ include "prometheus-node-exporter.selectorLabels" . | indent 6 }} + {{- end }} + endpoints: + - port: {{ .Values.service.portName }} + scheme: {{ .Values.prometheus.monitor.scheme }} + {{- with .Values.prometheus.monitor.basicAuth }} + basicAuth: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.monitor.bearerTokenFile }} + bearerTokenFile: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.monitor.proxyUrl }} + proxyUrl: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.interval }} + interval: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.relabelings }} + relabelings: + {{- toYaml . | nindent 8 }} + {{- end }} + metricRelabelings: + {{- with .Values.prometheus.monitor.metricRelabelings }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName }} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-node-exporter/values.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-node-exporter/values.yaml new file mode 100644 index 0000000000..ef45db9b4f --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/prometheus-node-exporter/values.yaml @@ -0,0 +1,252 @@ +# Default values for prometheus-node-exporter. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + +image: + repository: rancher/mirrored-prometheus-node-exporter + # Overrides the image tag whose default is {{ printf "v%s" .Chart.AppVersion }} + tag: v1.3.1 + pullPolicy: IfNotPresent + sha: "" + +imagePullSecrets: [] +# - name: "image-pull-secret" + +service: + type: ClusterIP + port: 9796 + targetPort: 9796 + nodePort: + portName: metrics + listenOnAllInterfaces: true + annotations: + prometheus.io/scrape: "true" + +# Additional environment variables that will be passed to the daemonset +env: {} +## env: +## VARIABLE: value + +prometheus: + monitor: + enabled: false + additionalLabels: {} + namespace: "" + + jobLabel: "" + + scheme: http + basicAuth: {} + bearerTokenFile: + tlsConfig: {} + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## Override serviceMonitor selector + ## + selectorOverride: {} + + relabelings: [] + metricRelabelings: [] + interval: "" + scrapeTimeout: 10s + +## Customize the updateStrategy if set +updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 200m + # memory: 50Mi + # requests: + # cpu: 100m + # memory: 30Mi + +serviceAccount: + # Specifies whether a ServiceAccount should be created + create: true + # The name of the ServiceAccount to use. + # If not set and create is true, a name is generated using the fullname template + name: + annotations: {} + imagePullSecrets: [] + automountServiceAccountToken: false + +securityContext: + fsGroup: 65534 + runAsGroup: 65534 + runAsNonRoot: true + runAsUser: 65534 + +containerSecurityContext: {} + # capabilities: + # add: + # - SYS_TIME + +rbac: + ## If true, create & use RBAC resources + ## + create: true + pspAnnotations: {} + +# for deployments that have node_exporter deployed outside of the cluster, list +# their addresses here +endpoints: [] + +# Expose the service to the host network +hostNetwork: true + +# Share the host process ID namespace +hostPID: true + +# Mount the node's root file system (/) at /host/root in the container +hostRootFsMount: + enabled: true + # Defines how new mounts in existing mounts on the node or in the container + # are propagated to the container or node, respectively. Possible values are + # None, HostToContainer, and Bidirectional. If this field is omitted, then + # None is used. More information on: + # https://kubernetes.io/docs/concepts/storage/volumes/#mount-propagation + mountPropagation: HostToContainer + +## Assign a group of affinity scheduling rules +## +affinity: {} +# nodeAffinity: +# requiredDuringSchedulingIgnoredDuringExecution: +# nodeSelectorTerms: +# - matchFields: +# - key: metadata.name +# operator: In +# values: +# - target-host-name + +# Annotations to be added to node exporter pods +podAnnotations: + # Fix for very slow GKE cluster upgrades + cluster-autoscaler.kubernetes.io/safe-to-evict: "true" + +# Extra labels to be added to node exporter pods +podLabels: {} + +## set to true to add the release label so scraping of the servicemonitor with kube-prometheus-stack works out of the box +releaseLabel: false + +# Custom DNS configuration to be added to prometheus-node-exporter pods +dnsConfig: {} +# nameservers: +# - 1.2.3.4 +# searches: +# - ns1.svc.cluster-domain.example +# - my.dns.search.suffix +# options: +# - name: ndots +# value: "2" +# - name: edns0 + +## Assign a nodeSelector if operating a hybrid cluster +## +nodeSelector: {} +# beta.kubernetes.io/arch: amd64 +# beta.kubernetes.io/os: linux + +tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + +## Assign a PriorityClassName to pods if set +# priorityClassName: "" + +## Additional container arguments +## +extraArgs: [] +# - --collector.diskstats.ignored-devices=^(ram|loop|fd|(h|s|v)d[a-z]|nvme\\d+n\\d+p)\\d+$ +# - --collector.textfile.directory=/run/prometheus + +## Additional mounts from the host to node-exporter container +## +extraHostVolumeMounts: [] +# - name: +# hostPath: +# mountPath: +# readOnly: true|false +# mountPropagation: None|HostToContainer|Bidirectional + +## Additional configmaps to be mounted. +## +configmaps: [] +# - name: +# mountPath: +secrets: [] +# - name: +# mountPath: +## Override the deployment namespace +## +namespaceOverride: "" + +## Additional containers for export metrics to text file +## +sidecars: [] +## - name: nvidia-dcgm-exporter +## image: nvidia/dcgm-exporter:1.4.3 + +## Volume for sidecar containers +## +sidecarVolumeMount: [] +## - name: collector-textfiles +## mountPath: /run/prometheus +## readOnly: false + +## Additional mounts from the host to sidecar containers +## +sidecarHostVolumeMounts: [] +# - name: +# hostPath: +# mountPath: +# readOnly: true|false +# mountPropagation: None|HostToContainer|Bidirectional + +## Additional InitContainers to initialize the pod +## +extraInitContainers: [] + +## Liveness probe +## +livenessProbe: + failureThreshold: 3 + httpGet: + httpHeaders: [] + scheme: http + initialDelaySeconds: 0 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + +## Readiness probe +## +readinessProbe: + failureThreshold: 3 + httpGet: + httpHeaders: [] + scheme: http + initialDelaySeconds: 0 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2ControllerManager/.helmignore b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2ControllerManager/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2ControllerManager/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2ControllerManager/Chart.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2ControllerManager/Chart.yaml new file mode 100644 index 0000000000..57f71229bb --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2ControllerManager/Chart.yaml @@ -0,0 +1,14 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.16.0-0 < 1.28.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +name: rke2ControllerManager +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2ControllerManager/README.md b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2ControllerManager/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2ControllerManager/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2ControllerManager/templates/_helpers.tpl b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2ControllerManager/templates/_helpers.tpl new file mode 100644 index 0000000000..8e651dccfd --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2ControllerManager/templates/_helpers.tpl @@ -0,0 +1,166 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $forceHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- end }} +{{- $metricRelabelings := gt (len (keys $clusterNameRelabel)) 0 | ternary (append ($metricRelabelings) ($clusterNameRelabel)) ($metricRelabelings) }} +{{- $metricRelabelings := gt (len (keys $clusterIdRelabel)) 0 | ternary (append ($metricRelabelings) ($clusterIdRelabel)) ($metricRelabelings) }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $forceHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2ControllerManager/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2ControllerManager/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2ControllerManager/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2ControllerManager/templates/pushprox-clients.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2ControllerManager/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2ControllerManager/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2ControllerManager/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2ControllerManager/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2ControllerManager/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2ControllerManager/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2ControllerManager/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2ControllerManager/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2ControllerManager/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2ControllerManager/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2ControllerManager/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2ControllerManager/templates/validate-install-crd.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2ControllerManager/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2ControllerManager/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2ControllerManager/templates/validate-psp-install.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2ControllerManager/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2ControllerManager/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2ControllerManager/values.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2ControllerManager/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2ControllerManager/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Etcd/.helmignore b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Etcd/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Etcd/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Etcd/Chart.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Etcd/Chart.yaml new file mode 100644 index 0000000000..89849a5813 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Etcd/Chart.yaml @@ -0,0 +1,14 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.16.0-0 < 1.28.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +name: rke2Etcd +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Etcd/README.md b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Etcd/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Etcd/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Etcd/templates/_helpers.tpl b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Etcd/templates/_helpers.tpl new file mode 100644 index 0000000000..8e651dccfd --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Etcd/templates/_helpers.tpl @@ -0,0 +1,166 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $forceHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- end }} +{{- $metricRelabelings := gt (len (keys $clusterNameRelabel)) 0 | ternary (append ($metricRelabelings) ($clusterNameRelabel)) ($metricRelabelings) }} +{{- $metricRelabelings := gt (len (keys $clusterIdRelabel)) 0 | ternary (append ($metricRelabelings) ($clusterIdRelabel)) ($metricRelabelings) }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $forceHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Etcd/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Etcd/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Etcd/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Etcd/templates/pushprox-clients.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Etcd/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Etcd/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Etcd/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Etcd/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Etcd/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Etcd/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Etcd/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Etcd/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Etcd/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Etcd/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Etcd/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Etcd/templates/validate-install-crd.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Etcd/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Etcd/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Etcd/templates/validate-psp-install.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Etcd/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Etcd/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Etcd/values.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Etcd/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Etcd/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2IngressNginx/.helmignore b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2IngressNginx/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2IngressNginx/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2IngressNginx/Chart.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2IngressNginx/Chart.yaml new file mode 100644 index 0000000000..6c0f921d3e --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2IngressNginx/Chart.yaml @@ -0,0 +1,14 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.16.0-0 < 1.28.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +name: rke2IngressNginx +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2IngressNginx/README.md b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2IngressNginx/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2IngressNginx/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2IngressNginx/templates/_helpers.tpl b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2IngressNginx/templates/_helpers.tpl new file mode 100644 index 0000000000..8e651dccfd --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2IngressNginx/templates/_helpers.tpl @@ -0,0 +1,166 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $forceHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- end }} +{{- $metricRelabelings := gt (len (keys $clusterNameRelabel)) 0 | ternary (append ($metricRelabelings) ($clusterNameRelabel)) ($metricRelabelings) }} +{{- $metricRelabelings := gt (len (keys $clusterIdRelabel)) 0 | ternary (append ($metricRelabelings) ($clusterIdRelabel)) ($metricRelabelings) }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $forceHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2IngressNginx/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2IngressNginx/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2IngressNginx/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2IngressNginx/templates/pushprox-clients.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2IngressNginx/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2IngressNginx/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2IngressNginx/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2IngressNginx/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2IngressNginx/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2IngressNginx/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2IngressNginx/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2IngressNginx/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2IngressNginx/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2IngressNginx/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2IngressNginx/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2IngressNginx/templates/validate-install-crd.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2IngressNginx/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2IngressNginx/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2IngressNginx/templates/validate-psp-install.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2IngressNginx/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2IngressNginx/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2IngressNginx/values.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2IngressNginx/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2IngressNginx/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Proxy/.helmignore b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Proxy/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Proxy/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Proxy/Chart.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Proxy/Chart.yaml new file mode 100644 index 0000000000..afd0c15446 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Proxy/Chart.yaml @@ -0,0 +1,14 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.16.0-0 < 1.28.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +name: rke2Proxy +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Proxy/README.md b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Proxy/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Proxy/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Proxy/templates/_helpers.tpl b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Proxy/templates/_helpers.tpl new file mode 100644 index 0000000000..8e651dccfd --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Proxy/templates/_helpers.tpl @@ -0,0 +1,166 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $forceHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- end }} +{{- $metricRelabelings := gt (len (keys $clusterNameRelabel)) 0 | ternary (append ($metricRelabelings) ($clusterNameRelabel)) ($metricRelabelings) }} +{{- $metricRelabelings := gt (len (keys $clusterIdRelabel)) 0 | ternary (append ($metricRelabelings) ($clusterIdRelabel)) ($metricRelabelings) }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $forceHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Proxy/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Proxy/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Proxy/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Proxy/templates/pushprox-clients.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Proxy/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Proxy/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Proxy/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Proxy/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Proxy/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Proxy/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Proxy/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Proxy/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Proxy/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Proxy/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Proxy/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Proxy/templates/validate-install-crd.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Proxy/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Proxy/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Proxy/templates/validate-psp-install.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Proxy/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Proxy/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Proxy/values.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Proxy/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Proxy/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Scheduler/.helmignore b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Scheduler/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Scheduler/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Scheduler/Chart.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Scheduler/Chart.yaml new file mode 100644 index 0000000000..a981edb04e --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Scheduler/Chart.yaml @@ -0,0 +1,14 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.16.0-0 < 1.28.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +name: rke2Scheduler +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Scheduler/README.md b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Scheduler/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Scheduler/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Scheduler/templates/_helpers.tpl b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Scheduler/templates/_helpers.tpl new file mode 100644 index 0000000000..8e651dccfd --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Scheduler/templates/_helpers.tpl @@ -0,0 +1,166 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $forceHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- end }} +{{- $metricRelabelings := gt (len (keys $clusterNameRelabel)) 0 | ternary (append ($metricRelabelings) ($clusterNameRelabel)) ($metricRelabelings) }} +{{- $metricRelabelings := gt (len (keys $clusterIdRelabel)) 0 | ternary (append ($metricRelabelings) ($clusterIdRelabel)) ($metricRelabelings) }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $forceHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Scheduler/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Scheduler/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Scheduler/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Scheduler/templates/pushprox-clients.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Scheduler/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Scheduler/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Scheduler/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Scheduler/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Scheduler/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Scheduler/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Scheduler/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Scheduler/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Scheduler/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Scheduler/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Scheduler/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Scheduler/templates/validate-install-crd.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Scheduler/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Scheduler/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Scheduler/templates/validate-psp-install.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Scheduler/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Scheduler/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Scheduler/values.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Scheduler/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rke2Scheduler/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeControllerManager/.helmignore b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeControllerManager/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeControllerManager/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeControllerManager/Chart.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeControllerManager/Chart.yaml new file mode 100644 index 0000000000..7f1c5acdf1 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeControllerManager/Chart.yaml @@ -0,0 +1,14 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.16.0-0 < 1.28.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +name: rkeControllerManager +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeControllerManager/README.md b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeControllerManager/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeControllerManager/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeControllerManager/templates/_helpers.tpl b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeControllerManager/templates/_helpers.tpl new file mode 100644 index 0000000000..8e651dccfd --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeControllerManager/templates/_helpers.tpl @@ -0,0 +1,166 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $forceHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- end }} +{{- $metricRelabelings := gt (len (keys $clusterNameRelabel)) 0 | ternary (append ($metricRelabelings) ($clusterNameRelabel)) ($metricRelabelings) }} +{{- $metricRelabelings := gt (len (keys $clusterIdRelabel)) 0 | ternary (append ($metricRelabelings) ($clusterIdRelabel)) ($metricRelabelings) }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $forceHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeControllerManager/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeControllerManager/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeControllerManager/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeControllerManager/templates/pushprox-clients.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeControllerManager/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeControllerManager/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeControllerManager/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeControllerManager/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeControllerManager/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeControllerManager/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeControllerManager/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeControllerManager/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeControllerManager/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeControllerManager/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeControllerManager/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeControllerManager/templates/validate-install-crd.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeControllerManager/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeControllerManager/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeControllerManager/templates/validate-psp-install.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeControllerManager/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeControllerManager/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeControllerManager/values.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeControllerManager/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeControllerManager/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeEtcd/.helmignore b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeEtcd/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeEtcd/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeEtcd/Chart.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeEtcd/Chart.yaml new file mode 100644 index 0000000000..d1a878f214 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeEtcd/Chart.yaml @@ -0,0 +1,14 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.16.0-0 < 1.28.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +name: rkeEtcd +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeEtcd/README.md b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeEtcd/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeEtcd/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeEtcd/templates/_helpers.tpl b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeEtcd/templates/_helpers.tpl new file mode 100644 index 0000000000..8e651dccfd --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeEtcd/templates/_helpers.tpl @@ -0,0 +1,166 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $forceHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- end }} +{{- $metricRelabelings := gt (len (keys $clusterNameRelabel)) 0 | ternary (append ($metricRelabelings) ($clusterNameRelabel)) ($metricRelabelings) }} +{{- $metricRelabelings := gt (len (keys $clusterIdRelabel)) 0 | ternary (append ($metricRelabelings) ($clusterIdRelabel)) ($metricRelabelings) }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $forceHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeEtcd/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeEtcd/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeEtcd/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeEtcd/templates/pushprox-clients.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeEtcd/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeEtcd/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeEtcd/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeEtcd/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeEtcd/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeEtcd/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeEtcd/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeEtcd/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeEtcd/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeEtcd/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeEtcd/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeEtcd/templates/validate-install-crd.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeEtcd/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeEtcd/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeEtcd/templates/validate-psp-install.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeEtcd/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeEtcd/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeEtcd/values.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeEtcd/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeEtcd/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeIngressNginx/.helmignore b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeIngressNginx/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeIngressNginx/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeIngressNginx/Chart.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeIngressNginx/Chart.yaml new file mode 100644 index 0000000000..1a2ddb0194 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeIngressNginx/Chart.yaml @@ -0,0 +1,14 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.16.0-0 < 1.28.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +name: rkeIngressNginx +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeIngressNginx/README.md b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeIngressNginx/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeIngressNginx/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeIngressNginx/templates/_helpers.tpl b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeIngressNginx/templates/_helpers.tpl new file mode 100644 index 0000000000..8e651dccfd --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeIngressNginx/templates/_helpers.tpl @@ -0,0 +1,166 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $forceHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- end }} +{{- $metricRelabelings := gt (len (keys $clusterNameRelabel)) 0 | ternary (append ($metricRelabelings) ($clusterNameRelabel)) ($metricRelabelings) }} +{{- $metricRelabelings := gt (len (keys $clusterIdRelabel)) 0 | ternary (append ($metricRelabelings) ($clusterIdRelabel)) ($metricRelabelings) }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $forceHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeIngressNginx/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeIngressNginx/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeIngressNginx/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeIngressNginx/templates/pushprox-clients.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeIngressNginx/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeIngressNginx/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeIngressNginx/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeIngressNginx/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeIngressNginx/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeIngressNginx/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeIngressNginx/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeIngressNginx/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeIngressNginx/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeIngressNginx/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeIngressNginx/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeIngressNginx/templates/validate-install-crd.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeIngressNginx/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeIngressNginx/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeIngressNginx/templates/validate-psp-install.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeIngressNginx/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeIngressNginx/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeIngressNginx/values.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeIngressNginx/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeIngressNginx/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeProxy/.helmignore b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeProxy/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeProxy/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeProxy/Chart.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeProxy/Chart.yaml new file mode 100644 index 0000000000..a9637ef8fe --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeProxy/Chart.yaml @@ -0,0 +1,14 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.16.0-0 < 1.28.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +name: rkeProxy +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeProxy/README.md b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeProxy/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeProxy/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeProxy/templates/_helpers.tpl b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeProxy/templates/_helpers.tpl new file mode 100644 index 0000000000..8e651dccfd --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeProxy/templates/_helpers.tpl @@ -0,0 +1,166 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $forceHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- end }} +{{- $metricRelabelings := gt (len (keys $clusterNameRelabel)) 0 | ternary (append ($metricRelabelings) ($clusterNameRelabel)) ($metricRelabelings) }} +{{- $metricRelabelings := gt (len (keys $clusterIdRelabel)) 0 | ternary (append ($metricRelabelings) ($clusterIdRelabel)) ($metricRelabelings) }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $forceHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeProxy/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeProxy/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeProxy/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeProxy/templates/pushprox-clients.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeProxy/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeProxy/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeProxy/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeProxy/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeProxy/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeProxy/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeProxy/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeProxy/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeProxy/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeProxy/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeProxy/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeProxy/templates/validate-install-crd.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeProxy/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeProxy/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeProxy/templates/validate-psp-install.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeProxy/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeProxy/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeProxy/values.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeProxy/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeProxy/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeScheduler/.helmignore b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeScheduler/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeScheduler/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeScheduler/Chart.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeScheduler/Chart.yaml new file mode 100644 index 0000000000..c4a6baeb76 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeScheduler/Chart.yaml @@ -0,0 +1,14 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.16.0-0 < 1.28.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +name: rkeScheduler +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeScheduler/README.md b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeScheduler/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeScheduler/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeScheduler/templates/_helpers.tpl b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeScheduler/templates/_helpers.tpl new file mode 100644 index 0000000000..8e651dccfd --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeScheduler/templates/_helpers.tpl @@ -0,0 +1,166 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $forceHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- end }} +{{- $metricRelabelings := gt (len (keys $clusterNameRelabel)) 0 | ternary (append ($metricRelabelings) ($clusterNameRelabel)) ($metricRelabelings) }} +{{- $metricRelabelings := gt (len (keys $clusterIdRelabel)) 0 | ternary (append ($metricRelabelings) ($clusterIdRelabel)) ($metricRelabelings) }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $forceHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeScheduler/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeScheduler/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeScheduler/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeScheduler/templates/pushprox-clients.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeScheduler/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeScheduler/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeScheduler/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeScheduler/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeScheduler/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeScheduler/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeScheduler/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeScheduler/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeScheduler/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeScheduler/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeScheduler/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeScheduler/templates/validate-install-crd.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeScheduler/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeScheduler/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeScheduler/templates/validate-psp-install.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeScheduler/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeScheduler/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeScheduler/values.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeScheduler/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/rkeScheduler/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/windowsExporter/.helmignore b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/windowsExporter/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/windowsExporter/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/windowsExporter/Chart.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/windowsExporter/Chart.yaml new file mode 100644 index 0000000000..f1cc32344e --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/windowsExporter/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/os: windows + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-windows-exporter +apiVersion: v1 +appVersion: 0.0.2 +description: Sets up monitoring metrics from Windows nodes via Prometheus windows-exporter +maintainers: +- email: arvind.iyengar@rancher.com + name: aiyengar2 +name: windowsExporter +type: application +version: 0.1.1 diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/windowsExporter/README.md b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/windowsExporter/README.md new file mode 100644 index 0000000000..6115b6f257 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/windowsExporter/README.md @@ -0,0 +1,17 @@ +# rancher-windows-exporter + +A Rancher chart based on the [prometheus-community/windows-exporter](https://github.com/prometheus-community/windows_exporter) project (previously called wmi-exporter) that sets up a DaemonSet of clients that can scrape windows-exporter metrics from Windows nodes on a Kubernetes cluster. + +A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR and PrometheusRule CR are also created by this chart to collect metrics and add some recording rules to map `windows_` series with their OS-agnostic counterparts. + +## Node Requirements + +Since Windows does not support privileged pods, this chart expects a Named Pipe (`\\.\pipe\rancher_wins`) to exist on the Windows host that allows containers to communicate with the host. This is done by deploying a [rancher/wins](https://github.com/rancher/wins) server on the host. + +The image used by the chart, [windows_exporter-package](https://github.com/rancher/windows_exporter-package), is configured to create a wins client that communicates with the wins server, alongside a running copy of a particular version of [windows-exporter](https://github.com/prometheus-community/windows_exporter). Through the wins client and wins server, the windows-exporter is able to communicate directly with the Windows host to collect metrics and expose them. + +If the cluster you are installing this chart on is a custom cluster that was created via RKE1 with Windows Support enabled, your nodes should already have the wins server running; this should have been added as part of [the bootstrapping process for adding the Windows node onto your RKE1 cluster](https://github.com/rancher/rancher/blob/master/package/windows/bootstrap.ps1). + +## Configuration + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for an example of how this chart can be used. diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/windowsExporter/scripts/check-wins-version.ps1 b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/windowsExporter/scripts/check-wins-version.ps1 new file mode 100644 index 0000000000..f8452bbef6 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/windowsExporter/scripts/check-wins-version.ps1 @@ -0,0 +1,20 @@ +$ErrorActionPreference = 'Stop' + +$winsPath = "c:\Windows\wins.exe" +$minWinsVersion = [System.Version]"0.1.0" + +function Get-Wins-Version +{ + $winsAppInfo = Invoke-Expression "& $winsPath cli app info | ConvertFrom-Json" + return [System.Version]($winsAppInfo.Server.Version.substring(1)) +} + +# Wait till the wins version installed is at least v0.1.0 +$winsVersion = Get-Wins-Version +while ($winsVersion -lt $minWinsVersion) { + Write-Host $('wins on host must be at least v{0}, found v{1}. Checking again in 10 seconds...' -f $minWinsVersion, $winsVersion) + Start-Sleep -s 10 + $winsVersion = Get-Wins-Version +} + +Write-Host $('Detected wins version on host is v{0}, which is >v{1}. Continuing with installation...' -f $winsVersion, $minWinsVersion) diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/windowsExporter/scripts/proxy-entry.ps1 b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/windowsExporter/scripts/proxy-entry.ps1 new file mode 100644 index 0000000000..9d0581b66f --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/windowsExporter/scripts/proxy-entry.ps1 @@ -0,0 +1,11 @@ +# default +$listenPort = "9796" + +if ($env:LISTEN_PORT) { + $listenPort = $env:LISTEN_PORT +} + +# format "UDP:4789 TCP:8080" +$winsPublish = $('TCP:{0}' -f $listenPort) + +wins.exe cli proxy --publish $winsPublish diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/windowsExporter/scripts/run.ps1 b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/windowsExporter/scripts/run.ps1 new file mode 100644 index 0000000000..c2e980a3f7 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/windowsExporter/scripts/run.ps1 @@ -0,0 +1,78 @@ +$ErrorActionPreference = 'Stop' + +function Create-Directory +{ + param ( + [parameter(Mandatory = $false, ValueFromPipeline = $true)] [string]$Path + ) + + if (Test-Path -Path $Path) { + if (-not (Test-Path -Path $Path -PathType Container)) { + # clean the same path file + Remove-Item -Recurse -Force -Path $Path -ErrorAction Ignore | Out-Null + } + + return + } + + New-Item -Force -ItemType Directory -Path $Path | Out-Null +} + +function Transfer-File +{ + param ( + [parameter(Mandatory = $true)] [string]$Src, + [parameter(Mandatory = $true)] [string]$Dst + ) + + if (Test-Path -PathType leaf -Path $Dst) { + $dstHasher = Get-FileHash -Path $Dst + $srcHasher = Get-FileHash -Path $Src + if ($dstHasher.Hash -eq $srcHasher.Hash) { + return + } + } + + $null = Copy-Item -Force -Path $Src -Destination $Dst +} + +# Copy binary into host +Create-Directory -Path "c:\host\etc\windows-exporter" +Transfer-File -Src "c:\etc\windows-exporter\windows-exporter.exe" -Dst "c:\host\etc\windows-exporter\windows-exporter.exe" + +# Copy binary into prefix path, since wins expects the same path on the host and on the container +$prefixPath = 'c:\' +if ($env:CATTLE_PREFIX_PATH) { + $prefixPath = $env:CATTLE_PREFIX_PATH +} +$winsDirPath = $('{0}etc\windows-exporter' -f $prefixPath) +$winsPath = $('{0}\windows-exporter.exe' -f $winsDirPath) + +Create-Directory -Path $winsDirPath +Transfer-File -Src "c:\etc\windows-exporter\windows-exporter.exe" $winsPath + +# Run wins with defaults +$listenPort = "9796" +$enabledCollectors = "net,os,service,system,cpu,cs,logical_disk" +$maxRequests = "5" + +if ($env:LISTEN_PORT) { + $listenPort = $env:LISTEN_PORT +} + +if ($env:ENABLED_COLLECTORS) { + $enabledCollectors = $env:ENABLED_COLLECTORS +} + +if ($env:MAX_REQUESTS) { + $maxRequests = $env:MAX_REQUESTS +} + +# format "UDP:4789 TCP:8080" +$winsExposes = $('TCP:{0}' -f $listenPort) + +# format "--a=b --c=d" +$winsArgs = $('--collectors.enabled={0} --telemetry.addr=:{1} --telemetry.max-requests={2} --telemetry.path=/metrics' -f $enabledCollectors, $listenPort, $maxRequests) + + +wins.exe cli prc run --path $winsPath --exposes $winsExposes --args "$winsArgs" diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/windowsExporter/templates/_helpers.tpl b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/windowsExporter/templates/_helpers.tpl new file mode 100644 index 0000000000..16975d9d05 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/windowsExporter/templates/_helpers.tpl @@ -0,0 +1,113 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# General + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +The components in this chart create additional resources that expand the longest created name strings. +The longest name that gets created adds and extra 37 characters, so truncation should be 63-35=26. +*/}} +{{- define "windowsExporter.name" -}} +{{ printf "%s-windows-exporter" .Release.Name }} +{{- end -}} + +{{- define "windowsExporter.namespace" -}} +{{- default .Release.Namespace .Values.namespaceOverride -}} +{{- end -}} + +{{- define "windowsExporter.labels" -}} +k8s-app: {{ template "windowsExporter.name" . }} +release: {{ .Release.Name }} +component: "windows-exporter" +provider: kubernetes +{{- end -}} + +# Client + +{{- define "windowsExporter.client.nodeSelector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: windows +{{- else -}} +kubernetes.io/os: windows +{{- end -}} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector }} +{{- end }} +{{- end -}} + +{{- define "windowsExporter.client.tolerations" -}} +{{- if .Values.clients.tolerations -}} +{{ toYaml .Values.clients.tolerations }} +{{- else -}} +- operator: Exists +{{- end -}} +{{- end -}} + +{{- define "windowsExporter.client.env" -}} +- name: LISTEN_PORT + value: {{ required "Need .Values.clients.port to figure out where to get metrics from" .Values.clients.port | quote }} +{{- if .Values.clients.enabledCollectors }} +- name: ENABLED_COLLECTORS + value: {{ .Values.clients.enabledCollectors | quote }} +{{- end }} +{{- if .Values.clients.env }} +{{ toYaml .Values.clients.env }} +{{- end }} +{{- end -}} + +{{- define "windowsExporter.validatePathPrefix" -}} +{{- if .Values.global.cattle.rkeWindowsPathPrefix -}} +{{- $prefixPath := (.Values.global.cattle.rkeWindowsPathPrefix | replace "/" "\\") -}} +{{- if (not (hasSuffix "\\" $prefixPath)) -}} +{{- fail (printf ".Values.global.cattle.rkeWindowsPathPrefix must end in '/' or '\\', found %s" $prefixPath) -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{- define "windowsExporter.renamedMetrics" -}} +{{- $renamed := dict -}} +{{/* v0.15.0 */}} +{{- $_ := set $renamed "windows_mssql_transactions_active_total" "windows_mssql_transactions_active" -}} +{{/* v0.16.0 */}} +{{- $_ := set $renamed "windows_adfs_ad_login_connection_failures" "windows_adfs_ad_login_connection_failures_total" -}} +{{- $_ := set $renamed "windows_adfs_certificate_authentications" "windows_adfs_certificate_authentications_total" -}} +{{- $_ := set $renamed "windows_adfs_device_authentications" "windows_adfs_device_authentications_total" -}} +{{- $_ := set $renamed "windows_adfs_extranet_account_lockouts" "windows_adfs_extranet_account_lockouts_total" -}} +{{- $_ := set $renamed "windows_adfs_federated_authentications" "windows_adfs_federated_authentications_total" -}} +{{- $_ := set $renamed "windows_adfs_passport_authentications" "windows_adfs_passport_authentications_total" -}} +{{- $_ := set $renamed "windows_adfs_password_change_failed" "windows_adfs_password_change_failed_total" -}} +{{- $_ := set $renamed "windows_adfs_password_change_succeeded" "windows_adfs_password_change_succeeded_total" -}} +{{- $_ := set $renamed "windows_adfs_token_requests" "windows_adfs_token_requests_total" -}} +{{- $_ := set $renamed "windows_adfs_windows_integrated_authentications" "windows_adfs_windows_integrated_authentications_total" -}} +{{- $_ := set $renamed "windows_net_packets_outbound_errors" "windows_net_packets_outbound_errors_total" -}} +{{- $_ := set $renamed "windows_net_packets_received_discarded" "windows_net_packets_received_discarded_total" -}} +{{- $_ := set $renamed "windows_net_packets_received_errors" "windows_net_packets_received_errors_total" -}} +{{- $_ := set $renamed "windows_net_packets_received_total" "windows_net_packets_received_total_total" -}} +{{- $_ := set $renamed "windows_net_packets_received_unknown" "windows_net_packets_received_unknown_total" -}} +{{- $_ := set $renamed "windows_dns_memory_used_bytes_total" "windows_dns_memory_used_bytes" -}} +{{- $renamed | toJson -}} +{{- end -}} + +{{- define "windowsExporter.renamedMetricsRelabeling" -}} +{{- range $original, $new := (include "windowsExporter.renamedMetrics" . | fromJson) -}} +- sourceLabels: [__name__] + regex: {{ $original }} + replacement: '{{ $new }}' + targetLabel: __name__ +{{ end -}} +{{- end -}} + +{{- define "windowsExporter.renamedMetricsRules" -}} +{{- range $original, $new := (include "windowsExporter.renamedMetrics" . | fromJson) -}} +- record: {{ $original }} + expr: {{ $new }} +{{ end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/windowsExporter/templates/configmap.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/windowsExporter/templates/configmap.yaml new file mode 100644 index 0000000000..e7647a4077 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/windowsExporter/templates/configmap.yaml @@ -0,0 +1,10 @@ +{{- if .Values.clients }}{{ if .Values.clients.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "windowsExporter.name" . }}-scripts + namespace: {{ template "windowsExporter.namespace" . }} + labels: {{ include "windowsExporter.labels" . | nindent 4 }} +data: +{{ (.Files.Glob "scripts/*").AsConfig | indent 2 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/windowsExporter/templates/daemonset.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/windowsExporter/templates/daemonset.yaml new file mode 100644 index 0000000000..a64d19a3e1 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/windowsExporter/templates/daemonset.yaml @@ -0,0 +1,77 @@ +{{- if .Values.clients }}{{ if .Values.clients.enabled }} +{{ include "windowsExporter.validatePathPrefix" . }} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ template "windowsExporter.name" . }} + namespace: {{ template "windowsExporter.namespace" . }} + labels: {{ include "windowsExporter.labels" . | nindent 4 }} +spec: + selector: + matchLabels: {{ include "windowsExporter.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "windowsExporter.labels" . | nindent 8 }} + spec: + nodeSelector: {{ include "windowsExporter.client.nodeSelector" . | nindent 8 }} + tolerations: {{ include "windowsExporter.client.tolerations" . | nindent 8 }} + serviceAccountName: {{ template "windowsExporter.name" . }} + containers: + - name: exporter-node-proxy + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: ["pwsh", "-f", "c:/scripts/proxy-entry.ps1"] + ports: + - name: http + containerPort: {{ required "Need .Values.clients.port to figure out where to get metrics from" .Values.clients.port }} + env: {{ include "windowsExporter.client.env" . | nindent 10 }} +{{- if .Values.resources }} + resources: {{ toYaml .Values.clients.proxy.resources | nindent 10 }} +{{- end }} + volumeMounts: + - name: wins-pipe-proxy + mountPath: \\.\pipe\rancher_wins_proxy + - name: exporter-scripts + mountPath: c:/scripts/ + - name: exporter-node + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: ["pwsh", "-f", "c:/scripts/run.ps1"] +{{- if .Values.clients.args }} + args: {{ .Values.clients.args }} +{{- end }} + env: {{ include "windowsExporter.client.env" . | nindent 8 }} + - name: CATTLE_PREFIX_PATH + value: {{ default "c:\\" .Values.global.cattle.rkeWindowsPathPrefix | replace "/" "\\" }} +{{- if .Values.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} +{{- end }} + volumeMounts: + - name: wins-pipe + mountPath: \\.\pipe\rancher_wins + - name: binary-host-path + mountPath: c:/host/etc/windows-exporter + - name: exporter-scripts + mountPath: c:/scripts/ + initContainers: + - name: check-wins-version + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: ["pwsh", "-f", "c:/scripts/check-wins-version.ps1"] + volumeMounts: + - name: wins-pipe + mountPath: \\.\pipe\rancher_wins + - name: exporter-scripts + mountPath: c:/scripts/ + volumes: + - name: wins-pipe + hostPath: + path: \\.\pipe\rancher_wins + - name: wins-pipe-proxy + hostPath: + path: \\.\pipe\rancher_wins_proxy + - name: binary-host-path + hostPath: + path: {{ default "c:\\" .Values.global.cattle.rkeWindowsPathPrefix | replace "\\" "/" }}etc/windows-exporter + type: DirectoryOrCreate + - name: exporter-scripts + configMap: + name: {{ template "windowsExporter.name" . }}-scripts +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/windowsExporter/templates/prometheusrule.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/windowsExporter/templates/prometheusrule.yaml new file mode 100644 index 0000000000..f31983122a --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/windowsExporter/templates/prometheusrule.yaml @@ -0,0 +1,13 @@ +{{- if and .Values.prometheusRule .Values.clients }}{{- if and .Values.prometheusRule.enabled .Values.clients.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + labels: {{ include "windowsExporter.labels" . | nindent 4 }} + name: {{ template "windowsExporter.name" . }} + namespace: {{ template "windowsExporter.namespace" . }} +spec: + groups: + - name: windows-exporter-relabel.rules + rules: +{{- include "windowsExporter.renamedMetricsRules" . | nindent 4 -}} +{{- end }}{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/windowsExporter/templates/rbac.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/windowsExporter/templates/rbac.yaml new file mode 100644 index 0000000000..e3da3e1605 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/windowsExporter/templates/rbac.yaml @@ -0,0 +1,81 @@ +{{- if .Values.clients }}{{ if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "windowsExporter.name" . }} + namespace: {{ template "windowsExporter.namespace" . }} + labels: {{ include "windowsExporter.labels" . | nindent 4 }} +rules: +- apiGroups: ['authentication.k8s.io'] + resources: ['tokenreviews'] + verbs: ['create'] +- apiGroups: ['authorization.k8s.io'] + resources: ['subjectaccessreviews'] + verbs: ['create'] +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: ['{{ template "windowsExporter.name" . }}'] +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "windowsExporter.name" . }} + namespace: {{ template "windowsExporter.namespace" . }} + labels: {{ include "windowsExporter.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "windowsExporter.name" . }} +subjects: +- kind: ServiceAccount + name: {{ template "windowsExporter.name" . }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "windowsExporter.name" . }} + namespace: {{ template "windowsExporter.namespace" . }} + labels: {{ include "windowsExporter.labels" . | nindent 4 }} +{{- if .Values.clients.imagePullSecrets }} +imagePullSecrets: {{ toYaml .Values.clients.imagePullSecrets | nindent 2 }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "windowsExporter.name" . }} + namespace: {{ template "windowsExporter.namespace" . }} + labels: {{ include "windowsExporter.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' + - 'hostPath' + allowedHostPaths: + - pathPrefix: \\.\pipe\rancher_wins + - pathPrefix: \\.\pipe\rancher_wins_proxy + - pathPrefix: c:/etc/windows-exporter +{{- end }}{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/windowsExporter/templates/service.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/windowsExporter/templates/service.yaml new file mode 100644 index 0000000000..03b87faaef --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/windowsExporter/templates/service.yaml @@ -0,0 +1,15 @@ +{{- if and .Values.clients }}{{- if and .Values.clients.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "windowsExporter.name" . }} + namespace: {{ template "windowsExporter.namespace" . }} + labels: {{ include "windowsExporter.labels" . | nindent 4 }} +spec: + ports: + - name: windows-metrics + port: {{ required "Need .Values.clients.port to figure out where to get metrics from" .Values.clients.port }} + protocol: TCP + targetPort: {{ .Values.clients.port }} + selector: {{ include "windowsExporter.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/windowsExporter/templates/servicemonitor.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/windowsExporter/templates/servicemonitor.yaml new file mode 100644 index 0000000000..26ece9b05a --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/windowsExporter/templates/servicemonitor.yaml @@ -0,0 +1,41 @@ +{{- if and .Values.serviceMonitor .Values.clients }}{{- if and .Values.serviceMonitor.enabled .Values.clients.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: {{ include "windowsExporter.labels" . | nindent 4 }} + name: {{ template "windowsExporter.name" . }} + namespace: {{ template "windowsExporter.namespace" . }} +spec: + selector: + matchLabels: {{ include "windowsExporter.labels" . | nindent 6 }} + namespaceSelector: + matchNames: + - {{ template "windowsExporter.namespace" . }} + jobLabel: component + podTargetLabels: + - component + endpoints: + - port: windows-metrics + metricRelabelings: +{{- include "windowsExporter.renamedMetricsRelabeling" . | nindent 4 -}} + - sourceLabels: [__name__] + regex: 'wmi_(.*)' + replacement: 'windows_$1' + targetLabel: __name__ + - sourceLabels: [volume, nic] + regex: (.*);(.*) + separator: '' + targetLabel: device + action: replace + replacement: $1$2 + - sourceLabels: [__name__] + regex: windows_cs_logical_processors + replacement: 'system' + targetLabel: mode + relabelings: + - separator: ':' + sourceLabels: + - __meta_kubernetes_pod_host_ip + - __meta_kubernetes_pod_container_port_number + targetLabel: instance +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/charts/windowsExporter/values.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/windowsExporter/values.yaml new file mode 100644 index 0000000000..6de9984ce4 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/charts/windowsExporter/values.yaml @@ -0,0 +1,52 @@ +# Default values for rancher-windows-exporter. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + rkeWindowsPathPrefix: "c:\\" + +# Configure ServiceMonitor that monitors metrics +serviceMonitor: + enabled: true + +# Configure PrometheusRule that renames existing metrics +prometheusRule: + enabled: true + +## Components scraping metrics from Windows nodes +## +clients: + enabled: true + + port: 9796 + image: + repository: rancher/windows_exporter-package + tag: v0.0.5 + os: "windows" + + # Specify the IP addresses of nodes that you want to collect metrics from + endpoints: [] + + # Get more details on https://github.com/prometheus-community/windows_exporter + args: [] + env: {} + enabledCollectors: "net,os,service,system,cpu,cs,logical_disk,tcp,memory,container" + + # Resource limits + resources: {} + + # Options to select nodes to target for scraping Windows metrics + nodeSelector: {} # Note: {kubernetes.io/os: windows} is default and cannot be overridden + tolerations: [] # Note: if not specified, the default option is to use [{operator: Exists}] + + # Image Pull Secrets for the service account used by the clients + imagePullSecrets: {} + + proxy: + resources: {} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/files/ingress-nginx/nginx.json b/charts/rancher-monitoring/102.0.5+up40.1.2/files/ingress-nginx/nginx.json new file mode 100644 index 0000000000..565352235a --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/files/ingress-nginx/nginx.json @@ -0,0 +1,1445 @@ +{ + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": "$datasource", + "enable": true, + "expr": "sum(changes(nginx_ingress_controller_config_last_reload_successful_timestamp_seconds{instance!=\"unknown\",controller_class=~\"$controller_class\",namespace=~\"$namespace\"}[30s])) by (controller_class)", + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "Config Reloads", + "showIn": 0, + "step": "30s", + "tagKeys": "controller_class", + "tags": [], + "titleFormat": "Config Reloaded", + "type": "tags" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "iteration": 1534359654832, + "links": [], + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "format": "ops", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 20, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "round(sum(irate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\"}[2m])), 0.001)", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "", + "title": "Controller Request Volume", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 6, + "y": 0 + }, + "id": 82, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(avg_over_time(nginx_ingress_controller_nginx_process_connections{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",state=\"active\"}[2m]))", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "", + "title": "Controller Connections", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "format": "percentunit", + "gauge": { + "maxValue": 100, + "minValue": 80, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": false + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 12, + "y": 0 + }, + "id": 21, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(rate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\",status!~\"[4-5].*\"}[2m])) / sum(rate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\"}[2m]))", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "95, 99, 99.5", + "title": "Controller Success Rate (non-4|5xx responses)", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 18, + "y": 0 + }, + "id": 81, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "avg(irate(nginx_ingress_controller_success{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}[1m])) * 60", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "", + "title": "Config Reloads", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "total" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 21, + "y": 0 + }, + "id": 83, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "count(nginx_ingress_controller_config_last_reload_successful{controller_pod=~\"$controller\",controller_namespace=~\"$namespace\"} == 0)", + "format": "time_series", + "instant": true, + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "", + "title": "Last Config Failed", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 3 + }, + "height": "200px", + "id": 86, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": false, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 300, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "round(sum(irate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (ingress), 0.001)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "metric": "network", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Ingress Request Volume", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "max - istio-proxy": "#890f02", + "max - master": "#bf1b00", + "max - prometheus": "#bf1b00" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 2, + "editable": false, + "error": false, + "fill": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 3 + }, + "id": 87, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": true, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 300, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\",ingress=~\"$ingress\",status!~\"[4-5].*\"}[2m])) by (ingress) / sum(rate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (ingress)", + "format": "time_series", + "instant": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "metric": "container_memory_usage:sort_desc", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Ingress Success Rate (non-4|5xx responses)", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 1, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 10 + }, + "height": "200px", + "id": 32, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (irate (nginx_ingress_controller_request_size_sum{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}[2m]))", + "format": "time_series", + "instant": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "Received", + "metric": "network", + "refId": "A", + "step": 10 + }, + { + "expr": "- sum (irate (nginx_ingress_controller_response_size_sum{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}[2m]))", + "format": "time_series", + "hide": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "Sent", + "metric": "network", + "refId": "B", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Network I/O pressure", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "max - istio-proxy": "#890f02", + "max - master": "#bf1b00", + "max - prometheus": "#bf1b00" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 2, + "editable": false, + "error": false, + "fill": 0, + "grid": {}, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 10 + }, + "id": 77, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(nginx_ingress_controller_nginx_process_resident_memory_bytes{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}) ", + "format": "time_series", + "instant": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "nginx", + "metric": "container_memory_usage:sort_desc", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Average Memory Usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "max - istio-proxy": "#890f02", + "max - master": "#bf1b00" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 3, + "editable": false, + "error": false, + "fill": 0, + "grid": {}, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 10 + }, + "height": "", + "id": 79, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sort": null, + "sortDesc": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg (rate (nginx_ingress_controller_nginx_process_cpu_seconds_total{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}[2m])) ", + "format": "time_series", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "nginx", + "metric": "container_cpu", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Average CPU Usage", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": "cores", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "columns": [], + "datasource": "$datasource", + "fontSize": "100%", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 16 + }, + "hideTimeOverride": false, + "id": 75, + "links": [], + "pageSize": 7, + "repeat": null, + "repeatDirection": "h", + "scroll": true, + "showHeader": true, + "sort": { + "col": 1, + "desc": true + }, + "styles": [ + { + "alias": "Ingress", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "ingress", + "preserveFormat": false, + "sanitize": false, + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "Requests", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Value #A", + "thresholds": [ + "" + ], + "type": "number", + "unit": "ops" + }, + { + "alias": "Errors", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Value #B", + "thresholds": [], + "type": "number", + "unit": "ops" + }, + { + "alias": "P50 Latency", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": false, + "pattern": "Value #C", + "thresholds": [], + "type": "number", + "unit": "dtdurations" + }, + { + "alias": "P90 Latency", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "pattern": "Value #D", + "thresholds": [], + "type": "number", + "unit": "dtdurations" + }, + { + "alias": "P99 Latency", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "pattern": "Value #E", + "thresholds": [], + "type": "number", + "unit": "dtdurations" + }, + { + "alias": "IN", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Value #F", + "thresholds": [ + "" + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Time", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "OUT", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Value #G", + "thresholds": [], + "type": "number", + "unit": "Bps" + } + ], + "targets": [ + { + "expr": "histogram_quantile(0.50, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (le, ingress))", + "format": "table", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "refId": "C" + }, + { + "expr": "histogram_quantile(0.90, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (le, ingress))", + "format": "table", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "refId": "D" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (le, ingress))", + "format": "table", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }}", + "refId": "E" + }, + { + "expr": "sum(irate(nginx_ingress_controller_request_size_sum{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (ingress)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "refId": "F" + }, + { + "expr": "sum(irate(nginx_ingress_controller_response_size_sum{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (ingress)", + "format": "table", + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "refId": "G" + } + ], + "timeFrom": null, + "title": "Ingress Percentile Response Times and Transfer Rates", + "transform": "table", + "transparent": false, + "type": "table" + }, + { + "columns": [ + { + "text": "Current", + "value": "current" + } + ], + "datasource": "$datasource", + "fontSize": "100%", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 24 + }, + "height": "1024", + "id": 85, + "links": [], + "pageSize": 7, + "scroll": true, + "showHeader": true, + "sort": { + "col": 1, + "desc": false + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + }, + { + "alias": "TTL", + "colorMode": "cell", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "pattern": "Current", + "thresholds": [ + "0", + "691200" + ], + "type": "number", + "unit": "s" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "avg(nginx_ingress_controller_ssl_expire_time_seconds{kubernetes_pod_name=~\"$controller\",namespace=~\"$namespace\",ingress=~\"$ingress\"}) by (host) - time()", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ host }}", + "metric": "gke_letsencrypt_cert_expiration", + "refId": "A", + "step": 1 + } + ], + "title": "Ingress Certificate Expiry", + "transform": "timeseries_aggregations", + "type": "table" + } + ], + "refresh": "5s", + "schemaVersion": 16, + "style": "dark", + "tags": [ + "nginx" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "Namespace", + "multi": false, + "name": "namespace", + "options": [], + "query": "label_values(nginx_ingress_controller_config_hash, controller_namespace)", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "Controller Class", + "multi": false, + "name": "controller_class", + "options": [], + "query": "label_values(nginx_ingress_controller_config_hash{namespace=~\"$namespace\"}, controller_class) ", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "Controller", + "multi": false, + "name": "controller", + "options": [], + "query": "label_values(nginx_ingress_controller_config_hash{namespace=~\"$namespace\",controller_class=~\"$controller_class\"}, controller_pod) ", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "tags": [], + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "Ingress", + "multi": false, + "name": "ingress", + "options": [], + "query": "label_values(nginx_ingress_controller_requests{namespace=~\"$namespace\",controller_class=~\"$controller_class\",controller_pod=~\"$controller\"}, ingress) ", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "2m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "NGINX / Ingress Controller", + "uid": "nginx", + "version": 1 +} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/files/ingress-nginx/request-handling-performance.json b/charts/rancher-monitoring/102.0.5+up40.1.2/files/ingress-nginx/request-handling-performance.json new file mode 100644 index 0000000000..156e33123d --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/files/ingress-nginx/request-handling-performance.json @@ -0,0 +1,963 @@ +{ + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "", + "editable": true, + "gnetId": 9614, + "graphTooltip": 1, + "id": null, + "iteration": 1582146566338, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Total time taken for nginx and upstream servers to process a request and send a response", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 91, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(\n 0.5,\n sum by (le)(\n rate(\n nginx_ingress_controller_request_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "interval": "", + "legendFormat": ".5", + "refId": "D" + }, + { + "expr": "histogram_quantile(\n 0.95,\n sum by (le)(\n rate(\n nginx_ingress_controller_request_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "interval": "", + "legendFormat": ".95", + "refId": "B" + }, + { + "expr": "histogram_quantile(\n 0.99,\n sum by (le)(\n rate(\n nginx_ingress_controller_request_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "interval": "", + "legendFormat": ".99", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Total request handling time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "The time spent on receiving the response from the upstream server", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 94, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(\n 0.5,\n sum by (le)(\n rate(\n nginx_ingress_controller_response_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": ".5", + "refId": "D" + }, + { + "expr": "histogram_quantile(\n 0.95,\n sum by (le)(\n rate(\n nginx_ingress_controller_response_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "interval": "", + "legendFormat": ".95", + "refId": "B" + }, + { + "expr": "histogram_quantile(\n 0.99,\n sum by (le)(\n rate(\n nginx_ingress_controller_response_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "interval": "", + "legendFormat": ".99", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Upstream response time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "hiddenSeries": false, + "id": 93, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": " sum by (path)(\n rate(\n nginx_ingress_controller_request_duration_seconds_count{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Request volume by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "For each path observed, its median upstream response time", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "hiddenSeries": false, + "id": 98, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(\n .5,\n sum by (le, path)(\n rate(\n nginx_ingress_controller_response_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Median upstream response time by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Percentage of 4xx and 5xx responses among all responses.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "hiddenSeries": false, + "id": 100, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (path) (rate(nginx_ingress_controller_request_duration_seconds_count{\n ingress =~ \"$ingress\",\n status =~ \"[4-5].*\"\n}[1m])) / sum by (path) (rate(nginx_ingress_controller_request_duration_seconds_count{\n ingress =~ \"$ingress\",\n}[1m]))", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Response error rate by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "For each path observed, the sum of upstream request time", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "hiddenSeries": false, + "id": 102, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (path) (rate(nginx_ingress_controller_response_duration_seconds_sum{ingress =~ \"$ingress\"}[1m]))", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Upstream time consumed by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "hiddenSeries": false, + "id": 101, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": " sum (\n rate(\n nginx_ingress_controller_request_duration_seconds_count{\n ingress =~ \"$ingress\",\n status =~\"[4-5].*\",\n }[1m]\n )\n ) by(path, status)\n", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }} {{ status }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Response error volume by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "hiddenSeries": false, + "id": 99, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (\n rate (\n nginx_ingress_controller_response_size_sum {\n ingress =~ \"$ingress\",\n }[1m]\n )\n) by (path) / sum (\n rate(\n nginx_ingress_controller_response_size_count {\n ingress =~ \"$ingress\",\n }[1m]\n )\n) by (path)\n", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }}", + "refId": "D" + }, + { + "expr": " sum (rate(nginx_ingress_controller_response_size_bucket{\n ingress =~ \"$ingress\",\n }[1m])) by (le)\n", + "hide": true, + "legendFormat": "{{le}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Average response size by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "hiddenSeries": false, + "id": 96, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (\n rate(\n nginx_ingress_controller_ingress_upstream_latency_seconds_sum {\n ingress =~ \"$ingress\",\n }[1m]\n)) / sum (\n rate(\n nginx_ingress_controller_ingress_upstream_latency_seconds_count {\n ingress =~ \"$ingress\",\n }[1m]\n )\n)\n", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "average", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Upstream service latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "30s", + "schemaVersion": 22, + "style": "dark", + "tags": [ + "nginx" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": ".*", + "current": {}, + "datasource": "$datasource", + "definition": "label_values(nginx_ingress_controller_requests, ingress) ", + "hide": 0, + "includeAll": true, + "label": "Service Ingress", + "multi": false, + "name": "ingress", + "options": [], + "query": "label_values(nginx_ingress_controller_requests, ingress) ", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "2m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "NGINX / Request Handling Performance", + "uid": "4GFbkOsZk", + "version": 1 +} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/files/rancher/cluster/rancher-cluster-nodes.json b/charts/rancher-monitoring/102.0.5+up40.1.2/files/rancher/cluster/rancher-cluster-nodes.json new file mode 100644 index 0000000000..1d4943501b --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/files/rancher/cluster/rancher-cluster-nodes.json @@ -0,0 +1,793 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "links": [], + "panels": [ + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\",mode=\"idle\"}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m] ({{instance}})" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_load5 OR avg_over_time(windows_system_processor_queue_length[5m])) by (instance)", + "interval": "", + "legendFormat": "Load[5m] ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(node_load1 OR avg_over_time(windows_system_processor_queue_length[1m])) by (instance)", + "interval": "", + "legendFormat": "Load[1m] ({{instance}})", + "refId": "B" + }, + { + "expr": "sum(node_load15 OR avg_over_time(windows_system_processor_queue_length[15m])) by (instance)", + "interval": "", + "legendFormat": "Load[15m] ({{instance}})", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Load Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - sum(node_memory_MemAvailable_bytes OR windows_os_physical_memory_free_bytes) by (instance) / sum(node_memory_MemTotal_bytes OR windows_cs_physical_memory_bytes) by (instance) ", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - (sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\"} OR windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) by (instance) / sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"} OR windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) by (instance))", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_disk_read_bytes_total[$__rate_interval]) OR rate(windows_logical_disk_read_bytes_total[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Read ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(rate(node_disk_written_bytes_total[$__rate_interval]) OR rate(windows_logical_disk_write_bytes_total[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Write ({{instance}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_network_receive_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) by (instance) OR sum(rate(windows_net_packets_received_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Receive Errors ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(rate(node_network_receive_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) by (instance) OR sum(rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Receive Total ({{instance}})", + "refId": "B" + }, + { + "expr": "sum(rate(node_network_transmit_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) by (instance) OR sum(rate(windows_net_packets_outbound_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Transmit Errors ({{instance}})", + "refId": "C" + }, + { + "expr": "sum(rate(node_network_receive_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) by (instance) OR sum(rate(windows_net_packets_received_discarded_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Receive Dropped ({{instance}})", + "refId": "D" + }, + { + "expr": "sum(rate(node_network_transmit_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) by (instance) OR sum(rate(windows_net_packets_outbound_discarded{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Transmit Dropped ({{instance}})", + "refId": "E" + }, + { + "expr": "sum(rate(node_network_transmit_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) by (instance) OR sum(rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Transmit Total ({{instance}})", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval]) OR rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Transmit Total ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(rate(node_network_receive_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval]) OR rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Receive Total ({{instance}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Cluster (Nodes)", + "uid": "rancher-cluster-nodes-1", + "version": 3 +} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/files/rancher/cluster/rancher-cluster.json b/charts/rancher-monitoring/102.0.5+up40.1.2/files/rancher/cluster/rancher-cluster.json new file mode 100644 index 0000000000..24385a237a --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/files/rancher/cluster/rancher-cluster.json @@ -0,0 +1,776 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\",mode=\"idle\"}[$__rate_interval]))", + "legendFormat": "Total", + "interval": "", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m]" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_load5 OR avg_over_time(windows_system_processor_queue_length[5m]))", + "interval": "", + "legendFormat": "Load[5m]", + "refId": "A" + }, + { + "expr": "sum(node_load1 OR avg_over_time(windows_system_processor_queue_length[1m]))", + "interval": "", + "legendFormat": "Load[1m]", + "refId": "B" + }, + { + "expr": "sum(node_load15 OR avg_over_time(windows_system_processor_queue_length[15m]))", + "interval": "", + "legendFormat": "Load[15m]", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Load Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - sum(node_memory_MemAvailable_bytes OR windows_os_physical_memory_free_bytes) / sum(node_memory_MemTotal_bytes OR windows_cs_physical_memory_bytes)", + "legendFormat": "Total", + "interval": "", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - (sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\"} OR windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) / sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"} OR windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}))", + "legendFormat": "Total", + "interval": "", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_disk_read_bytes_total[$__rate_interval]) OR rate(windows_logical_disk_read_bytes_total[$__rate_interval]))", + "interval": "", + "legendFormat": "Read", + "refId": "A" + }, + { + "expr": "sum(rate(node_disk_written_bytes_total[$__rate_interval]) OR rate(windows_logical_disk_write_bytes_total[$__rate_interval]))", + "interval": "", + "legendFormat": "Write", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(rate(node_network_receive_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_received_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Receive Errors", + "refId": "A" + }, + { + "expr": "(sum(rate(node_network_receive_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Receive Total", + "refId": "B" + }, + { + "expr": "(sum(rate(node_network_transmit_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_outbound_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Transmit Errors", + "refId": "C" + }, + { + "expr": "(sum(rate(node_network_receive_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_received_discarded_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Receive Dropped", + "refId": "D" + }, + { + "expr": "(sum(rate(node_network_transmit_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_outbound_discarded{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Transmit Dropped", + "refId": "E" + }, + { + "expr": "(sum(rate(node_network_transmit_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval]) OR rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval]))", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "A" + }, + { + "expr": "sum(rate(node_network_receive_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval]) OR rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval]))", + "interval": "", + "legendFormat": "Receive Total", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Cluster", + "uid": "rancher-cluster-1", + "version": 3 +} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/files/rancher/home/rancher-default-home.json b/charts/rancher-monitoring/102.0.5+up40.1.2/files/rancher/home/rancher-default-home.json new file mode 100644 index 0000000000..3fce207561 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/files/rancher/home/rancher-default-home.json @@ -0,0 +1,1290 @@ +{ + "annotations": { + "list": [] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "title": "", + "type": "welcome" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 4 + }, + "height": "180px", + "id": 6, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(1 - (avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\",mode=\"idle\"}[5m])))) * 100", + "format": "time_series", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "65, 90", + "title": "CPU Utilization", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 4 + }, + "height": "180px", + "id": 4, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(1 - sum({__name__=~\"node_memory_MemAvailable_bytes|windows_os_physical_memory_free_bytes\"}) / sum({__name__=~\"node_memory_MemTotal_bytes|windows_cs_physical_memory_bytes\"})) * 100", + "format": "time_series", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "65, 90", + "title": "Memory Utilization", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 4 + }, + "height": "180px", + "id": 7, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(1 - (((sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\"}) OR on() vector(0)) + (sum(windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) OR on() vector(0))) / ((sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"}) OR on() vector(0)) + (sum(windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) OR on() vector(0))))) * 100", + "format": "time_series", + "interval": "10s", + "intervalFactor": 1, + "metric": "", + "refId": "A", + "step": 10 + } + ], + "thresholds": "65, 90", + "title": "Disk Utilization", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 9 + }, + "height": "1px", + "id": 11, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": " cores", + "postfixFontSize": "30%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\",mode!=\"idle\"}[5m]))", + "format": "time_series", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "CPU Used", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 9 + }, + "height": "1px", + "id": 12, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": " cores", + "postfixFontSize": "30%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kube_node_status_allocatable_cpu_cores{}) OR sum(kube_node_status_allocatable{resource=\"cpu\",unit=\"core\"})", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "CPU Total", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 9 + }, + "height": "1px", + "id": 9, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "20%", + "prefix": "", + "prefixFontSize": "20%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum({__name__=~\"node_memory_MemTotal_bytes|windows_cs_physical_memory_bytes\"}) - sum({__name__=~\"node_memory_MemAvailable_bytes|windows_os_physical_memory_free_bytes\"})", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "Memory Used", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 12, + "y": 9 + }, + "height": "1px", + "id": 10, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kube_node_status_allocatable_memory_bytes{}) OR sum(kube_node_status_allocatable{resource=\"memory\", unit=\"byte\"})", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "Memory Total", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 16, + "y": 9 + }, + "height": "1px", + "id": 13, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"}) - sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\"}) OR on() vector(0)) + (sum(windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) - sum(windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) OR on() vector(0))", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "Disk Used", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 9 + }, + "height": "1px", + "id": 14, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"}) OR on() vector(0)) + (sum(windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) OR on() vector(0))", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "Disk Total", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 12 + }, + "hiddenSeries": false, + "id": 2051, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - (avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\",mode=\"idle\"}[$__rate_interval])))", + "format": "time_series", + "hide": false, + "instant": false, + "intervalFactor": 1, + "legendFormat": "Cluster", + "refId": "A" + }, + { + "expr": "1 - avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\", mode=\"idle\"}[$__rate_interval])) by (instance)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ instance }}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 12 + }, + "hiddenSeries": false, + "id": 2052, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "100 * (1 - sum({__name__=~\"node_memory_MemAvailable_bytes|windows_os_physical_memory_free_bytes\"}) / sum({__name__=~\"node_memory_MemTotal_bytes|windows_cs_physical_memory_bytes\"}))", + "format": "time_series", + "hide": false, + "instant": false, + "intervalFactor": 1, + "legendFormat": "Cluster", + "refId": "A" + }, + { + "expr": "100 * (1- sum({__name__=~\"node_memory_MemAvailable_bytes|windows_os_physical_memory_free_bytes\"}) by (instance) / sum({__name__=~\"node_memory_MemTotal_bytes|windows_cs_physical_memory_bytes\"}) by (instance))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ instance }}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 12 + }, + "hiddenSeries": false, + "id": 2053, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(1 - ((sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\"}) OR on() vector(0)) + (sum(windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"} OR on() vector(0)))) / ((sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"}) OR on() vector(0)) + (sum(windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) OR on() vector(0)))) * 100", + "legendFormat": "Cluster", + "refId": "A" + }, + { + "expr": "(1 - (sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\"}) by (instance)) / sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"}) by (instance)) * 100", + "hide": false, + "legendFormat": "{{ instance }}", + "refId": "B" + }, + { + "expr": "(1 - (sum(windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) by (instance)) / sum(windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) by (instance)) * 100", + "hide": false, + "legendFormat": "{{ instance }}", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "percent", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "folderId": 0, + "gridPos": { + "h": 15, + "w": 12, + "x": 0, + "y": 18 + }, + "headings": true, + "id": 3, + "limit": 30, + "links": [], + "query": "", + "recent": true, + "search": true, + "starred": false, + "tags": [], + "title": "Dashboards", + "type": "dashlist" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 2055, + "options": { + "content": "## About Rancher Monitoring\n\nRancher Monitoring is a Helm chart developed by Rancher that is powered by [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator). It is based on the upstream [kube-prometheus-stack](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack) Helm chart maintained by the Prometheus community.\n\nBy default, the chart deploys Grafana alongside a set of Grafana dashboards curated by the [kube-prometheus](https://github.com/prometheus-operator/kube-prometheus) project.\n\nFor more information on how Rancher Monitoring differs from [kube-prometheus-stack](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack), please view the CHANGELOG.md of the rancher-monitoring chart located in the [rancher/charts](https://github.com/rancher/charts) repository.\n\nFor more information about how to configure Rancher Monitoring, please view the [Rancher docs](https://rancher.com/docs/rancher/v2.x/en/).\n\n", + "mode": "markdown" + }, + "pluginVersion": "7.1.0", + "timeFrom": null, + "timeShift": null, + "title": "", + "type": "text" + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "hidden": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ], + "type": "timepicker" + }, + "timezone": "browser", + "title": "Home", + "uid": "rancher-home-1", + "version": 5 +} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/files/rancher/k8s/rancher-etcd-nodes.json b/charts/rancher-monitoring/102.0.5+up40.1.2/files/rancher/k8s/rancher-etcd-nodes.json new file mode 100644 index 0000000000..8c4bdcef53 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/files/rancher/k8s/rancher-etcd-nodes.json @@ -0,0 +1,687 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 32, + "links": [], + "panels": [ + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_network_client_grpc_received_bytes_total{job=\"kube-etcd\"}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Client Traffic In ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(rate(etcd_network_client_grpc_sent_bytes_total{job=\"kube-etcd\"}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Client Traffic Out ({{instance}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "GRPC Client Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m]({{instance}})" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(etcd_mvcc_db_total_size_in_bytes) by (instance)", + "interval": "", + "legendFormat": "DB Size ({{instance}})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "DB Size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(grpc_server_started_total{grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"}) by (instance) - sum(grpc_server_handled_total{grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"}) by (instance)", + "interval": "", + "legendFormat": "Watch Streams ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(grpc_server_started_total{grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"}) by (instance) - sum(grpc_server_handled_total{grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"}) by (instance)", + "interval": "", + "legendFormat": "Lease Watch Stream ({{instance}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Active Streams", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_server_proposals_committed_total[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Proposal Committed ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(rate(etcd_server_proposals_applied_total[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Proposal Applied ({{instance}})", + "refId": "B" + }, + { + "expr": "sum(rate(etcd_server_proposals_failed_total[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Proposal Failed ({{instance}})", + "refId": "C" + }, + { + "expr": "sum(etcd_server_proposals_pending) by (instance)", + "interval": "", + "legendFormat": "Proposal Pending ({{instance}})", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Raft Proposals", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_started_total{grpc_type=\"unary\"}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "RPC Rate ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(rate(grpc_server_handled_total{grpc_type=\"unary\",grpc_code!=\"OK\"}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "RPC Failure Rate ({{instance}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "RPC Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket[$__rate_interval])) by (instance, le))", + "interval": "", + "legendFormat": "WAL fsync ({{instance}})", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket[$__rate_interval])) by (instance, le))", + "interval": "", + "legendFormat": "DB fsync ({{instance}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Sync Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 2, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / etcd (Nodes)", + "uid": "rancher-etcd-nodes-1", + "version": 5 +} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/files/rancher/k8s/rancher-etcd.json b/charts/rancher-monitoring/102.0.5+up40.1.2/files/rancher/k8s/rancher-etcd.json new file mode 100644 index 0000000000..a305fe8adc --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/files/rancher/k8s/rancher-etcd.json @@ -0,0 +1,669 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 33, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_network_client_grpc_received_bytes_total{job=\"kube-etcd\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Client Traffic In", + "refId": "A" + }, + { + "expr": "sum(rate(etcd_network_client_grpc_sent_bytes_total{job=\"kube-etcd\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Client Traffic Out", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "GRPC Client Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(etcd_mvcc_db_total_size_in_bytes)", + "interval": "", + "legendFormat": "DB Size", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "DB Size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(grpc_server_started_total{grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"})", + "interval": "", + "legendFormat": "Watch Streams", + "refId": "A" + }, + { + "expr": "sum(grpc_server_started_total{grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"})", + "interval": "", + "legendFormat": "Lease Watch Stream", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Active Streams", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_server_proposals_committed_total[$__rate_interval]))", + "interval": "", + "legendFormat": "Proposal Committed", + "refId": "A" + }, + { + "expr": "sum(rate(etcd_server_proposals_applied_total[$__rate_interval]))", + "interval": "", + "legendFormat": "Proposal Applied", + "refId": "B" + }, + { + "expr": "sum(rate(etcd_server_proposals_failed_total[$__rate_interval]))", + "interval": "", + "legendFormat": "Proposal Failed", + "refId": "C" + }, + { + "expr": "sum(etcd_server_proposals_pending)", + "interval": "", + "legendFormat": "Proposal Pending", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Raft Proposals", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_started_total{grpc_type=\"unary\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "RPC Rate", + "refId": "A" + }, + { + "expr": "sum(rate(grpc_server_handled_total{grpc_type=\"unary\",grpc_code!=\"OK\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "RPC Failure Rate", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "RPC Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket[$__rate_interval])) by (instance, le))", + "interval": "", + "legendFormat": "WAL fsync", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket[$__rate_interval])) by (instance, le))", + "interval": "", + "legendFormat": "DB fsync", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Sync Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 2, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / etcd", + "uid": "rancher-etcd-1", + "version": 4 +} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/files/rancher/k8s/rancher-k8s-components-nodes.json b/charts/rancher-monitoring/102.0.5+up40.1.2/files/rancher/k8s/rancher-k8s-components-nodes.json new file mode 100644 index 0000000000..b31358eaaf --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/files/rancher/k8s/rancher-k8s-components-nodes.json @@ -0,0 +1,527 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 30, + "links": [], + "panels": [ + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(apiserver_request_total[$__rate_interval])) by (instance, code)", + "interval": "", + "legendFormat": "{{code}}({{instance}})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "API Server Request Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m]({{instance}})" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"deployment\"}) by (instance, name)", + "interval": "", + "legendFormat": "Deployment Depth ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"volumes\"}) by (instance, name)", + "interval": "", + "legendFormat": "Volumes Depth ({{instance}})", + "refId": "B" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"replicaset\"}) by (instance, name)", + "interval": "", + "legendFormat": "ReplicaSet Depth ({{instance}})", + "refId": "C" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"service\"}) by (instance, name)", + "interval": "", + "legendFormat": "Service Depth ({{instance}})", + "refId": "D" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"serviceaccount\"}) by (instance, name)", + "interval": "", + "legendFormat": "ServiceAccount Depth ({{instance}})", + "refId": "E" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"endpoint\"}) by (instance, name)", + "interval": "", + "legendFormat": "Endpoint Depth ({{instance}})", + "refId": "F" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"daemonset\"}) by (instance, name)", + "interval": "", + "legendFormat": "DaemonSet Depth ({{instance}})", + "refId": "G" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"statefulset\"}) by (instance, name)", + "interval": "", + "legendFormat": "StatefulSet Depth ({{instance}})", + "refId": "H" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"replicationmanager\"}) by (instance, name)", + "interval": "", + "legendFormat": "ReplicationManager Depth ({{instance}})", + "refId": "I" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Controller Manager Queue Depth", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_pod_status_scheduled{condition=\"false\"})", + "interval": "", + "legendFormat": "Failed To Schedule", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Pod Scheduling Status", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(nginx_ingress_controller_nginx_process_connections{state=\"reading\"}) by (instance)", + "interval": "", + "legendFormat": "Reading ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(nginx_ingress_controller_nginx_process_connections{state=\"waiting\"}) by (instance)", + "interval": "", + "legendFormat": "Waiting ({{instance}})", + "refId": "B" + }, + { + "expr": "sum(nginx_ingress_controller_nginx_process_connections{state=\"writing\"}) by (instance)", + "interval": "", + "legendFormat": "Writing ({{instance}})", + "refId": "C" + }, + { + "expr": "sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state=\"accepted\"}[$__rate_interval]))) by (instance)", + "interval": "", + "legendFormat": "Accepted ({{instance}})", + "refId": "D" + }, + { + "expr": "sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state=\"handled\"}[$__rate_interval]))) by (instance)", + "interval": "", + "legendFormat": "Handled ({{instance}})", + "refId": "E" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Ingress Controller Connections", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Kubernetes Components (Nodes)", + "uid": "rancher-k8s-components-nodes-1", + "version": 5 +} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/files/rancher/k8s/rancher-k8s-components.json b/charts/rancher-monitoring/102.0.5+up40.1.2/files/rancher/k8s/rancher-k8s-components.json new file mode 100644 index 0000000000..44cf97f9fd --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/files/rancher/k8s/rancher-k8s-components.json @@ -0,0 +1,519 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 31, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(apiserver_request_total[$__rate_interval])) by (code)", + "interval": "", + "legendFormat": "{{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "API Server Request Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m]({{instance}})" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"deployment\"}) by (name)", + "interval": "", + "legendFormat": "Deployment Depth", + "refId": "A" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"volumes\"}) by (name)", + "interval": "", + "legendFormat": "Volumes Depth", + "refId": "B" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"replicaset\"}) by (name)", + "interval": "", + "legendFormat": "Replicaset Depth", + "refId": "C" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"service\"}) by (name)", + "interval": "", + "legendFormat": "Service Depth", + "refId": "D" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"serviceaccount\"}) by (name)", + "interval": "", + "legendFormat": "ServiceAccount Depth", + "refId": "E" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"endpoint\"}) by (name)", + "interval": "", + "legendFormat": "Endpoint Depth", + "refId": "F" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"daemonset\"}) by (name)", + "interval": "", + "legendFormat": "DaemonSet Depth", + "refId": "G" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"statefulset\"}) by (name)", + "interval": "", + "legendFormat": "StatefulSet Depth", + "refId": "H" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"replicationmanager\"}) by (name)", + "interval": "", + "legendFormat": "ReplicationManager Depth", + "refId": "I" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Controller Manager Queue Depth", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_pod_status_scheduled{condition=\"false\"})", + "interval": "", + "legendFormat": "Failed To Schedule", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Pod Scheduling Status", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(nginx_ingress_controller_nginx_process_connections{state=\"reading\"})", + "interval": "", + "legendFormat": "Reading", + "refId": "A" + }, + { + "expr": "sum(nginx_ingress_controller_nginx_process_connections{state=\"waiting\"})", + "interval": "", + "legendFormat": "Waiting", + "refId": "B" + }, + { + "expr": "sum(nginx_ingress_controller_nginx_process_connections{state=\"writing\"})", + "interval": "", + "legendFormat": "Writing", + "refId": "C" + }, + { + "expr": "sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state=\"accepted\"}[$__rate_interval])))", + "interval": "", + "legendFormat": "Accepted", + "refId": "D" + }, + { + "expr": "sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state=\"handled\"}[$__rate_interval])))", + "interval": "", + "legendFormat": "Handled", + "refId": "E" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Ingress Controller Connections", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Kubernetes Components", + "uid": "rancher-k8s-components-1", + "version": 5 +} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/files/rancher/nodes/rancher-node-detail.json b/charts/rancher-monitoring/102.0.5+up40.1.2/files/rancher/nodes/rancher-node-detail.json new file mode 100644 index 0000000000..920fb94cf7 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/files/rancher/nodes/rancher-node-detail.json @@ -0,0 +1,805 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "links": [], + "panels": [ + { + "aliasColors": { + "{{mode}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\", instance=\"$instance\"}[$__rate_interval])) by (mode)", + "interval": "", + "legendFormat": "{{mode}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m]" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_load5{instance=~\"$instance\"} OR avg_over_time(windows_system_processor_queue_length{instance=~\"$instance\"}[5m]))", + "interval": "", + "legendFormat": "Load[5m]", + "refId": "A" + }, + { + "expr": "sum(node_load1{instance=~\"$instance\"} OR avg_over_time(windows_system_processor_queue_length{instance=~\"$instance\"}[1m]))", + "interval": "", + "legendFormat": "Load[1m]", + "refId": "B" + }, + { + "expr": "sum(node_load15{instance=~\"$instance\"} OR avg_over_time(windows_system_processor_queue_length{instance=~\"$instance\"}[15m]))", + "interval": "", + "legendFormat": "Load[15m]", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Load Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - (node_memory_MemAvailable_bytes{instance=~\"$instance\"} OR windows_os_physical_memory_free_bytes{instance=~\"$instance\"}) / (node_memory_MemTotal_bytes{instance=~\"$instance\"} OR windows_cs_physical_memory_bytes{instance=~\"$instance\"})", + "interval": "", + "legendFormat": "Total", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{device}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - (sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\", instance=~\"$instance\"} OR windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\", instance=~\"$instance\"}) by (device) / sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\", instance=~\"$instance\"} OR windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\", instance=~\"$instance\"}) by (device))", + "interval": "", + "legendFormat": "{{device}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{device}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_disk_read_bytes_total{instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_logical_disk_read_bytes_total{instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Read ({{device}})", + "refId": "A" + }, + { + "expr": "sum(rate(node_disk_written_bytes_total{instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_logical_disk_write_bytes_total{instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Write ({{device}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{device}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_network_receive_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) by (device) OR sum(rate(windows_net_packets_received_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Receive Errors ({{device}})", + "refId": "A" + }, + { + "expr": "sum(rate(node_network_receive_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) by (device) OR sum(rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Receive Total ({{device}})", + "refId": "B" + }, + { + "expr": "sum(rate(node_network_transmit_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) by (device) OR sum(rate(windows_net_packets_outbound_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Transmit Errors ({{device}})", + "refId": "C" + }, + { + "expr": "sum(rate(node_network_receive_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) by (device) OR sum(rate(windows_net_packets_received_discarded_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Receive Dropped ({{device}})", + "refId": "D" + }, + { + "expr": "sum(rate(node_network_transmit_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) by (device) OR sum(rate(windows_net_packets_outbound_discarded{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Transmit Dropped ({{device}})", + "refId": "E" + }, + { + "expr": "sum(rate(node_network_transmit_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) by (device) OR sum(rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Transmit Total ({{device}})", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{device}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Transmit Total ({{device}})", + "refId": "A" + }, + { + "expr": "sum(rate(node_network_receive_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Receive Total ({{device}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "instance", + "query": "label_values({__name__=~\"node_exporter_build_info|windows_exporter_build_info\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Node (Detail)", + "uid": "rancher-node-detail-1", + "version": 3 +} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/files/rancher/nodes/rancher-node.json b/charts/rancher-monitoring/102.0.5+up40.1.2/files/rancher/nodes/rancher-node.json new file mode 100644 index 0000000000..367df3cc9d --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/files/rancher/nodes/rancher-node.json @@ -0,0 +1,792 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\", instance=\"$instance\", mode=\"idle\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Total", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m]" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_load5{instance=~\"$instance\"} OR avg_over_time(windows_system_processor_queue_length{instance=~\"$instance\"}[5m]))", + "interval": "", + "legendFormat": "Load[5m]", + "refId": "A" + }, + { + "expr": "sum(node_load1{instance=~\"$instance\"} OR avg_over_time(windows_system_processor_queue_length{instance=~\"$instance\"}[1m]))", + "interval": "", + "legendFormat": "Load[1m]", + "refId": "B" + }, + { + "expr": "sum(node_load15{instance=~\"$instance\"} OR avg_over_time(windows_system_processor_queue_length{instance=~\"$instance\"}[15m]))", + "interval": "", + "legendFormat": "Load[15m]", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Load Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - sum(node_memory_MemAvailable_bytes{instance=~\"$instance\"} OR windows_os_physical_memory_free_bytes{instance=~\"$instance\"}) / sum(node_memory_MemTotal_bytes{instance=~\"$instance\"} OR windows_cs_physical_memory_bytes{instance=~\"$instance\"})", + "interval": "", + "legendFormat": "Total", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - (sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\", instance=~\"$instance\"} OR windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\", instance=~\"$instance\"}) / sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\", instance=~\"$instance\"} OR windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\", instance=~\"$instance\"}))", + "interval": "", + "legendFormat": "Total", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_disk_read_bytes_total{instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_logical_disk_read_bytes_total{instance=~\"$instance\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Read", + "refId": "A" + }, + { + "expr": "sum(rate(node_disk_written_bytes_total{instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_logical_disk_write_bytes_total{instance=~\"$instance\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Write", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(rate(node_network_receive_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_received_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Receive Errors", + "refId": "A" + }, + { + "expr": "(sum(rate(node_network_receive_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Receive Total", + "refId": "B" + }, + { + "expr": "(sum(rate(node_network_transmit_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_outbound_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Transmit Errors", + "refId": "C" + }, + { + "expr": "(sum(rate(node_network_receive_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_received_discarded_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Receive Dropped", + "refId": "D" + }, + { + "expr": "(sum(rate(node_network_transmit_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_outbound_discarded{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Transmit Dropped", + "refId": "E" + }, + { + "expr": "(sum(rate(node_network_transmit_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "A" + }, + { + "expr": "sum(rate(node_network_receive_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Receive Total", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "instance", + "query": "label_values({__name__=~\"node_exporter_build_info|windows_exporter_build_info\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Node", + "uid": "rancher-node-1", + "version": 3 +} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/files/rancher/performance/performance-debugging.json b/charts/rancher-monitoring/102.0.5+up40.1.2/files/rancher/performance/performance-debugging.json new file mode 100644 index 0000000000..b4ac76dfce --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/files/rancher/performance/performance-debugging.json @@ -0,0 +1,1707 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 3, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 22, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.11", + "pointradius": 2, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "topk(20,sum by (handler_name) (rate(lasso_controller_reconcile_time_seconds_sum[5m]))\n/\nsum by (handler_name) (rate(lasso_controller_reconcile_time_seconds_count[5m])))", + "interval": "", + "legendFormat": "{{handler_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Handler Average Execution Times Over Last 5 Minutes (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1390", + "format": "short", + "label": "Execution Time in Seconds", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:1391", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 8 + }, + "hiddenSeries": false, + "id": 28, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.11", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "topk(20,sum by (resource, method, code) (rate(steve_api_request_time_sum{resource!=\"subscribe\"}[5m]))\n/\nsum by (resource, method, code) (rate(steve_api_request_time_count{resource!=\"subscribe\"}[5m])))", + "interval": "", + "legendFormat": "{{resource}} {{method}} {{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Rancher API Average Request Times Over Last 5 Minutes (Top 20) (Subscribes Omitted)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:178", + "format": "ms", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:179", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 16 + }, + "hiddenSeries": false, + "id": 30, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.11", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "rate(steve_api_request_time_sum{resource=\"subscribe\"}[5m])\n/\nrate(steve_api_request_time_count{resource=\"subscribe\"}[5m])", + "interval": "", + "legendFormat": "{{resource}} {{method}} {{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Subscribe Average Request Times Over Last 5 Minutes", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:368", + "format": "ms", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:369", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 24 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.11", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "topk(20,workqueue_depth)", + "interval": "", + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Lasso Controller Work Queue Depth (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1553", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:1554", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 13, + "w": 16, + "x": 0, + "y": 32 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.11", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "topk(20,sum by (id, resource, method, code) (steve_api_total_requests))", + "instant": false, + "interval": "", + "legendFormat": "{{id}} {{resource}} {{method}} {{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Number of Rancher Requests (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:290", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:291", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 16, + "x": 0, + "y": 45 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.11", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "topk(20,sum by (id, resource, method) (steve_api_total_requests{code!=\"200\",code!=\"201\"}))", + "interval": "", + "legendFormat": "{{id}} {{resource}} {{method}} {{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Number of Failed Rancher API Requests (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:428", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:429", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 54 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.11", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "topk(20,sum by (resource, method, code) (rate(k8s_proxy_store_request_time_sum[5m]))\n/\nsum by (resource, method, code) (rate(k8s_proxy_store_request_time_count[5m])))", + "interval": "", + "legendFormat": "{{resource}} {{method}} {{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "K8s Proxy Store Average Request Times Over Last 5 Minutes (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:662", + "format": "ms", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:663", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 62 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.11", + "pointradius": 2, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "topk(20,sum by (resource, method, code) (rate(k8s_proxy_client_request_time_sum[5m]))\n/\nsum by (resource, method, code) (rate(k8s_proxy_client_request_time_count[5m])))", + "interval": "", + "legendFormat": "{{resource}} {{method}} {{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "K8s Proxy Client Average Request Times Over Last 5 Minutes (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1710", + "format": "ms", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:1711", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 70 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.11", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "topk(20,lasso_controller_total_cached_object)", + "interval": "", + "legendFormat": "{{kind}} {{version}} {{group}} {{pod}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cached Objects by GroupVersionKind (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:744", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:745", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 78 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.11", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "topk(20,sum by (handler_name) (\nlasso_controller_total_handler_execution\n))", + "interval": "", + "legendFormat": "{{handler_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Lasso Handler Executions (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:824", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:825", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 86 + }, + "hiddenSeries": false, + "id": 32, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.11", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "topk(20, sum by (handler_name,controller_name) (\nincrease(lasso_controller_total_handler_execution[2m])\n))", + "interval": "", + "legendFormat": "{{controller_name}}.{{handler_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Handler Executions Over Last 2 Minutes (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 94 + }, + "hiddenSeries": false, + "id": 20, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.11", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "topk(20,sum by (handler_name) (\nlasso_controller_total_handler_execution{has_error=\"true\"}\n))", + "interval": "", + "legendFormat": "{{handler_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Total Handler Executions with Error (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1230", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:1231", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 102 + }, + "hiddenSeries": false, + "id": 34, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.11", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "topk(20,sum by (handler_name,controller_name) (\nincrease(lasso_controller_total_handler_execution{has_error=\"true\"}[2m])\n))", + "interval": "", + "legendFormat": "{{controller_name}}.{{handler_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Handler Executions Over Last 2 Minutes (Top 20)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 110 + }, + "hiddenSeries": false, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.11", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "topk(20,session_server_total_transmit_bytes)", + "interval": "", + "legendFormat": "{{clientkey}} {{pod}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Data Transmitted by Remote Dialer Sessions (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1953", + "format": "decbytes", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:1954", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 118 + }, + "hiddenSeries": false, + "id": 18, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.11", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "session_server_total_transmit_error_bytes", + "interval": "", + "legendFormat": "{{clientkey}} {{pod}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Errors for Remote Dialer Sessions (Top 20)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:2045", + "format": "ms", + "label": "Error Data", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:2046", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 126 + }, + "hiddenSeries": false, + "id": 26, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.11", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "session_server_total_remove_connections", + "interval": "", + "legendFormat": "{{clientkey}} {{pod}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Remote Dialer Connections Removed (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:2199", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:2200", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 134 + }, + "hiddenSeries": false, + "id": 24, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.11", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "session_server_total_add_connections", + "interval": "", + "legendFormat": "{{clientkey}} {{pod}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Remote Dialer Connections Added by Client (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:2117", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:2118", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 27, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Rancher Performance Debugging", + "uid": "tfrfU0a7k", + "version": 1 +} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/files/rancher/pods/rancher-pod-containers.json b/charts/rancher-monitoring/102.0.5+up40.1.2/files/rancher/pods/rancher-pod-containers.json new file mode 100644 index 0000000000..9e53081a73 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/files/rancher/pods/rancher-pod-containers.json @@ -0,0 +1,636 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "iteration": 1618265214337, + "links": [], + "panels": [ + { + "aliasColors": { + "{{container}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_cpu_cfs_throttled_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "CFS throttled ({{container}})", + "refId": "A" + }, + { + "expr": "sum(rate(container_cpu_system_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container) OR sum(rate(windows_container_cpu_usage_seconds_kernelmode{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "System ({{container}})", + "refId": "B" + }, + { + "expr": "sum(rate(container_cpu_usage_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container) OR sum(rate(windows_container_cpu_usage_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Total ({{container}})", + "refId": "C" + }, + { + "expr": "sum(rate(container_cpu_user_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container) OR sum(rate(windows_container_cpu_usage_seconds_usermode{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "User ({{container}})", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "cpu", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{container}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"\"} OR windows_container_memory_usage_commit_bytes{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"\"}) by (container)", + "interval": "", + "legendFormat": "({{container}})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{container}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container) OR sum(irate(windows_container_network_receive_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Receive Total ({{container}})", + "refId": "A" + }, + { + "expr": "sum(irate(container_network_transmit_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container) OR sum(irate(windows_container_network_transmit_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Transmit Total ({{container}})", + "refId": "B" + }, + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container) OR sum(irate(windows_container_network_receive_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Receive Dropped ({{container}})", + "refId": "C" + }, + { + "expr": "sum(irate(container_network_receive_errors_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Receive Errors ({{container}})", + "refId": "D" + }, + { + "expr": "sum(irate(container_network_transmit_errors_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Transmit Errors ({{container}})", + "refId": "E" + }, + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container) OR sum(irate(windows_container_network_transmit_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Transmit Dropped ({{container}})", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{container}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container) OR sum(irate(windows_container_network_receive_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Receive Total ({{container}})", + "refId": "A" + }, + { + "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container) OR sum(irate(windows_container_network_transmit_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Transmit Total ({{container}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{container}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_fs_writes_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Write ({{container}})", + "refId": "A" + }, + { + "expr": "sum(rate(container_fs_reads_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Read ({{container}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "pod", + "query": "label_values(kube_pod_info{cluster=\"$cluster\", namespace=\"$namespace\"}, pod)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Pod (Containers)", + "uid": "rancher-pod-containers-1", + "version": 8 +} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/files/rancher/pods/rancher-pod.json b/charts/rancher-monitoring/102.0.5+up40.1.2/files/rancher/pods/rancher-pod.json new file mode 100644 index 0000000000..65c6bf18ed --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/files/rancher/pods/rancher-pod.json @@ -0,0 +1,636 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "iteration": 1618265214337, + "links": [], + "panels": [ + { + "aliasColors": { + "": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_cpu_cfs_throttled_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "CFS throttled", + "refId": "A" + }, + { + "expr": "sum(rate(container_cpu_system_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) OR sum(rate(windows_container_cpu_usage_seconds_kernelmode{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "System", + "refId": "B" + }, + { + "expr": "sum(rate(container_cpu_usage_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) OR sum(rate(windows_container_cpu_usage_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Total", + "refId": "C" + }, + { + "expr": "sum(rate(container_cpu_user_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) OR sum(rate(windows_container_cpu_usage_seconds_usermode{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "User", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "cpu", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"\"} OR windows_container_memory_usage_commit_bytes{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"\"})", + "interval": "", + "legendFormat": "Total", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) OR sum(irate(windows_container_network_receive_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Receive Total", + "refId": "A" + }, + { + "expr": "sum(irate(container_network_transmit_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) OR sum(irate(windows_container_network_transmit_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "B" + }, + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) OR sum(irate(windows_container_network_receive_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Receive Dropped", + "refId": "C" + }, + { + "expr": "sum(irate(container_network_receive_errors_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Receive Errors", + "refId": "D" + }, + { + "expr": "sum(irate(container_network_transmit_errors_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Transmit Errors", + "refId": "E" + }, + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) OR sum(irate(windows_container_network_transmit_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Transmit Dropped", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) OR sum(irate(windows_container_network_receive_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Receive Total", + "refId": "A" + }, + { + "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) OR sum(irate(windows_container_network_transmit_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_fs_writes_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Write", + "refId": "A" + }, + { + "expr": "sum(rate(container_fs_reads_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Read", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "query": "label_values({__name__=~\"container_.*|windows_container_.*\", namespace!=\"\"}, namespace)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "pod", + "query": "label_values({__name__=~\"container_.*|windows_container_.*\", namespace=\"$namespace\", pod!=\"\"}, pod)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Pod", + "uid": "rancher-pod-1", + "version": 8 +} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/files/rancher/workloads/rancher-workload-pods.json b/charts/rancher-monitoring/102.0.5+up40.1.2/files/rancher/workloads/rancher-workload-pods.json new file mode 100644 index 0000000000..f6b5078afb --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/files/rancher/workloads/rancher-workload-pods.json @@ -0,0 +1,652 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "iteration": 1618265214337, + "links": [], + "panels": [ + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(rate(container_cpu_cfs_throttled_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "CFS throttled ({{pod}})", + "refId": "A" + }, + { + "expr": "(sum(rate(container_cpu_system_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(rate(windows_container_cpu_usage_seconds_kernelmode{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "System ({{pod}})", + "refId": "B" + }, + { + "expr": "(sum(rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(rate(windows_container_cpu_usage_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Total ({{pod}})", + "refId": "C" + }, + { + "expr": "(sum(rate(container_cpu_user_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(rate(windows_container_cpu_usage_seconds_usermode{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "User ({{pod}})", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "cpu", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(container_memory_working_set_bytes{namespace=~\"$namespace\"} OR windows_container_memory_usage_commit_bytes{namespace=~\"$namespace\"}) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "({{pod}})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_receive_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Receive Total ({{pod}})", + "refId": "A" + }, + { + "expr": "(sum(irate(container_network_transmit_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_transmit_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Transmit Total ({{pod}})", + "refId": "B" + }, + { + "expr": "(sum(irate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_receive_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Receive Dropped ({{pod}})", + "refId": "C" + }, + { + "expr": "(sum(irate(container_network_receive_errors_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Receive Errors ({{pod}})", + "refId": "D" + }, + { + "expr": "(sum(irate(container_network_transmit_errors_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Transmit Errors ({{pod}})", + "refId": "E" + }, + { + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_transmit_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Transmit Dropped ({{pod}})", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_receive_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Receive Total ({{pod}})", + "refId": "A" + }, + { + "expr": "(sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Transmit Total ({{pod}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(rate(container_fs_writes_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Write ({{pod}})", + "refId": "A" + }, + { + "expr": "(sum(rate(container_fs_reads_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Read ({{pod}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "query": "query_result(kube_pod_info{namespace!=\"\"} * on(pod) group_right(namespace, created_by_kind, created_by_name) count({__name__=~\"container_.*|windows_container_.*\", pod!=\"\"}) by (pod))", + "refresh": 2, + "regex": "/.*namespace=\"([^\"]*)\"/", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "kind", + "query": "query_result(kube_pod_info{namespace=\"$namespace\", created_by_kind!=\"\"} * on(pod) group_right(namespace, created_by_kind, created_by_name) count({__name__=~\"container_.*|windows_container_.*\", pod!=\"\"}) by (pod))", + "refresh": 2, + "regex": "/.*created_by_kind=\"([^\"]*)\"/", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "workload", + "query": "query_result(kube_pod_info{namespace=\"$namespace\", created_by_kind=\"$kind\", created_by_name!=\"\"} * on(pod) group_right(namespace, created_by_kind, created_by_name) count({__name__=~\"container_.*|windows_container_.*\", pod!=\"\"}) by (pod))", + "refresh": 2, + "regex": "/.*created_by_name=\"([^\"]*)\"/", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Workload (Pods)", + "uid": "rancher-workload-pods-1", + "version": 8 +} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/files/rancher/workloads/rancher-workload.json b/charts/rancher-monitoring/102.0.5+up40.1.2/files/rancher/workloads/rancher-workload.json new file mode 100644 index 0000000000..9f5317c2f0 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/files/rancher/workloads/rancher-workload.json @@ -0,0 +1,652 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "iteration": 1618265214337, + "links": [], + "panels": [ + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum((sum(rate(container_cpu_cfs_throttled_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "CFS throttled", + "refId": "A" + }, + { + "expr": "sum((sum(rate(container_cpu_system_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(rate(windows_container_cpu_usage_seconds_kernelmode{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "System", + "refId": "B" + }, + { + "expr": "sum((sum(rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(rate(windows_container_cpu_usage_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Total", + "refId": "C" + }, + { + "expr": "sum((sum(rate(container_cpu_user_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(rate(windows_container_cpu_usage_seconds_usermode{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "User", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "cpu", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum((sum(container_memory_working_set_bytes{namespace=~\"$namespace\"} OR windows_container_memory_usage_commit_bytes{namespace=~\"$namespace\"}) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Total", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum((sum(irate(container_network_receive_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_receive_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Receive Total", + "refId": "A" + }, + { + "expr": "sum((sum(irate(container_network_transmit_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_transmit_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "B" + }, + { + "expr": "sum((sum(irate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_receive_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Receive Dropped", + "refId": "C" + }, + { + "expr": "sum((sum(irate(container_network_receive_errors_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Receive Errors", + "refId": "D" + }, + { + "expr": "sum((sum(irate(container_network_transmit_errors_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Transmit Errors", + "refId": "E" + }, + { + "expr": "sum((sum(irate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_transmit_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Transmit Dropped", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum((sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_receive_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Receive Total", + "refId": "A" + }, + { + "expr": "sum((sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum((sum(rate(container_fs_writes_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Write", + "refId": "A" + }, + { + "expr": "sum((sum(rate(container_fs_reads_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Read", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "query": "query_result(kube_pod_info{namespace!=\"\"} * on(pod) group_right(namespace, created_by_kind, created_by_name) count({__name__=~\"container_.*|windows_container_.*\", pod!=\"\"}) by (pod))", + "refresh": 2, + "regex": "/.*namespace=\"([^\"]*)\"/", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "kind", + "query": "query_result(kube_pod_info{namespace=\"$namespace\", created_by_kind!=\"\"} * on(pod) group_right(namespace, created_by_kind, created_by_name) count({__name__=~\"container_.*|windows_container_.*\", pod!=\"\"}) by (pod))", + "refresh": 2, + "regex": "/.*created_by_kind=\"([^\"]*)\"/", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "workload", + "query": "query_result(kube_pod_info{namespace=\"$namespace\", created_by_kind=\"$kind\", created_by_name!=\"\"} * on(pod) group_right(namespace, created_by_kind, created_by_name) count({__name__=~\"container_.*|windows_container_.*\", pod!=\"\"}) by (pod))", + "refresh": 2, + "regex": "/.*created_by_name=\"([^\"]*)\"/", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Workload", + "uid": "rancher-workload-1", + "version": 8 +} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/files/upgrade/scripts/delete-workloads-with-old-labels.sh b/charts/rancher-monitoring/102.0.5+up40.1.2/files/upgrade/scripts/delete-workloads-with-old-labels.sh new file mode 100644 index 0000000000..89431e7132 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/files/upgrade/scripts/delete-workloads-with-old-labels.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -e +set -x + +# node-exporter +kubectl delete daemonset -l app=prometheus-node-exporter,release=rancher-monitoring --ignore-not-found=true + +# prometheus-adapter +kubectl delete deployments -l app=prometheus-adapter,release=rancher-monitoring --ignore-not-found=true + +# kube-state-metrics +kubectl delete deployments -l app.kubernetes.io/instance=rancher-monitoring,app.kubernetes.io/name=kube-state-metrics --cascade=orphan --ignore-not-found=true +kubectl delete statefulsets -l app.kubernetes.io/instance=rancher-monitoring,app.kubernetes.io/name=kube-state-metrics --cascade=orphan --ignore-not-found=true diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/NOTES.txt b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/NOTES.txt new file mode 100644 index 0000000000..371f3ae398 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/NOTES.txt @@ -0,0 +1,4 @@ +{{ $.Chart.Name }} has been installed. Check its status by running: + kubectl --namespace {{ template "kube-prometheus-stack.namespace" . }} get pods -l "release={{ $.Release.Name }}" + +Visit https://github.com/prometheus-operator/kube-prometheus for instructions on how to create & configure Alertmanager and Prometheus instances using the Operator. diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/_helpers.tpl b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/_helpers.tpl new file mode 100644 index 0000000000..7dd8f73aed --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/_helpers.tpl @@ -0,0 +1,384 @@ +# Rancher +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +{{/* +https://github.com/helm/helm/issues/4535#issuecomment-477778391 +Usage: {{ include "call-nested" (list . "SUBCHART_NAME" "TEMPLATE") }} +e.g. {{ include "call-nested" (list . "grafana" "grafana.fullname") }} +*/}} +{{- define "call-nested" }} +{{- $dot := index . 0 }} +{{- $subchart := index . 1 | splitList "." }} +{{- $template := index . 2 }} +{{- $values := $dot.Values }} +{{- range $subchart }} +{{- $values = index $values . }} +{{- end }} +{{- include $template (dict "Chart" (dict "Name" (last $subchart)) "Values" $values "Release" $dot.Release "Capabilities" $dot.Capabilities) }} +{{- end }} + +# Special Exporters +{{- define "exporter.kubeEtcd.enabled" -}} +{{- if or .Values.kubeEtcd.enabled .Values.rkeEtcd.enabled .Values.kubeAdmEtcd.enabled .Values.rke2Etcd.enabled -}} +"true" +{{- end -}} +{{- end }} + +{{- define "exporter.kubeControllerManager.enabled" -}} +{{- if or .Values.kubeControllerManager.enabled .Values.rkeControllerManager.enabled .Values.k3sServer.enabled .Values.kubeAdmControllerManager.enabled .Values.rke2ControllerManager.enabled -}} +"true" +{{- end -}} +{{- end }} + +{{- define "exporter.kubeScheduler.enabled" -}} +{{- if or .Values.kubeScheduler.enabled .Values.rkeScheduler.enabled .Values.k3sServer.enabled .Values.kubeAdmScheduler.enabled .Values.rke2Scheduler.enabled -}} +"true" +{{- end -}} +{{- end }} + +{{- define "exporter.kubeProxy.enabled" -}} +{{- if or .Values.kubeProxy.enabled .Values.rkeProxy.enabled .Values.k3sServer.enabled .Values.kubeAdmProxy.enabled .Values.rke2Proxy.enabled -}} +"true" +{{- end -}} +{{- end }} + +{{- define "exporter.kubelet.enabled" -}} +{{- if or .Values.kubelet.enabled .Values.hardenedKubelet.enabled .Values.k3sServer.enabled -}} +"true" +{{- end -}} +{{- end }} + +{{- define "exporter.kubeControllerManager.jobName" -}} +{{- if .Values.k3sServer.enabled -}} +k3s-server +{{- else -}} +kube-controller-manager +{{- end -}} +{{- end }} + +{{- define "exporter.kubeScheduler.jobName" -}} +{{- if .Values.k3sServer.enabled -}} +k3s-server +{{- else -}} +kube-scheduler +{{- end -}} +{{- end }} + +{{- define "exporter.kubeProxy.jobName" -}} +{{- if .Values.k3sServer.enabled -}} +k3s-server +{{- else -}} +kube-proxy +{{- end -}} +{{- end }} + +{{- define "exporter.kubelet.jobName" -}} +{{- if .Values.k3sServer.enabled -}} +k3s-server +{{- else -}} +kubelet +{{- end -}} +{{- end }} + +{{- define "kubelet.serviceMonitor.resourcePath" -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if not (eq .Values.kubelet.serviceMonitor.resourcePath "/metrics/resource/v1alpha1") -}} +{{ .Values.kubelet.serviceMonitor.resourcePath }} +{{- else if semverCompare ">=1.20.0-0" $kubeTargetVersion -}} +/metrics/resource +{{- else -}} +/metrics/resource/v1alpha1 +{{- end -}} +{{- end }} + +{{- define "rancher.serviceMonitor.selector" -}} +{{- if .Values.rancherMonitoring.selector }} +{{ .Values.rancherMonitoring.selector | toYaml }} +{{- else }} +{{- $rancherDeployment := (lookup "apps/v1" "Deployment" "cattle-system" "rancher") }} +{{- if $rancherDeployment }} +matchLabels: + app: rancher + chart: {{ index $rancherDeployment.metadata.labels "chart" }} + release: rancher +{{- end }} +{{- end }} +{{- end }} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# Prometheus Operator + +{{/* vim: set filetype=mustache: */}} +{{/* Expand the name of the chart. This is suffixed with -alertmanager, which means subtract 13 from longest 63 available */}} +{{- define "kube-prometheus-stack.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 50 | trimSuffix "-" -}} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +The components in this chart create additional resources that expand the longest created name strings. +The longest name that gets created adds and extra 37 characters, so truncation should be 63-35=26. +*/}} +{{- define "kube-prometheus-stack.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 26 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 26 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 26 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* Fullname suffixed with operator */}} +{{- define "kube-prometheus-stack.operator.fullname" -}} +{{- printf "%s-operator" (include "kube-prometheus-stack.fullname" .) -}} +{{- end }} + +{{/* Prometheus custom resource instance name */}} +{{- define "kube-prometheus-stack.prometheus.crname" -}} +{{- if .Values.cleanPrometheusOperatorObjectNames }} +{{- include "kube-prometheus-stack.fullname" . }} +{{- else }} +{{- print (include "kube-prometheus-stack.fullname" .) "-prometheus" }} +{{- end }} +{{- end }} + +{{/* Alertmanager custom resource instance name */}} +{{- define "kube-prometheus-stack.alertmanager.crname" -}} +{{- if .Values.cleanPrometheusOperatorObjectNames }} +{{- include "kube-prometheus-stack.fullname" . }} +{{- else }} +{{- print (include "kube-prometheus-stack.fullname" .) "-alertmanager" -}} +{{- end }} +{{- end }} + +{{/* Fullname suffixed with thanos-ruler */}} +{{- define "kube-prometheus-stack.thanosRuler.fullname" -}} +{{- printf "%s-thanos-ruler" (include "kube-prometheus-stack.fullname" .) -}} +{{- end }} + +{{/* Create chart name and version as used by the chart label. */}} +{{- define "kube-prometheus-stack.chartref" -}} +{{- replace "+" "_" .Chart.Version | printf "%s-%s" .Chart.Name -}} +{{- end }} + +{{/* Generate basic labels */}} +{{- define "kube-prometheus-stack.labels" }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/version: "{{ replace "+" "_" .Chart.Version }}" +app.kubernetes.io/part-of: {{ template "kube-prometheus-stack.name" . }} +chart: {{ template "kube-prometheus-stack.chartref" . }} +release: {{ $.Release.Name | quote }} +heritage: {{ $.Release.Service | quote }} +{{- if .Values.commonLabels}} +{{ toYaml .Values.commonLabels }} +{{- end }} +{{- end }} + +{{/* Create the name of kube-prometheus-stack service account to use */}} +{{- define "kube-prometheus-stack.operator.serviceAccountName" -}} +{{- if .Values.prometheusOperator.serviceAccount.create -}} + {{ default (include "kube-prometheus-stack.operator.fullname" .) .Values.prometheusOperator.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.prometheusOperator.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* Create the name of prometheus service account to use */}} +{{- define "kube-prometheus-stack.prometheus.serviceAccountName" -}} +{{- if .Values.prometheus.serviceAccount.create -}} + {{ default (print (include "kube-prometheus-stack.fullname" .) "-prometheus") .Values.prometheus.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.prometheus.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* Create the name of alertmanager service account to use */}} +{{- define "kube-prometheus-stack.alertmanager.serviceAccountName" -}} +{{- if .Values.alertmanager.serviceAccount.create -}} + {{ default (print (include "kube-prometheus-stack.fullname" .) "-alertmanager") .Values.alertmanager.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.alertmanager.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* Create the name of thanosRuler service account to use */}} +{{- define "kube-prometheus-stack.thanosRuler.serviceAccountName" -}} +{{- if .Values.thanosRuler.serviceAccount.create -}} + {{ default (include "kube-prometheus-stack.thanosRuler.fullname" .) .Values.thanosRuler.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.thanosRuler.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "kube-prometheus-stack.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* +Use the grafana namespace override for multi-namespace deployments in combined charts +*/}} +{{- define "kube-prometheus-stack-grafana.namespace" -}} + {{- if .Values.grafana.namespaceOverride -}} + {{- .Values.grafana.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* +Use the kube-state-metrics namespace override for multi-namespace deployments in combined charts +*/}} +{{- define "kube-prometheus-stack-kube-state-metrics.namespace" -}} + {{- if index .Values "kube-state-metrics" "namespaceOverride" -}} + {{- index .Values "kube-state-metrics" "namespaceOverride" -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* +Use the prometheus-node-exporter namespace override for multi-namespace deployments in combined charts +*/}} +{{- define "kube-prometheus-stack-prometheus-node-exporter.namespace" -}} + {{- if index .Values "prometheus-node-exporter" "namespaceOverride" -}} + {{- index .Values "prometheus-node-exporter" "namespaceOverride" -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* Allow KubeVersion to be overridden. */}} +{{- define "kube-prometheus-stack.kubeVersion" -}} + {{- default .Capabilities.KubeVersion.Version .Values.kubeVersionOverride -}} +{{- end -}} + +{{/* Get Ingress API Version */}} +{{- define "kube-prometheus-stack.ingress.apiVersion" -}} + {{- if and (.Capabilities.APIVersions.Has "networking.k8s.io/v1") (semverCompare ">= 1.19-0" (include "kube-prometheus-stack.kubeVersion" .)) -}} + {{- print "networking.k8s.io/v1" -}} + {{- else if .Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" -}} + {{- print "networking.k8s.io/v1beta1" -}} + {{- else -}} + {{- print "extensions/v1beta1" -}} + {{- end -}} +{{- end -}} + +{{/* Check Ingress stability */}} +{{- define "kube-prometheus-stack.ingress.isStable" -}} + {{- eq (include "kube-prometheus-stack.ingress.apiVersion" .) "networking.k8s.io/v1" -}} +{{- end -}} + +{{/* Check Ingress supports pathType */}} +{{/* pathType was added to networking.k8s.io/v1beta1 in Kubernetes 1.18 */}} +{{- define "kube-prometheus-stack.ingress.supportsPathType" -}} + {{- or (eq (include "kube-prometheus-stack.ingress.isStable" .) "true") (and (eq (include "kube-prometheus-stack.ingress.apiVersion" .) "networking.k8s.io/v1beta1") (semverCompare ">= 1.18-0" (include "kube-prometheus-stack.kubeVersion" .))) -}} +{{- end -}} + +{{/* Get Policy API Version */}} +{{- define "kube-prometheus-stack.pdb.apiVersion" -}} + {{- if and (.Capabilities.APIVersions.Has "policy/v1") (semverCompare ">= 1.21-0" (include "kube-prometheus-stack.kubeVersion" .)) -}} + {{- print "policy/v1" -}} + {{- else -}} + {{- print "policy/v1beta1" -}} + {{- end -}} + {{- end -}} + +{{/* Get value based on current Kubernetes version */}} +{{- define "kube-prometheus-stack.kubeVersionDefaultValue" -}} + {{- $values := index . 0 -}} + {{- $kubeVersion := index . 1 -}} + {{- $old := index . 2 -}} + {{- $new := index . 3 -}} + {{- $default := index . 4 -}} + {{- if kindIs "invalid" $default -}} + {{- if semverCompare $kubeVersion (include "kube-prometheus-stack.kubeVersion" $values) -}} + {{- print $new -}} + {{- else -}} + {{- print $old -}} + {{- end -}} + {{- else -}} + {{- print $default }} + {{- end -}} +{{- end -}} + +{{/* Get value for kube-controller-manager depending on insecure scraping availability */}} +{{- define "kube-prometheus-stack.kubeControllerManager.insecureScrape" -}} + {{- $values := index . 0 -}} + {{- $insecure := index . 1 -}} + {{- $secure := index . 2 -}} + {{- $userValue := index . 3 -}} + {{- include "kube-prometheus-stack.kubeVersionDefaultValue" (list $values ">= 1.22-0" $insecure $secure $userValue) -}} +{{- end -}} + +{{/* Get value for kube-scheduler depending on insecure scraping availability */}} +{{- define "kube-prometheus-stack.kubeScheduler.insecureScrape" -}} + {{- $values := index . 0 -}} + {{- $insecure := index . 1 -}} + {{- $secure := index . 2 -}} + {{- $userValue := index . 3 -}} + {{- include "kube-prometheus-stack.kubeVersionDefaultValue" (list $values ">= 1.23-0" $insecure $secure $userValue) -}} +{{- end -}} + +{{/* +To help compatibility with other charts which use global.imagePullSecrets. +Allow either an array of {name: pullSecret} maps (k8s-style), or an array of strings (more common helm-style). +global: + imagePullSecrets: + - name: pullSecret1 + - name: pullSecret2 + +or + +global: + imagePullSecrets: + - pullSecret1 + - pullSecret2 +*/}} +{{- define "kube-prometheus-stack.imagePullSecrets" -}} +{{- range .Values.global.imagePullSecrets }} + {{- if eq (typeOf .) "map[string]interface {}" }} +- {{ toYaml . | trim }} + {{- else }} +- name: {{ . }} + {{- end }} +{{- end }} +{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/alertmanager/alertmanager.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/alertmanager/alertmanager.yaml new file mode 100644 index 0000000000..c609852858 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/alertmanager/alertmanager.yaml @@ -0,0 +1,170 @@ +{{- if .Values.alertmanager.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: Alertmanager +metadata: + name: {{ template "kube-prometheus-stack.alertmanager.crname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.alertmanager.annotations }} + annotations: +{{ toYaml .Values.alertmanager.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.alertmanager.alertmanagerSpec.image }} + {{- if and .Values.alertmanager.alertmanagerSpec.image.tag .Values.alertmanager.alertmanagerSpec.image.sha }} + image: "{{ template "system_default_registry" . }}{{ .Values.alertmanager.alertmanagerSpec.image.repository }}:{{ .Values.alertmanager.alertmanagerSpec.image.tag }}@sha256:{{ .Values.alertmanager.alertmanagerSpec.image.sha }}" + {{- else if .Values.alertmanager.alertmanagerSpec.image.sha }} + image: "{{ template "system_default_registry" . }}{{ .Values.alertmanager.alertmanagerSpec.image.repository }}@sha256:{{ .Values.alertmanager.alertmanagerSpec.image.sha }}" + {{- else if .Values.alertmanager.alertmanagerSpec.image.tag }} + image: "{{ template "system_default_registry" . }}{{ .Values.alertmanager.alertmanagerSpec.image.repository }}:{{ .Values.alertmanager.alertmanagerSpec.image.tag }}" + {{- else }} + image: "{{ template "system_default_registry" . }}{{ .Values.alertmanager.alertmanagerSpec.image.repository }}" + {{- end }} + version: {{ .Values.alertmanager.alertmanagerSpec.image.tag }} + {{- if .Values.alertmanager.alertmanagerSpec.image.sha }} + sha: {{ .Values.alertmanager.alertmanagerSpec.image.sha }} + {{- end }} +{{- end }} + replicas: {{ .Values.alertmanager.alertmanagerSpec.replicas }} + listenLocal: {{ .Values.alertmanager.alertmanagerSpec.listenLocal }} + serviceAccountName: {{ template "kube-prometheus-stack.alertmanager.serviceAccountName" . }} +{{- if .Values.alertmanager.alertmanagerSpec.externalUrl }} + externalUrl: "{{ tpl .Values.alertmanager.alertmanagerSpec.externalUrl . }}" +{{- else if and .Values.alertmanager.ingress.enabled .Values.alertmanager.ingress.hosts }} + externalUrl: "http://{{ tpl (index .Values.alertmanager.ingress.hosts 0) . }}{{ .Values.alertmanager.alertmanagerSpec.routePrefix }}" +{{- else if not (or (kindIs "invalid" .Values.global.cattle.url) (kindIs "invalid" .Values.global.cattle.clusterId)) }} + externalUrl: "{{ .Values.global.cattle.url }}/k8s/clusters/{{ .Values.global.cattle.clusterId }}/api/v1/namespaces/{{ .Values.namespaceOverride }}/services/http:{{ template "kube-prometheus-stack.fullname" . }}-alertmanager:{{ .Values.alertmanager.service.port }}/proxy" +{{- else }} + externalUrl: http://{{ template "kube-prometheus-stack.fullname" . }}-alertmanager.{{ template "kube-prometheus-stack.namespace" . }}:{{ .Values.alertmanager.service.port }} +{{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 4 }} +{{- if .Values.alertmanager.alertmanagerSpec.nodeSelector }} +{{ toYaml .Values.alertmanager.alertmanagerSpec.nodeSelector | indent 4 }} +{{- end }} + paused: {{ .Values.alertmanager.alertmanagerSpec.paused }} + logFormat: {{ .Values.alertmanager.alertmanagerSpec.logFormat | quote }} + logLevel: {{ .Values.alertmanager.alertmanagerSpec.logLevel | quote }} + retention: {{ .Values.alertmanager.alertmanagerSpec.retention | quote }} +{{- if .Values.alertmanager.alertmanagerSpec.secrets }} + secrets: +{{ toYaml .Values.alertmanager.alertmanagerSpec.secrets | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.configSecret }} + configSecret: {{ .Values.alertmanager.alertmanagerSpec.configSecret }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.configMaps }} + configMaps: +{{ toYaml .Values.alertmanager.alertmanagerSpec.configMaps | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.alertmanagerConfigSelector }} + alertmanagerConfigSelector: +{{ toYaml .Values.alertmanager.alertmanagerSpec.alertmanagerConfigSelector | indent 4}} +{{ else }} + alertmanagerConfigSelector: {} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.alertmanagerConfigNamespaceSelector }} + alertmanagerConfigNamespaceSelector: +{{ toYaml .Values.alertmanager.alertmanagerSpec.alertmanagerConfigNamespaceSelector | indent 4}} +{{ else }} + alertmanagerConfigNamespaceSelector: {} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.web }} + web: +{{ toYaml .Values.alertmanager.alertmanagerSpec.web | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.alertmanagerConfiguration }} + alertmanagerConfiguration: +{{ toYaml .Values.alertmanager.alertmanagerSpec.alertmanagerConfiguration | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.resources }} + resources: +{{ toYaml .Values.alertmanager.alertmanagerSpec.resources | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.routePrefix }} + routePrefix: "{{ .Values.alertmanager.alertmanagerSpec.routePrefix }}" +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.securityContext }} + securityContext: +{{ toYaml .Values.alertmanager.alertmanagerSpec.securityContext | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.storage }} + storage: +{{ tpl (toYaml .Values.alertmanager.alertmanagerSpec.storage | indent 4) . }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.podMetadata }} + podMetadata: +{{ toYaml .Values.alertmanager.alertmanagerSpec.podMetadata | indent 4 }} +{{- end }} +{{- if or .Values.alertmanager.alertmanagerSpec.podAntiAffinity .Values.alertmanager.alertmanagerSpec.affinity }} + affinity: +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.affinity }} +{{ toYaml .Values.alertmanager.alertmanagerSpec.affinity | indent 4 }} +{{- end }} +{{- if eq .Values.alertmanager.alertmanagerSpec.podAntiAffinity "hard" }} + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: {{ .Values.alertmanager.alertmanagerSpec.podAntiAffinityTopologyKey }} + labelSelector: + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [alertmanager]} + - {key: alertmanager, operator: In, values: [{{ template "kube-prometheus-stack.alertmanager.crname" . }}]} +{{- else if eq .Values.alertmanager.alertmanagerSpec.podAntiAffinity "soft" }} + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: {{ .Values.alertmanager.alertmanagerSpec.podAntiAffinityTopologyKey }} + labelSelector: + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [alertmanager]} + - {key: alertmanager, operator: In, values: [{{ template "kube-prometheus-stack.alertmanager.crname" . }}]} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 4 }} +{{- if .Values.alertmanager.alertmanagerSpec.tolerations }} +{{ toYaml .Values.alertmanager.alertmanagerSpec.tolerations | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.topologySpreadConstraints }} + topologySpreadConstraints: +{{ toYaml .Values.alertmanager.alertmanagerSpec.topologySpreadConstraints | indent 4 }} +{{- end }} +{{- if .Values.global.imagePullSecrets }} + imagePullSecrets: +{{ include "kube-prometheus-stack.imagePullSecrets" . | trim | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.containers }} + containers: +{{ toYaml .Values.alertmanager.alertmanagerSpec.containers | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.initContainers }} + initContainers: +{{ toYaml .Values.alertmanager.alertmanagerSpec.initContainers | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.priorityClassName }} + priorityClassName: {{.Values.alertmanager.alertmanagerSpec.priorityClassName }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.additionalPeers }} + additionalPeers: +{{ toYaml .Values.alertmanager.alertmanagerSpec.additionalPeers | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.volumes }} + volumes: +{{ toYaml .Values.alertmanager.alertmanagerSpec.volumes | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.volumeMounts }} + volumeMounts: +{{ toYaml .Values.alertmanager.alertmanagerSpec.volumeMounts | indent 4 }} +{{- end }} + portName: {{ .Values.alertmanager.alertmanagerSpec.portName }} +{{- if .Values.alertmanager.alertmanagerSpec.clusterAdvertiseAddress }} + clusterAdvertiseAddress: {{ .Values.alertmanager.alertmanagerSpec.clusterAdvertiseAddress }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.forceEnableClusterMode }} + forceEnableClusterMode: {{ .Values.alertmanager.alertmanagerSpec.forceEnableClusterMode }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.minReadySeconds }} + minReadySeconds: {{ .Values.alertmanager.alertmanagerSpec.minReadySeconds }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/alertmanager/extrasecret.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/alertmanager/extrasecret.yaml new file mode 100644 index 0000000000..ecd8f47021 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/alertmanager/extrasecret.yaml @@ -0,0 +1,20 @@ +{{- if .Values.alertmanager.extraSecret.data -}} +{{- $secretName := printf "alertmanager-%s-extra" (include "kube-prometheus-stack.fullname" . ) -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ default $secretName .Values.alertmanager.extraSecret.name }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.alertmanager.extraSecret.annotations }} + annotations: +{{ toYaml .Values.alertmanager.extraSecret.annotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager + app.kubernetes.io/component: alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: +{{- range $key, $val := .Values.alertmanager.extraSecret.data }} + {{ $key }}: {{ $val | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/alertmanager/ingress.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/alertmanager/ingress.yaml new file mode 100644 index 0000000000..29c9cbceb3 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/alertmanager/ingress.yaml @@ -0,0 +1,77 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.ingress.enabled }} +{{- $pathType := .Values.alertmanager.ingress.pathType | default "ImplementationSpecific" }} +{{- $serviceName := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "alertmanager" }} +{{- $servicePort := .Values.alertmanager.ingress.servicePort | default .Values.alertmanager.service.port -}} +{{- $routePrefix := list .Values.alertmanager.alertmanagerSpec.routePrefix }} +{{- $paths := .Values.alertmanager.ingress.paths | default $routePrefix -}} +{{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "kube-prometheus-stack.ingress.supportsPathType" .) "true" -}} +apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" . }} +kind: Ingress +metadata: + name: {{ $serviceName }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.alertmanager.ingress.annotations }} + annotations: +{{ toYaml .Values.alertmanager.ingress.annotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{- if .Values.alertmanager.ingress.labels }} +{{ toYaml .Values.alertmanager.ingress.labels | indent 4 }} +{{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- if $apiIsStable }} + {{- if .Values.alertmanager.ingress.ingressClassName }} + ingressClassName: {{ .Values.alertmanager.ingress.ingressClassName }} + {{- end }} + {{- end }} + rules: + {{- if .Values.alertmanager.ingress.hosts }} + {{- range $host := .Values.alertmanager.ingress.hosts }} + - host: {{ tpl $host $ }} + http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $serviceName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $serviceName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- else }} + - http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $serviceName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $serviceName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- if .Values.alertmanager.ingress.tls }} + tls: +{{ tpl (toYaml .Values.alertmanager.ingress.tls | indent 4) . }} + {{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/alertmanager/ingressperreplica.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/alertmanager/ingressperreplica.yaml new file mode 100644 index 0000000000..f21bf96169 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/alertmanager/ingressperreplica.yaml @@ -0,0 +1,67 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.servicePerReplica.enabled .Values.alertmanager.ingressPerReplica.enabled }} +{{- $pathType := .Values.alertmanager.ingressPerReplica.pathType | default "" }} +{{- $count := .Values.alertmanager.alertmanagerSpec.replicas | int -}} +{{- $servicePort := .Values.alertmanager.service.port -}} +{{- $ingressValues := .Values.alertmanager.ingressPerReplica -}} +{{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "kube-prometheus-stack.ingress.supportsPathType" .) "true" -}} +apiVersion: v1 +kind: List +metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-ingressperreplica + namespace: {{ template "kube-prometheus-stack.namespace" . }} +items: +{{ range $i, $e := until $count }} + - kind: Ingress + apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" $ }} + metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-{{ $i }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ include "kube-prometheus-stack.name" $ }}-alertmanager + {{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if $ingressValues.labels }} +{{ toYaml $ingressValues.labels | indent 8 }} + {{- end }} + {{- if $ingressValues.annotations }} + annotations: +{{ toYaml $ingressValues.annotations | indent 8 }} + {{- end }} + spec: + {{- if $apiIsStable }} + {{- if $ingressValues.ingressClassName }} + ingressClassName: {{ $ingressValues.ingressClassName }} + {{- end }} + {{- end }} + rules: + - host: {{ $ingressValues.hostPrefix }}-{{ $i }}.{{ $ingressValues.hostDomain }} + http: + paths: + {{- range $p := $ingressValues.paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-{{ $i }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-{{ $i }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- if or $ingressValues.tlsSecretName $ingressValues.tlsSecretPerReplica.enabled }} + tls: + - hosts: + - {{ $ingressValues.hostPrefix }}-{{ $i }}.{{ $ingressValues.hostDomain }} + {{- if $ingressValues.tlsSecretPerReplica.enabled }} + secretName: {{ $ingressValues.tlsSecretPerReplica.prefix }}-{{ $i }} + {{- else }} + secretName: {{ $ingressValues.tlsSecretName }} + {{- end }} + {{- end }} +{{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/alertmanager/podDisruptionBudget.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/alertmanager/podDisruptionBudget.yaml new file mode 100644 index 0000000000..b183403125 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/alertmanager/podDisruptionBudget.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.podDisruptionBudget.enabled }} +apiVersion: {{ include "kube-prometheus-stack.pdb.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- if .Values.alertmanager.podDisruptionBudget.minAvailable }} + minAvailable: {{ .Values.alertmanager.podDisruptionBudget.minAvailable }} + {{- end }} + {{- if .Values.alertmanager.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ .Values.alertmanager.podDisruptionBudget.maxUnavailable }} + {{- end }} + selector: + matchLabels: + app.kubernetes.io/name: alertmanager + alertmanager: {{ template "kube-prometheus-stack.alertmanager.crname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/alertmanager/psp-role.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/alertmanager/psp-role.yaml new file mode 100644 index 0000000000..c645452639 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/alertmanager/psp-role.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.alertmanager.enabled .Values.global.rbac.create .Values.global.cattle.psp.enabled }} +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +rules: +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if semverCompare "> 1.15.0-0" $kubeTargetVersion }} +- apiGroups: ['policy'] +{{- else }} +- apiGroups: ['extensions'] +{{- end }} + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "kube-prometheus-stack.fullname" . }}-alertmanager +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/alertmanager/psp-rolebinding.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/alertmanager/psp-rolebinding.yaml new file mode 100644 index 0000000000..6f014c570b --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/alertmanager/psp-rolebinding.yaml @@ -0,0 +1,18 @@ +{{- if and .Values.alertmanager.enabled .Values.global.rbac.create .Values.global.cattle.psp.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.alertmanager.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/alertmanager/psp.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/alertmanager/psp.yaml new file mode 100644 index 0000000000..57abbf3340 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/alertmanager/psp.yaml @@ -0,0 +1,45 @@ +{{- if and .Values.alertmanager.enabled .Values.global.rbac.create .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{- if .Values.global.rbac.pspAnnotations }} + annotations: +{{ toYaml .Values.global.rbac.pspAnnotations | indent 4 }} +{{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + privileged: false + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + # Permits the container to run with root privileges as well. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/alertmanager/secret.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/alertmanager/secret.yaml new file mode 100644 index 0000000000..c0a8b9fe66 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/alertmanager/secret.yaml @@ -0,0 +1,33 @@ +{{- if and (.Values.alertmanager.enabled) (not .Values.alertmanager.alertmanagerSpec.useExistingSecret) }} +{{/* This file is applied when the operation is helm install and the target secret does not exist. */}} +{{- $secretName := (printf "alertmanager-%s" (include "kube-prometheus-stack.alertmanager.crname" .)) }} +{{- if (not (lookup "v1" "Secret" (include "kube-prometheus-stack.namespace" .) $secretName)) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install, pre-upgrade + "helm.sh/hook-weight": "3" + "helm.sh/resource-policy": keep +{{- if .Values.alertmanager.secret.annotations }} +{{ toYaml .Values.alertmanager.secret.annotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: +{{- if .Values.alertmanager.tplConfig }} +{{- if eq (typeOf .Values.alertmanager.config) "string" }} + alertmanager.yaml: {{ tpl (.Values.alertmanager.config) . | b64enc | quote }} +{{- else }} + alertmanager.yaml: {{ tpl (toYaml .Values.alertmanager.config) . | b64enc | quote }} +{{- end }} +{{- else }} + alertmanager.yaml: {{ toYaml .Values.alertmanager.config | b64enc | quote }} +{{- end}} +{{- range $key, $val := .Values.alertmanager.templateFiles }} + {{ $key }}: {{ $val | b64enc | quote }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/alertmanager/service.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/alertmanager/service.yaml new file mode 100644 index 0000000000..44100ec1cc --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/alertmanager/service.yaml @@ -0,0 +1,53 @@ +{{- if .Values.alertmanager.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager + self-monitor: {{ .Values.alertmanager.serviceMonitor.selfMonitor | quote }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.alertmanager.service.labels }} +{{ toYaml .Values.alertmanager.service.labels | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.service.annotations }} + annotations: +{{ toYaml .Values.alertmanager.service.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.alertmanager.service.clusterIP }} + clusterIP: {{ .Values.alertmanager.service.clusterIP }} +{{- end }} +{{- if .Values.alertmanager.service.externalIPs }} + externalIPs: +{{ toYaml .Values.alertmanager.service.externalIPs | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.alertmanager.service.loadBalancerIP }} +{{- end }} +{{- if .Values.alertmanager.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.alertmanager.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if ne .Values.alertmanager.service.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.alertmanager.service.externalTrafficPolicy }} +{{- end }} + ports: + - name: {{ .Values.alertmanager.alertmanagerSpec.portName }} + {{- if eq .Values.alertmanager.service.type "NodePort" }} + nodePort: {{ .Values.alertmanager.service.nodePort }} + {{- end }} + port: {{ .Values.alertmanager.service.port }} + targetPort: {{ .Values.alertmanager.service.targetPort }} + protocol: TCP +{{- if .Values.alertmanager.service.additionalPorts }} +{{ toYaml .Values.alertmanager.service.additionalPorts | indent 2 }} +{{- end }} + selector: + app.kubernetes.io/name: alertmanager + alertmanager: {{ template "kube-prometheus-stack.alertmanager.crname" . }} + type: "{{ .Values.alertmanager.service.type }}" +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/alertmanager/serviceaccount.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/alertmanager/serviceaccount.yaml new file mode 100644 index 0000000000..ae433d5531 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/alertmanager/serviceaccount.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "kube-prometheus-stack.alertmanager.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager + app.kubernetes.io/name: {{ template "kube-prometheus-stack.name" . }}-alertmanager + app.kubernetes.io/component: alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.alertmanager.serviceAccount.annotations }} + annotations: +{{ toYaml .Values.alertmanager.serviceAccount.annotations | indent 4 }} +{{- end }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{ include "kube-prometheus-stack.imagePullSecrets" . | trim | indent 2}} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/alertmanager/servicemonitor.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/alertmanager/servicemonitor.yaml new file mode 100644 index 0000000000..b1e0ec34f6 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/alertmanager/servicemonitor.yaml @@ -0,0 +1,55 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.serviceMonitor.selfMonitor }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager + release: {{ $.Release.Name | quote }} + self-monitor: "true" + namespaceSelector: + matchNames: + - {{ printf "%s" (include "kube-prometheus-stack.namespace" .) | quote }} + endpoints: + - port: {{ .Values.alertmanager.alertmanagerSpec.portName }} + {{- if .Values.alertmanager.serviceMonitor.interval }} + interval: {{ .Values.alertmanager.serviceMonitor.interval }} + {{- end }} + {{- if .Values.alertmanager.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.alertmanager.serviceMonitor.proxyUrl}} + {{- end }} + {{- if .Values.alertmanager.serviceMonitor.scheme }} + scheme: {{ .Values.alertmanager.serviceMonitor.scheme }} + {{- end }} + {{- if .Values.alertmanager.serviceMonitor.bearerTokenFile }} + bearerTokenFile: {{ .Values.alertmanager.serviceMonitor.bearerTokenFile }} + {{- end }} + {{- if .Values.alertmanager.serviceMonitor.tlsConfig }} + tlsConfig: {{ toYaml .Values.alertmanager.serviceMonitor.tlsConfig | nindent 6 }} + {{- end }} + path: "{{ trimSuffix "/" .Values.alertmanager.alertmanagerSpec.routePrefix }}/metrics" + metricRelabelings: + {{- if .Values.alertmanager.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.alertmanager.serviceMonitor.metricRelabelings | indent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName }} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.alertmanager.serviceMonitor.relabelings }} + relabelings: +{{ toYaml .Values.alertmanager.serviceMonitor.relabelings | indent 6 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/alertmanager/serviceperreplica.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/alertmanager/serviceperreplica.yaml new file mode 100644 index 0000000000..75a13bdf97 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/alertmanager/serviceperreplica.yaml @@ -0,0 +1,49 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.servicePerReplica.enabled }} +{{- $count := .Values.alertmanager.alertmanagerSpec.replicas | int -}} +{{- $serviceValues := .Values.alertmanager.servicePerReplica -}} +apiVersion: v1 +kind: List +metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-serviceperreplica + namespace: {{ template "kube-prometheus-stack.namespace" . }} +items: +{{- range $i, $e := until $count }} + - apiVersion: v1 + kind: Service + metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-{{ $i }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ include "kube-prometheus-stack.name" $ }}-alertmanager +{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if $serviceValues.annotations }} + annotations: +{{ toYaml $serviceValues.annotations | indent 8 }} + {{- end }} + spec: + {{- if $serviceValues.clusterIP }} + clusterIP: {{ $serviceValues.clusterIP }} + {{- end }} + {{- if $serviceValues.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := $serviceValues.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} + {{- end }} + {{- if ne $serviceValues.type "ClusterIP" }} + externalTrafficPolicy: {{ $serviceValues.externalTrafficPolicy }} + {{- end }} + ports: + - name: {{ $.Values.alertmanager.alertmanagerSpec.portName }} + {{- if eq $serviceValues.type "NodePort" }} + nodePort: {{ $serviceValues.nodePort }} + {{- end }} + port: {{ $serviceValues.port }} + targetPort: {{ $serviceValues.targetPort }} + selector: + app.kubernetes.io/name: alertmanager + alertmanager: {{ template "kube-prometheus-stack.alertmanager.crname" $ }} + statefulset.kubernetes.io/pod-name: alertmanager-{{ include "kube-prometheus-stack.alertmanager.crname" $ }}-{{ $i }} + type: "{{ $serviceValues.type }}" +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/core-dns/service.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/core-dns/service.yaml new file mode 100644 index 0000000000..f77db41991 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/core-dns/service.yaml @@ -0,0 +1,24 @@ +{{- if .Values.coreDns.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-coredns + labels: + app: {{ template "kube-prometheus-stack.name" . }}-coredns + jobLabel: coredns +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +spec: + clusterIP: None + ports: + - name: http-metrics + port: {{ .Values.coreDns.service.port }} + protocol: TCP + targetPort: {{ .Values.coreDns.service.targetPort }} + selector: + {{- if .Values.coreDns.service.selector }} +{{ toYaml .Values.coreDns.service.selector | indent 4 }} + {{- else}} + k8s-app: kube-dns + {{- end}} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/core-dns/servicemonitor.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/core-dns/servicemonitor.yaml new file mode 100644 index 0000000000..c3049e2a0e --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/core-dns/servicemonitor.yaml @@ -0,0 +1,49 @@ +{{- if .Values.coreDns.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-coredns + namespace: "kube-system" + labels: + app: {{ template "kube-prometheus-stack.name" . }}-coredns + {{- with .Values.coreDns.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: jobLabel + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-coredns + release: {{ $.Release.Name | quote }} + namespaceSelector: + matchNames: + - "kube-system" + endpoints: + - port: http-metrics + {{- if .Values.coreDns.serviceMonitor.interval}} + interval: {{ .Values.coreDns.serviceMonitor.interval }} + {{- end }} + {{- if .Values.coreDns.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.coreDns.serviceMonitor.proxyUrl}} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + metricRelabelings: + {{- if .Values.coreDns.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.coreDns.serviceMonitor.metricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName }} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.coreDns.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.coreDns.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-api-server/servicemonitor.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-api-server/servicemonitor.yaml new file mode 100644 index 0000000000..1b57f302f7 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-api-server/servicemonitor.yaml @@ -0,0 +1,52 @@ +{{- if .Values.kubeApiServer.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-apiserver + namespace: default + labels: + app: {{ template "kube-prometheus-stack.name" . }}-apiserver + {{- with .Values.kubeApiServer.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + endpoints: + - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + {{- if .Values.kubeApiServer.serviceMonitor.interval }} + interval: {{ .Values.kubeApiServer.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubeApiServer.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubeApiServer.serviceMonitor.proxyUrl}} + {{- end }} + port: https + scheme: https + metricRelabelings: + {{- if .Values.kubeApiServer.serviceMonitor.metricRelabelings }} +{{ tpl (toYaml .Values.kubeApiServer.serviceMonitor.metricRelabelings | indent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubeApiServer.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubeApiServer.serviceMonitor.relabelings | indent 6) . }} +{{- end }} + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + serverName: {{ .Values.kubeApiServer.tlsConfig.serverName }} + insecureSkipVerify: {{ .Values.kubeApiServer.tlsConfig.insecureSkipVerify }} + jobLabel: {{ .Values.kubeApiServer.serviceMonitor.jobLabel }} + namespaceSelector: + matchNames: + - default + selector: +{{ toYaml .Values.kubeApiServer.serviceMonitor.selector | indent 4 }} +{{- end}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-controller-manager/endpoints.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-controller-manager/endpoints.yaml new file mode 100644 index 0000000000..eca337dab9 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-controller-manager/endpoints.yaml @@ -0,0 +1,22 @@ +{{- if and .Values.kubeControllerManager.enabled .Values.kubeControllerManager.endpoints }} +apiVersion: v1 +kind: Endpoints +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-controller-manager + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-controller-manager + k8s-app: kube-controller-manager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +subsets: + - addresses: + {{- range .Values.kubeControllerManager.endpoints }} + - ip: {{ . }} + {{- end }} + ports: + - name: http-metrics + {{- $kubeControllerManagerDefaultInsecurePort := 10252 }} + {{- $kubeControllerManagerDefaultSecurePort := 10257 }} + port: {{ include "kube-prometheus-stack.kubeControllerManager.insecureScrape" (list . $kubeControllerManagerDefaultInsecurePort $kubeControllerManagerDefaultSecurePort .Values.kubeControllerManager.service.port) }} + protocol: TCP +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-controller-manager/service.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-controller-manager/service.yaml new file mode 100644 index 0000000000..197f0f4f6e --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-controller-manager/service.yaml @@ -0,0 +1,29 @@ +{{- if and .Values.kubeControllerManager.enabled .Values.kubeControllerManager.service.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-controller-manager + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-controller-manager + jobLabel: kube-controller-manager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +spec: + clusterIP: None + ports: + - name: http-metrics + {{- $kubeControllerManagerDefaultInsecurePort := 10252 }} + {{- $kubeControllerManagerDefaultSecurePort := 10257 }} + port: {{ include "kube-prometheus-stack.kubeControllerManager.insecureScrape" (list . $kubeControllerManagerDefaultInsecurePort $kubeControllerManagerDefaultSecurePort .Values.kubeControllerManager.service.port) }} + protocol: TCP + targetPort: {{ include "kube-prometheus-stack.kubeControllerManager.insecureScrape" (list . $kubeControllerManagerDefaultInsecurePort $kubeControllerManagerDefaultSecurePort .Values.kubeControllerManager.service.targetPort) }} +{{- if .Values.kubeControllerManager.endpoints }}{{- else }} + selector: + {{- if .Values.kubeControllerManager.service.selector }} +{{ toYaml .Values.kubeControllerManager.service.selector | indent 4 }} + {{- else}} + component: kube-controller-manager + {{- end}} +{{- end }} + type: ClusterIP +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-controller-manager/servicemonitor.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-controller-manager/servicemonitor.yaml new file mode 100644 index 0000000000..1c7e778ed6 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-controller-manager/servicemonitor.yaml @@ -0,0 +1,60 @@ +{{- if and .Values.kubeControllerManager.enabled .Values.kubeControllerManager.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-controller-manager + namespace: "kube-system" + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-controller-manager + {{- with .Values.kubeControllerManager.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: jobLabel + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-controller-manager + release: {{ $.Release.Name | quote }} + namespaceSelector: + matchNames: + - "kube-system" + endpoints: + - port: http-metrics + {{- if .Values.kubeControllerManager.serviceMonitor.interval }} + interval: {{ .Values.kubeControllerManager.serviceMonitor.interval }} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + {{- if .Values.kubeControllerManager.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubeControllerManager.serviceMonitor.proxyUrl}} + {{- end }} + {{- if eq (include "kube-prometheus-stack.kubeControllerManager.insecureScrape" (list . false true .Values.kubeControllerManager.serviceMonitor.https )) "true" }} + scheme: https + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + {{- if eq (include "kube-prometheus-stack.kubeControllerManager.insecureScrape" (list . nil true .Values.kubeControllerManager.serviceMonitor.insecureSkipVerify)) "true" }} + insecureSkipVerify: true + {{- end }} + {{- if .Values.kubeControllerManager.serviceMonitor.serverName }} + serverName: {{ .Values.kubeControllerManager.serviceMonitor.serverName }} + {{- end }} + {{- end }} + metricRelabelings: + {{- if.Values.kubeControllerManager.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.kubeControllerManager.serviceMonitor.metricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubeControllerManager.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubeControllerManager.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-dns/service.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-dns/service.yaml new file mode 100644 index 0000000000..c7bf142d52 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-dns/service.yaml @@ -0,0 +1,28 @@ +{{- if .Values.kubeDns.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-dns + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-dns + jobLabel: kube-dns +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +spec: + clusterIP: None + ports: + - name: http-metrics-dnsmasq + port: {{ .Values.kubeDns.service.dnsmasq.port }} + protocol: TCP + targetPort: {{ .Values.kubeDns.service.dnsmasq.targetPort }} + - name: http-metrics-skydns + port: {{ .Values.kubeDns.service.skydns.port }} + protocol: TCP + targetPort: {{ .Values.kubeDns.service.skydns.targetPort }} + selector: + {{- if .Values.kubeDns.service.selector }} +{{ toYaml .Values.kubeDns.service.selector | indent 4 }} + {{- else}} + k8s-app: kube-dns + {{- end}} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-dns/servicemonitor.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-dns/servicemonitor.yaml new file mode 100644 index 0000000000..c2fb9c426a --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-dns/servicemonitor.yaml @@ -0,0 +1,62 @@ +{{- if .Values.kubeDns.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-dns + namespace: "kube-system" + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-dns + {{- with .Values.kubeDns.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: jobLabel + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-dns + release: {{ $.Release.Name | quote }} + namespaceSelector: + matchNames: + - "kube-system" + endpoints: + - port: http-metrics-dnsmasq + {{- if .Values.kubeDns.serviceMonitor.interval }} + interval: {{ .Values.kubeDns.serviceMonitor.interval }} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + {{- if .Values.kubeDns.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubeDns.serviceMonitor.proxyUrl}} + {{- end }} + metricRelabelings: + {{- if .Values.kubeDns.serviceMonitor.dnsmasqMetricRelabelings }} + {{ tpl (toYaml .Values.kubeDns.serviceMonitor.dnsmasqMetricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubeDns.serviceMonitor.dnsmasqRelabelings }} + relabelings: +{{ toYaml .Values.kubeDns.serviceMonitor.dnsmasqRelabelings | indent 4 }} +{{- end }} + - port: http-metrics-skydns + {{- if .Values.kubeDns.serviceMonitor.interval }} + interval: {{ .Values.kubeDns.serviceMonitor.interval }} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +{{- if .Values.kubeDns.serviceMonitor.metricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubeDns.serviceMonitor.metricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubeDns.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubeDns.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-etcd/endpoints.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-etcd/endpoints.yaml new file mode 100644 index 0000000000..8f07a5cc24 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-etcd/endpoints.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.kubeEtcd.enabled .Values.kubeEtcd.endpoints }} +apiVersion: v1 +kind: Endpoints +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-etcd + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-etcd + k8s-app: etcd-server +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +subsets: + - addresses: + {{- range .Values.kubeEtcd.endpoints }} + - ip: {{ . }} + {{- end }} + ports: + - name: http-metrics + port: {{ .Values.kubeEtcd.service.port }} + protocol: TCP +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-etcd/service.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-etcd/service.yaml new file mode 100644 index 0000000000..b2677e2807 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-etcd/service.yaml @@ -0,0 +1,27 @@ +{{- if and .Values.kubeEtcd.enabled .Values.kubeEtcd.service.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-etcd + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-etcd + jobLabel: kube-etcd +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +spec: + clusterIP: None + ports: + - name: http-metrics + port: {{ .Values.kubeEtcd.service.port }} + protocol: TCP + targetPort: {{ .Values.kubeEtcd.service.targetPort }} +{{- if .Values.kubeEtcd.endpoints }}{{- else }} + selector: + {{- if .Values.kubeEtcd.service.selector }} +{{ toYaml .Values.kubeEtcd.service.selector | indent 4 }} + {{- else}} + component: etcd + {{- end}} +{{- end }} + type: ClusterIP +{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-etcd/servicemonitor.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-etcd/servicemonitor.yaml new file mode 100644 index 0000000000..82388082f4 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-etcd/servicemonitor.yaml @@ -0,0 +1,66 @@ +{{- if and .Values.kubeEtcd.enabled .Values.kubeEtcd.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-etcd + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-etcd + {{- with .Values.kubeEtcd.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: jobLabel + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-etcd + release: {{ $.Release.Name | quote }} + namespaceSelector: + matchNames: + - "kube-system" + endpoints: + - port: http-metrics + {{- if .Values.kubeEtcd.serviceMonitor.interval }} + interval: {{ .Values.kubeEtcd.serviceMonitor.interval }} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + {{- if .Values.kubeEtcd.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubeEtcd.serviceMonitor.proxyUrl}} + {{- end }} + {{- if eq .Values.kubeEtcd.serviceMonitor.scheme "https" }} + scheme: https + tlsConfig: + {{- if .Values.kubeEtcd.serviceMonitor.serverName }} + serverName: {{ .Values.kubeEtcd.serviceMonitor.serverName }} + {{- end }} + {{- if .Values.kubeEtcd.serviceMonitor.caFile }} + caFile: {{ .Values.kubeEtcd.serviceMonitor.caFile }} + {{- end }} + {{- if .Values.kubeEtcd.serviceMonitor.certFile }} + certFile: {{ .Values.kubeEtcd.serviceMonitor.certFile }} + {{- end }} + {{- if .Values.kubeEtcd.serviceMonitor.keyFile }} + keyFile: {{ .Values.kubeEtcd.serviceMonitor.keyFile }} + {{- end}} + insecureSkipVerify: {{ .Values.kubeEtcd.serviceMonitor.insecureSkipVerify }} + {{- end }} + metricRelabelings: + {{- if .Values.kubeEtcd.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.kubeEtcd.serviceMonitor.metricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubeEtcd.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubeEtcd.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-proxy/endpoints.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-proxy/endpoints.yaml new file mode 100644 index 0000000000..2cb756d154 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-proxy/endpoints.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.kubeProxy.enabled .Values.kubeProxy.endpoints }} +apiVersion: v1 +kind: Endpoints +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-proxy + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-proxy + k8s-app: kube-proxy +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +subsets: + - addresses: + {{- range .Values.kubeProxy.endpoints }} + - ip: {{ . }} + {{- end }} + ports: + - name: http-metrics + port: {{ .Values.kubeProxy.service.port }} + protocol: TCP +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-proxy/service.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-proxy/service.yaml new file mode 100644 index 0000000000..6a93319ef3 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-proxy/service.yaml @@ -0,0 +1,27 @@ +{{- if and .Values.kubeProxy.enabled .Values.kubeProxy.service.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-proxy + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-proxy + jobLabel: kube-proxy +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +spec: + clusterIP: None + ports: + - name: http-metrics + port: {{ .Values.kubeProxy.service.port }} + protocol: TCP + targetPort: {{ .Values.kubeProxy.service.targetPort }} +{{- if .Values.kubeProxy.endpoints }}{{- else }} + selector: + {{- if .Values.kubeProxy.service.selector }} +{{ toYaml .Values.kubeProxy.service.selector | indent 4 }} + {{- else}} + k8s-app: kube-proxy + {{- end}} +{{- end }} + type: ClusterIP +{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-proxy/servicemonitor.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-proxy/servicemonitor.yaml new file mode 100644 index 0000000000..4add74ba14 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-proxy/servicemonitor.yaml @@ -0,0 +1,54 @@ +{{- if and .Values.kubeProxy.enabled .Values.kubeProxy.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-proxy + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-proxy + {{- with .Values.kubeProxy.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: jobLabel + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-proxy + release: {{ $.Release.Name | quote }} + namespaceSelector: + matchNames: + - "kube-system" + endpoints: + - port: http-metrics + {{- if .Values.kubeProxy.serviceMonitor.interval }} + interval: {{ .Values.kubeProxy.serviceMonitor.interval }} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + {{- if .Values.kubeProxy.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubeProxy.serviceMonitor.proxyUrl}} + {{- end }} + {{- if .Values.kubeProxy.serviceMonitor.https }} + scheme: https + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + {{- end}} + metricRelabelings: + {{- if .Values.kubeProxy.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.kubeProxy.serviceMonitor.metricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubeProxy.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubeProxy.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-scheduler/endpoints.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-scheduler/endpoints.yaml new file mode 100644 index 0000000000..84a14ae612 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-scheduler/endpoints.yaml @@ -0,0 +1,22 @@ +{{- if and .Values.kubeScheduler.enabled .Values.kubeScheduler.endpoints }} +apiVersion: v1 +kind: Endpoints +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-scheduler + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-scheduler + k8s-app: kube-scheduler +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +subsets: + - addresses: + {{- range .Values.kubeScheduler.endpoints }} + - ip: {{ . }} + {{- end }} + ports: + - name: http-metrics + {{- $kubeSchedulerDefaultInsecurePort := 10251 }} + {{- $kubeSchedulerDefaultSecurePort := 10259 }} + port: {{ include "kube-prometheus-stack.kubeScheduler.insecureScrape" (list . $kubeSchedulerDefaultInsecurePort $kubeSchedulerDefaultSecurePort .Values.kubeScheduler.service.port) }} + protocol: TCP +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-scheduler/service.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-scheduler/service.yaml new file mode 100644 index 0000000000..eef9df01d0 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-scheduler/service.yaml @@ -0,0 +1,29 @@ +{{- if and .Values.kubeScheduler.enabled .Values.kubeScheduler.service.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-scheduler + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-scheduler + jobLabel: kube-scheduler +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +spec: + clusterIP: None + ports: + - name: http-metrics + {{- $kubeSchedulerDefaultInsecurePort := 10251 }} + {{- $kubeSchedulerDefaultSecurePort := 10259 }} + port: {{ include "kube-prometheus-stack.kubeScheduler.insecureScrape" (list . $kubeSchedulerDefaultInsecurePort $kubeSchedulerDefaultSecurePort .Values.kubeScheduler.service.port) }} + protocol: TCP + targetPort: {{ include "kube-prometheus-stack.kubeScheduler.insecureScrape" (list . $kubeSchedulerDefaultInsecurePort $kubeSchedulerDefaultSecurePort .Values.kubeScheduler.service.targetPort) }} +{{- if .Values.kubeScheduler.endpoints }}{{- else }} + selector: + {{- if .Values.kubeScheduler.service.selector }} +{{ toYaml .Values.kubeScheduler.service.selector | indent 4 }} + {{- else}} + component: kube-scheduler + {{- end}} +{{- end }} + type: ClusterIP +{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-scheduler/servicemonitor.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-scheduler/servicemonitor.yaml new file mode 100644 index 0000000000..e6555b448f --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-scheduler/servicemonitor.yaml @@ -0,0 +1,60 @@ +{{- if and .Values.kubeScheduler.enabled .Values.kubeScheduler.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-scheduler + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-scheduler + {{- with .Values.kubeScheduler.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: jobLabel + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-scheduler + release: {{ $.Release.Name | quote }} + namespaceSelector: + matchNames: + - "kube-system" + endpoints: + - port: http-metrics + {{- if .Values.kubeScheduler.serviceMonitor.interval }} + interval: {{ .Values.kubeScheduler.serviceMonitor.interval }} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + {{- if .Values.kubeScheduler.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubeScheduler.serviceMonitor.proxyUrl}} + {{- end }} + {{- if eq (include "kube-prometheus-stack.kubeScheduler.insecureScrape" (list . false true .Values.kubeScheduler.serviceMonitor.https )) "true" }} + scheme: https + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + {{- if eq (include "kube-prometheus-stack.kubeScheduler.insecureScrape" (list . nil true .Values.kubeScheduler.serviceMonitor.insecureSkipVerify)) "true" }} + insecureSkipVerify: true + {{- end }} + {{- if .Values.kubeScheduler.serviceMonitor.serverName }} + serverName: {{ .Values.kubeScheduler.serviceMonitor.serverName }} + {{- end}} + {{- end}} + metricRelabelings: + {{- if .Values.kubeScheduler.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.kubeScheduler.serviceMonitor.metricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubeScheduler.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubeScheduler.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-state-metrics/validate.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-state-metrics/validate.yaml new file mode 100644 index 0000000000..9211b3d771 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kube-state-metrics/validate.yaml @@ -0,0 +1,7 @@ +{{- if .Values.kubeStateMetrics.enabled }} +{{- if not (kindIs "invalid" .Values.kubeStateMetrics.serviceMonitor) }} +{{- if .Values.kubeStateMetrics.serviceMonitor.namespaceOverride }} +{{- fail "kubeStateMetrics.serviceMonitor.namespaceOverride was removed. Please use kube-state-metrics.namespaceOverride instead." }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kubelet/servicemonitor.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kubelet/servicemonitor.yaml new file mode 100644 index 0000000000..7d85a2d918 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/kubelet/servicemonitor.yaml @@ -0,0 +1,229 @@ +{{- if (and (not .Values.kubelet.enabled) .Values.hardenedKubelet.enabled) }} +{{ required "Cannot set .Values.hardenedKubelet.enabled=true when .Values.kubelet.enabled=false" "" }} +{{- end }} +{{- if (and .Values.kubelet.enabled (not .Values.hardenedKubelet.enabled) (not .Values.k3sServer.enabled)) }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kubelet + namespace: {{ .Values.kubelet.namespace }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kubelet + {{- with .Values.kubelet.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{- include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + endpoints: + {{- if .Values.kubelet.serviceMonitor.https }} + - port: https-metrics + scheme: https + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecureSkipVerify: true + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + honorLabels: true + metricRelabelings: + {{- if .Values.kubelet.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.kubelet.serviceMonitor.metricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubelet.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.cAdvisor }} + - port: https-metrics + scheme: https + path: /metrics/cadvisor + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: true + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecureSkipVerify: true + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +{{- if .Values.kubelet.serviceMonitor.cAdvisorMetricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.cAdvisorMetricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.cAdvisorRelabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.cAdvisorRelabelings | indent 4) . }} +{{- end }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.probes }} + - port: https-metrics + scheme: https + path: /metrics/probes + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: true + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecureSkipVerify: true + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +{{- if .Values.kubelet.serviceMonitor.probesMetricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.probesMetricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.probesRelabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.probesRelabelings | indent 4) . }} +{{- end }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.resource }} + - port: https-metrics + scheme: https + path: {{ include "kubelet.serviceMonitor.resourcePath" . }} + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: true + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecureSkipVerify: true + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +{{- if .Values.kubelet.serviceMonitor.resourceMetricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.resourceMetricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.resourceRelabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.resourceRelabelings | indent 4) . }} +{{- end }} +{{- end }} + {{- else }} + - port: http-metrics + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: true +{{- if .Values.kubelet.serviceMonitor.metricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.metricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.cAdvisor }} + - port: http-metrics + path: /metrics/cadvisor + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: true +{{- if .Values.kubelet.serviceMonitor.cAdvisorMetricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.cAdvisorMetricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.cAdvisorRelabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.cAdvisorRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.probes }} + - port: http-metrics + path: /metrics/probes + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: true +{{- if .Values.kubelet.serviceMonitor.probesMetricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.probesMetricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.probesRelabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.probesRelabelings | indent 4) . }} +{{- end }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.resource }} + - port: http-metrics + path: {{ include "kubelet.serviceMonitor.resourcePath" . }} + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: true +{{- if .Values.kubelet.serviceMonitor.resourceMetricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.resourceMetricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.resourceRelabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.resourceRelabelings | indent 4) . }} +{{- end }} +{{- end }} +{{- end }} + {{- end }} + jobLabel: k8s-app + namespaceSelector: + matchNames: + - {{ .Values.kubelet.namespace }} + selector: + matchLabels: + app.kubernetes.io/name: kubelet + k8s-app: kubelet +{{- end}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/node-exporter/validate.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/node-exporter/validate.yaml new file mode 100644 index 0000000000..bdc73d6165 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/exporters/node-exporter/validate.yaml @@ -0,0 +1,3 @@ +{{- if (and (not .Values.nodeExporter.enabled) .Values.hardenedNodeExporter.enabled) }} +{{ required "Cannot set .Values.hardenedNodeExporter.enabled=true when .Values.nodeExporter.enabled=false" "" }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/configmap-dashboards.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/configmap-dashboards.yaml new file mode 100644 index 0000000000..e719009ffe --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/configmap-dashboards.yaml @@ -0,0 +1,24 @@ +{{- if or (and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled) .Values.grafana.forceDeployDashboards }} +{{- $files := .Files.Glob "dashboards-1.14/*.json" }} +{{- if $files }} +apiVersion: v1 +kind: ConfigMapList +items: +{{- range $path, $fileContents := $files }} +{{- $dashboardName := regexReplaceAll "(^.*/)(.*)\\.json$" $path "${2}" }} +- apiVersion: v1 + kind: ConfigMap + metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) $dashboardName | trunc 63 | trimSuffix "-" }} + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 6 }} + data: + {{ $dashboardName }}.json: {{ $.Files.Get $path | toJson }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/configmaps-datasources.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/configmaps-datasources.yaml new file mode 100644 index 0000000000..44ed3273bd --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/configmaps-datasources.yaml @@ -0,0 +1,63 @@ +{{- if or (and .Values.grafana.enabled .Values.grafana.sidecar.datasources.enabled) .Values.grafana.forceDeployDatasources }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-grafana-datasource + namespace: {{ default .Values.grafana.sidecar.datasources.searchNamespace (include "kube-prometheus-stack.namespace" .) }} +{{- if .Values.grafana.sidecar.datasources.annotations }} + annotations: +{{ toYaml .Values.grafana.sidecar.datasources.annotations | indent 4 }} +{{- end }} + labels: + {{ $.Values.grafana.sidecar.datasources.label }}: {{ $.Values.grafana.sidecar.datasources.labelValue | quote }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + datasource.yaml: |- + apiVersion: 1 +{{- if .Values.grafana.deleteDatasources }} + deleteDatasources: +{{ tpl (toYaml .Values.grafana.deleteDatasources | indent 6) . }} +{{- end }} + datasources: +{{- $scrapeInterval := .Values.grafana.sidecar.datasources.defaultDatasourceScrapeInterval | default .Values.prometheus.prometheusSpec.scrapeInterval | default "30s" }} +{{- if .Values.grafana.sidecar.datasources.defaultDatasourceEnabled }} + - name: Prometheus + type: prometheus + uid: {{ .Values.grafana.sidecar.datasources.uid }} + {{- if .Values.grafana.sidecar.datasources.url }} + url: {{ .Values.grafana.sidecar.datasources.url }} + {{- else }} + url: http://{{ template "kube-prometheus-stack.fullname" . }}-prometheus.{{ template "kube-prometheus-stack.namespace" . }}:{{ .Values.prometheus.service.port }}/{{ trimPrefix "/" .Values.prometheus.prometheusSpec.routePrefix }} + {{- end }} + access: proxy + isDefault: true + jsonData: + timeInterval: {{ $scrapeInterval }} +{{- if .Values.grafana.sidecar.datasources.exemplarTraceIdDestinations }} + exemplarTraceIdDestinations: + - datasourceUid: {{ .Values.grafana.sidecar.datasources.exemplarTraceIdDestinations.datasourceUid }} + name: {{ .Values.grafana.sidecar.datasources.exemplarTraceIdDestinations.traceIdLabelName }} +{{- end }} +{{- if .Values.grafana.sidecar.datasources.createPrometheusReplicasDatasources }} +{{- range until (int .Values.prometheus.prometheusSpec.replicas) }} + - name: Prometheus-{{ . }} + type: prometheus + uid: {{ $.Values.grafana.sidecar.datasources.uid }}-replica-{{ . }} + url: http://prometheus-{{ template "kube-prometheus-stack.prometheus.crname" $ }}-{{ . }}.prometheus-operated:9090/{{ trimPrefix "/" $.Values.prometheus.prometheusSpec.routePrefix }} + access: proxy + isDefault: false + jsonData: + timeInterval: {{ $scrapeInterval }} +{{- if $.Values.grafana.sidecar.datasources.exemplarTraceIdDestinations }} + exemplarTraceIdDestinations: + - datasourceUid: {{ .Values.grafana.sidecar.datasources.exemplarTraceIdDestinations.datasourceUid }} + name: {{ .Values.grafana.sidecar.datasources.exemplarTraceIdDestinations.traceIdLabelName }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} +{{- if .Values.grafana.additionalDataSources }} +{{ tpl (toYaml .Values.grafana.additionalDataSources | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/alertmanager-overview.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/alertmanager-overview.yaml new file mode 100644 index 0000000000..3fe5bcdd02 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/alertmanager-overview.yaml @@ -0,0 +1,616 @@ +{{- /* +Generated from 'alertmanager-overview' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +{{- if and .Values.alertmanager.enabled .Values.alertmanager.serviceMonitor.selfMonitor }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "alertmanager-overview" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + alertmanager-overview.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 1, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "30s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "current set of alerts stored in the Alertmanager", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(alertmanager_alerts{namespace=~\"$namespace\",service=~\"$service\"}) by (namespace,service,instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Alerts", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "rate of successful and invalid alerts received by the Alertmanager", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(alertmanager_alerts_received_total{namespace=~\"$namespace\",service=~\"$service\"}[$__rate_interval])) by (namespace,service,instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Received", + "refId": "A" + }, + { + "expr": "sum(rate(alertmanager_alerts_invalid_total{namespace=~\"$namespace\",service=~\"$service\"}[$__rate_interval])) by (namespace,service,instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Invalid", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Alerts receive rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Alerts", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "rate of successful and invalid notifications sent by the Alertmanager", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "integration", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(alertmanager_notifications_total{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (integration,namespace,service,instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Total", + "refId": "A" + }, + { + "expr": "sum(rate(alertmanager_notifications_failed_total{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (integration,namespace,service,instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Failed", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "$integration: Notifications Send Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "latency of notifications sent by the Alertmanager", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "integration", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99,\n sum(rate(alertmanager_notification_latency_seconds_bucket{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (le,namespace,service,instance)\n) \n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} 99th Percentile", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.50,\n sum(rate(alertmanager_notification_latency_seconds_bucket{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (le,namespace,service,instance)\n) \n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Median", + "refId": "B" + }, + { + "expr": "sum(rate(alertmanager_notification_latency_seconds_sum{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (namespace,service,instance)\n/\nsum(rate(alertmanager_notification_latency_seconds_count{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (namespace,service,instance)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Average", + "refId": "C" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "$integration: Notification Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Notifications", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "alertmanager-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "namespace", + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(alertmanager_alerts, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "service", + "multi": false, + "name": "service", + "options": [ + + ], + "query": "label_values(alertmanager_alerts, service)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "all", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": true, + "label": null, + "multi": false, + "name": "integration", + "options": [ + + ], + "query": "label_values(alertmanager_notifications_total{integration=~\".*\"}, integration)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Alertmanager / Overview", + "uid": "alertmanager-overview", + "version": 0 + } +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/apiserver.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/apiserver.yaml new file mode 100644 index 0000000000..d4cf09f180 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/apiserver.yaml @@ -0,0 +1,1772 @@ +{{- /* +Generated from 'apiserver' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.kubeApiServer.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "apiserver" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + apiserver.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "content": "The SLO (service level objective) and other metrics displayed on this dashboard are for informational purposes only.", + "datasource": null, + "description": "The SLO (service level objective) and other metrics displayed on this dashboard are for informational purposes only.", + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "mode": "markdown", + "span": 12, + "title": "Notice", + "type": "text" + } + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 3, + "description": "How many percent of requests (both read and write) in 30 days have been answered successfully and fast enough?", + "format": "percentunit", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 4, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "apiserver_request:availability30d{verb=\"all\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Availability (30d) > 99.000%", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 3, + "description": "How much error budget is left looking at our 0.990% availability guarantees?", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 8, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "100 * (apiserver_request:availability30d{verb=\"all\", cluster=\"$cluster\"} - 0.990000)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "errorbudget", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "ErrorBudget (30d) > 99.000%", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "decimals": 3, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": 3, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 3, + "description": "How many percent of read requests (LIST,GET) in 30 days have been answered successfully and fast enough?", + "format": "percentunit", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "apiserver_request:availability30d{verb=\"read\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Read Availability (30d)", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "How many read requests (LIST,GET) per second do the apiservers get by code?", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/2../i", + "color": "#56A64B" + }, + { + "alias": "/3../i", + "color": "#F2CC0C" + }, + { + "alias": "/4../i", + "color": "#3274D9" + }, + { + "alias": "/5../i", + "color": "#E02F44" + } + ], + "spaceLength": 10, + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (code) (code_resource:apiserver_request_total:rate5m{verb=\"read\", cluster=\"$cluster\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} code {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Read SLI - Requests", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "How many percent of read requests (LIST,GET) per second are returned with errors (5xx)?", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (resource) (code_resource:apiserver_request_total:rate5m{verb=\"read\",code=~\"5..\", cluster=\"$cluster\"}) / sum by (resource) (code_resource:apiserver_request_total:rate5m{verb=\"read\", cluster=\"$cluster\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} resource {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Read SLI - Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "How many seconds is the 99th percentile for reading (LIST|GET) a given resource?", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "cluster_quantile:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds:histogram_quantile{verb=\"read\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} resource {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Read SLI - Duration", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 3, + "description": "How many percent of write requests (POST|PUT|PATCH|DELETE) in 30 days have been answered successfully and fast enough?", + "format": "percentunit", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "apiserver_request:availability30d{verb=\"write\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Write Availability (30d)", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "How many write requests (POST|PUT|PATCH|DELETE) per second do the apiservers get by code?", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/2../i", + "color": "#56A64B" + }, + { + "alias": "/3../i", + "color": "#F2CC0C" + }, + { + "alias": "/4../i", + "color": "#3274D9" + }, + { + "alias": "/5../i", + "color": "#E02F44" + } + ], + "spaceLength": 10, + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (code) (code_resource:apiserver_request_total:rate5m{verb=\"write\", cluster=\"$cluster\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} code {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Write SLI - Requests", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "How many percent of write requests (POST|PUT|PATCH|DELETE) per second are returned with errors (5xx)?", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (resource) (code_resource:apiserver_request_total:rate5m{verb=\"write\",code=~\"5..\", cluster=\"$cluster\"}) / sum by (resource) (code_resource:apiserver_request_total:rate5m{verb=\"write\", cluster=\"$cluster\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} resource {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Write SLI - Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "How many seconds is the 99th percentile for writing (POST|PUT|PATCH|DELETE) a given resource?", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "cluster_quantile:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds:histogram_quantile{verb=\"write\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} resource {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Write SLI - Duration", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 13, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(workqueue_adds_total{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[$__rate_interval])) by (instance, name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Add Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 14, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(workqueue_depth{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[$__rate_interval])) by (instance, name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Depth", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 15, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[$__rate_interval])) by (instance, name, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Latency", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 16, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 17, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 18, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"apiserver\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(up{job=\"apiserver\", cluster=\"$cluster\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / API server", + "uid": "09ec8aa1e996d6ffcd6817bbaff4db1b", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/cluster-total.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/cluster-total.yaml new file mode 100644 index 0000000000..c351d0c0c2 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/cluster-total.yaml @@ -0,0 +1,1882 @@ +{{- /* +Generated from 'cluster-total' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "cluster-total" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + cluster-total.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 3, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "columns": [ + { + "text": "Time", + "value": "Time" + }, + { + "text": "Value #A", + "value": "Value #A" + }, + { + "text": "Value #B", + "value": "Value #B" + }, + { + "text": "Value #C", + "value": "Value #C" + }, + { + "text": "Value #D", + "value": "Value #D" + }, + { + "text": "Value #E", + "value": "Value #E" + }, + { + "text": "Value #F", + "value": "Value #F" + }, + { + "text": "Value #G", + "value": "Value #G" + }, + { + "text": "Value #H", + "value": "Value #H" + }, + { + "text": "namespace", + "value": "namespace" + } + ], + "datasource": "$datasource", + "fill": 1, + "fontSize": "90%", + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 5, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null as zero", + "renderer": "flot", + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": false + }, + "spaceLength": 10, + "span": 24, + "styles": [ + { + "alias": "Time", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Time", + "thresholds": [ + + ], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Current Bandwidth Received", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Bandwidth Transmitted", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Average Bandwidth Received", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Average Bandwidth Transmitted", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "d/8b7a8b326d7a6f1f04244066368c67af/kubernetes-networking-namespace-pods?orgId=1&refresh=30s&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Status", + "type": "table" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 6, + "panels": [ + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 9, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth History", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 10, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 11, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 12, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 13, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 14, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Packets", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 15, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 50 + }, + "id": 16, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 59 + }, + "id": 17, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 59 + }, + "id": 18, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + { + "targetBlank": true, + "title": "What is TCP Retransmit?", + "url": "https://accedian.com/enterprises/blog/network-packet-loss-retransmissions-and-duplicate-acknowledgements/" + } + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(rate(node_netstat_Tcp_RetransSegs{cluster=\"$cluster\"}[$interval:$resolution]) / rate(node_netstat_Tcp_OutSegs{cluster=\"$cluster\"}[$interval:$resolution])) by (instance))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of TCP Retransmits out of all sent segments", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 59 + }, + "id": 19, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + { + "targetBlank": true, + "title": "Why monitor SYN retransmits?", + "url": "https://github.com/prometheus/node_exporter/issues/1023#issuecomment-408128365" + } + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(rate(node_netstat_TcpExt_TCPSynRetrans{cluster=\"$cluster\"}[$interval:$resolution]) / rate(node_netstat_Tcp_RetransSegs{cluster=\"$cluster\"}[$interval:$resolution])) by (instance))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of TCP SYN Retransmits out of all retransmits", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Errors", + "titleSize": "h6", + "type": "row" + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "resolution", + "options": [ + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "30s,5m,1h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Networking / Cluster", + "uid": "ff635a025bcfea7bc3dd4f508990a3e9", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/controller-manager.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/controller-manager.yaml new file mode 100644 index 0000000000..c1946dd8bf --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/controller-manager.yaml @@ -0,0 +1,1196 @@ +{{- /* +Generated from 'controller-manager' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +{{- if (include "exporter.kubeControllerManager.enabled" .)}} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "controller-manager" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + controller-manager.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + {{- if .Values.k3sServer.enabled }} + "expr": "sum(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", metrics_path=\"/metrics\"})", + {{- else }} + "expr": "sum(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\"})", + {{- end }} + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Up", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "min" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(workqueue_adds_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Add Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(workqueue_depth{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Depth", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, name, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Latency", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rest_client_requests_total{job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\",code=~\"2..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "2xx", + "refId": "A" + }, + { + "expr": "sum(rate(rest_client_requests_total{job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\",code=~\"3..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "3xx", + "refId": "B" + }, + { + "expr": "sum(rate(rest_client_requests_total{job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\",code=~\"4..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "4xx", + "refId": "C" + }, + { + "expr": "sum(rate(rest_client_requests_total{job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\",code=~\"5..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5xx", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Kube API Request Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 8, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\", verb=\"POST\"}[$__rate_interval])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Post Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\", verb=\"GET\"}[$__rate_interval])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Get Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\",instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Controller Manager", + "uid": "72e0e05bef5099e5f049b05fdc429ed4", + "version": 0 + } +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/etcd.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/etcd.yaml new file mode 100644 index 0000000000..3956638cbe --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/etcd.yaml @@ -0,0 +1,1229 @@ +{{- /* +Generated from 'etcd' from https://raw.githubusercontent.com/etcd-io/etcd/main/contrib/mixin/mixin.libsonnet +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +{{- if (include "exporter.kubeEtcd.enabled" .)}} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "etcd" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + etcd.json: |- + { + "annotations": { + "list": [] + }, + "description": "etcd sample Grafana dashboard with Prometheus", + "editable": true, + "gnetId": null, + "hideControls": false, + "links": [], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "id": 28, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [ + { + "expr": "sum(etcd_server_has_leader{job=\"$cluster\"})", + "intervalFactor": 2, + "legendFormat": "", + "metric": "etcd_server_has_leader", + "refId": "A", + "step": 20 + } + ], + "thresholds": "", + "title": "Up", + "type": "singlestat", + "valueFontSize": "200%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 23, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_started_total{job=\"$cluster\",grpc_type=\"unary\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "RPC Rate", + "metric": "grpc_server_started_total", + "refId": "A", + "step": 2 + }, + { + "expr": "sum(rate(grpc_server_handled_total{job=\"$cluster\",grpc_type=\"unary\",grpc_code=~\"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "RPC Failed Rate", + "metric": "grpc_server_handled_total", + "refId": "B", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "RPC Rate", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 41, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(grpc_server_started_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"})", + "intervalFactor": 2, + "legendFormat": "Watch Streams", + "metric": "grpc_server_handled_total", + "refId": "A", + "step": 4 + }, + { + "expr": "sum(grpc_server_started_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"})", + "intervalFactor": 2, + "legendFormat": "Lease Streams", + "metric": "grpc_server_handled_total", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Active Streams", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "showTitle": false, + "title": "Row" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "decimals": null, + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "etcd_mvcc_db_total_size_in_bytes{job=\"$cluster\"}", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} DB Size", + "metric": "", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "DB Size", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": true, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=\"$cluster\"}[$__rate_interval])) by (instance, le))", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} WAL fsync", + "metric": "etcd_disk_wal_fsync_duration_seconds_bucket", + "refId": "A", + "step": 4 + }, + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket{job=\"$cluster\"}[$__rate_interval])) by (instance, le))", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} DB fsync", + "metric": "etcd_disk_backend_commit_duration_seconds_bucket", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk Sync Duration", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 29, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{job=\"$cluster\"}", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Resident Memory", + "metric": "process_resident_memory_bytes", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": "New row" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 5, + "id": 22, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(etcd_network_client_grpc_received_bytes_total{job=\"$cluster\"}[$__rate_interval])", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Client Traffic In", + "metric": "etcd_network_client_grpc_received_bytes_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Client Traffic In", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 5, + "id": 21, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(etcd_network_client_grpc_sent_bytes_total{job=\"$cluster\"}[$__rate_interval])", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Client Traffic Out", + "metric": "etcd_network_client_grpc_sent_bytes_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Client Traffic Out", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 20, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_network_peer_received_bytes_total{job=\"$cluster\"}[$__rate_interval])) by (instance)", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Peer Traffic In", + "metric": "etcd_network_peer_received_bytes_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Peer Traffic In", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "decimals": null, + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_network_peer_sent_bytes_total{job=\"$cluster\"}[$__rate_interval])) by (instance)", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Peer Traffic Out", + "metric": "etcd_network_peer_sent_bytes_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Peer Traffic Out", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": "New row" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 40, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_server_proposals_failed_total{job=\"$cluster\"}[$__rate_interval]))", + "intervalFactor": 2, + "legendFormat": "Proposal Failure Rate", + "metric": "etcd_server_proposals_failed_total", + "refId": "A", + "step": 2 + }, + { + "expr": "sum(etcd_server_proposals_pending{job=\"$cluster\"})", + "intervalFactor": 2, + "legendFormat": "Proposal Pending Total", + "metric": "etcd_server_proposals_pending", + "refId": "B", + "step": 2 + }, + { + "expr": "sum(rate(etcd_server_proposals_committed_total{job=\"$cluster\"}[$__rate_interval]))", + "intervalFactor": 2, + "legendFormat": "Proposal Commit Rate", + "metric": "etcd_server_proposals_committed_total", + "refId": "C", + "step": 2 + }, + { + "expr": "sum(rate(etcd_server_proposals_applied_total{job=\"$cluster\"}[$__rate_interval]))", + "intervalFactor": 2, + "legendFormat": "Proposal Apply Rate", + "refId": "D", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Raft Proposals", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "decimals": 0, + "editable": true, + "error": false, + "fill": 0, + "id": 19, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "changes(etcd_server_leader_changes_seen_total{job=\"$cluster\"}[1d])", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Total Leader Elections Per Day", + "metric": "etcd_server_leader_changes_seen_total", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Total Leader Elections Per Day", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 0, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 28 + }, + "hiddenSeries": false, + "id": 42, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum by (instance, le) (rate(etcd_network_peer_round_trip_time_seconds_bucket{job=\"$cluster\"}[$__rate_interval])))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Peer round trip time", + "metric": "etcd_network_peer_round_trip_time_seconds_bucket", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Peer round trip time", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:925", + "decimals": null, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:926", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "title": "New row" + } + ], + "schemaVersion": 13, + "sharedCrosshair": false, + "style": "dark", + "tags": [ + "etcd-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": {{ if (or .Values.grafana.sidecar.dashboards.multicluster.global.enabled .Values.grafana.sidecar.dashboards.multicluster.etcd.enabled) }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [], + "query": "label_values(etcd_server_has_leader, job)", + "refresh": 2, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "etcd", + "uid": "c2f4e12cdf69feb95caa41a5a1b423d9", + "version": 215 + } +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/grafana-overview.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/grafana-overview.yaml new file mode 100644 index 0000000000..8d08b055f5 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/grafana-overview.yaml @@ -0,0 +1,635 @@ +{{- /* +Generated from 'grafana-overview' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "kube-prometheus-stack-grafana.namespace" . }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "grafana-overview" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + grafana-overview.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [ + + ], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 3085, + "iteration": 1631554945276, + "links": [ + + ], + "panels": [ + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "mappings": [ + + ], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "text": { + + }, + "textMode": "auto" + }, + "pluginVersion": "8.1.3", + "targets": [ + { + "expr": "grafana_alerting_result_total{job=~\"$job\", instance=~\"$instance\", state=\"alerting\"}", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Firing Alerts", + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 0 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "text": { + + }, + "textMode": "auto" + }, + "pluginVersion": "8.1.3", + "targets": [ + { + "expr": "sum(grafana_stat_totals_dashboard{job=~\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Dashboards", + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "align": null, + "displayMode": "auto" + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 10, + "options": { + "showHeader": true + }, + "pluginVersion": "8.1.3", + "targets": [ + { + "expr": "grafana_build_info{job=~\"$job\", instance=~\"$instance\"}", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Build Info", + "transformations": [ + { + "id": "labelsToFields", + "options": { + + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": true, + "branch": true, + "container": true, + "goversion": true, + "namespace": true, + "pod": true, + "revision": true + }, + "indexByName": { + "Time": 7, + "Value": 11, + "branch": 4, + "container": 8, + "edition": 2, + "goversion": 6, + "instance": 1, + "job": 0, + "namespace": 9, + "pod": 10, + "revision": 5, + "version": 3 + }, + "renameByName": { + + } + } + } + ], + "type": "table" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ] + }, + "overrides": [ + + ] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 5 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.1.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (status_code) (irate(grafana_http_request_duration_seconds_count{job=~\"$job\", instance=~\"$instance\"}[1m])) ", + "interval": "", + "legendFormat": "{{`{{`}}status_code{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeRegions": [ + + ], + "timeShift": null, + "title": "RPS", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "$$hashKey": "object:157", + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:158", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ] + }, + "overrides": [ + + ] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 5 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.1.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(irate(grafana_http_request_duration_seconds_bucket{instance=~\"$instance\", job=~\"$job\"}[$__rate_interval])) by (le)) * 1", + "interval": "", + "legendFormat": "99th Percentile", + "refId": "A" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.50, sum(irate(grafana_http_request_duration_seconds_bucket{instance=~\"$instance\", job=~\"$job\"}[$__rate_interval])) by (le)) * 1", + "interval": "", + "legendFormat": "50th Percentile", + "refId": "B" + }, + { + "exemplar": true, + "expr": "sum(irate(grafana_http_request_duration_seconds_sum{instance=~\"$instance\", job=~\"$job\"}[$__rate_interval])) * 1 / sum(irate(grafana_http_request_duration_seconds_count{instance=~\"$instance\", job=~\"$job\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Average", + "refId": "C" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeRegions": [ + + ], + "timeShift": null, + "title": "Request Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "$$hashKey": "object:210", + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:211", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 30, + "style": "dark", + "tags": [ + + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "description": null, + "error": null, + "hide": 0, + "includeAll": false, + "label": "Data Source", + "multi": false, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "selected": false, + "text": [ + "default/grafana" + ], + "value": [ + "default/grafana" + ] + }, + "datasource": "$datasource", + "definition": "label_values(grafana_build_info, job)", + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "name": "job", + "options": [ + + ], + "query": { + "query": "label_values(grafana_build_info, job)", + "refId": "Billing Admin-job-Variable-Query" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "definition": "label_values(grafana_build_info, instance)", + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "name": "instance", + "options": [ + + ], + "query": { + "query": "label_values(grafana_build_info, instance)", + "refId": "Billing Admin-instance-Variable-Query" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Grafana Overview", + "uid": "6be0s85Mk", + "version": 2 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/k8s-coredns.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/k8s-coredns.yaml new file mode 100644 index 0000000000..5d5840e024 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/k8s-coredns.yaml @@ -0,0 +1,1530 @@ +{{- /* Added manually, can be changed in-place. */ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.coreDns.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-coredns" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-coredns.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "A dashboard for the CoreDNS DNS server with updated metrics for version 1.7.0+. Based on the CoreDNS dashboard by buhay.", + "editable": true, + "gnetId": 12539, + "graphTooltip": 0, + "iteration": 1603798405693, + "links": [ + { + "icon": "external link", + "tags": [], + "targetBlank": true, + "title": "CoreDNS.io", + "type": "link", + "url": "https://coredns.io" + } + ], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "total", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_request_count_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (proto) or\nsum(rate(coredns_dns_requests_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (proto)", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}", + "refId": "A", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests (total)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "total", + "yaxis": 2 + }, + { + "alias": "other", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_request_type_count_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (type) or \nsum(rate(coredns_dns_requests_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (type)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{type}}"}}", + "refId": "A", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests (by qtype)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "total", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_request_count_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (zone) or\nsum(rate(coredns_dns_requests_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (zone)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{zone}}"}}", + "refId": "A", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests (by zone)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "total", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_request_do_count_total{job=\"coredns\",instance=~\"$instance\"}[5m])) or\nsum(rate(coredns_dns_do_requests_total{job=\"coredns\",instance=~\"$instance\"}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "DO", + "refId": "A", + "step": 40 + }, + { + "expr": "sum(rate(coredns_dns_request_count_total{job=\"coredns\",instance=~\"$instance\"}[5m])) or\nsum(rate(coredns_dns_requests_total{job=\"coredns\",instance=~\"$instance\"}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "total", + "refId": "B", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests (DO bit)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 7 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "tcp:90", + "yaxis": 2 + }, + { + "alias": "tcp:99 ", + "yaxis": 2 + }, + { + "alias": "tcp:50", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_request_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:99 ", + "refId": "A", + "step": 60 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_request_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto))", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:90", + "refId": "B", + "step": 60 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_request_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto))", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:50", + "refId": "C", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests (size, udp)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 7 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "tcp:90", + "yaxis": 1 + }, + { + "alias": "tcp:99 ", + "yaxis": 1 + }, + { + "alias": "tcp:50", + "yaxis": 1 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_request_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto))", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:99 ", + "refId": "A", + "step": 60 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_request_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto))", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:90", + "refId": "B", + "step": 60 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_request_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto))", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:50", + "refId": "C", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests (size,tcp)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_response_rcode_count_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (rcode) or\nsum(rate(coredns_dns_responses_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (rcode)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{rcode}}"}}", + "refId": "A", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Responses (by rcode)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 14 + }, + "hiddenSeries": false, + "id": 32, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_request_duration_seconds_bucket{job=\"coredns\",instance=~\"$instance\"}[5m])) by (le, job))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "99%", + "refId": "A", + "step": 40 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_request_duration_seconds_bucket{job=\"coredns\",instance=~\"$instance\"}[5m])) by (le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "90%", + "refId": "B", + "step": 40 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_request_duration_seconds_bucket{job=\"coredns\",instance=~\"$instance\"}[5m])) by (le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "50%", + "refId": "C", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Responses (duration)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 21 + }, + "hiddenSeries": false, + "id": 18, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "udp:50%", + "yaxis": 1 + }, + { + "alias": "tcp:50%", + "yaxis": 2 + }, + { + "alias": "tcp:90%", + "yaxis": 2 + }, + { + "alias": "tcp:99%", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_response_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto)) ", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:99%", + "refId": "A", + "step": 40 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_response_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto)) ", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:90%", + "refId": "B", + "step": 40 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_response_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto)) ", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:50%", + "metric": "", + "refId": "C", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Responses (size, udp)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 21 + }, + "hiddenSeries": false, + "id": 20, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "udp:50%", + "yaxis": 1 + }, + { + "alias": "tcp:50%", + "yaxis": 1 + }, + { + "alias": "tcp:90%", + "yaxis": 1 + }, + { + "alias": "tcp:99%", + "yaxis": 1 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_response_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto)) ", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:99%", + "refId": "A", + "step": 40 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_response_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto)) ", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:90%", + "refId": "B", + "step": 40 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_response_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le, proto)) ", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:50%", + "metric": "", + "refId": "C", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Responses (size, tcp)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 28 + }, + "hiddenSeries": false, + "id": 22, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(coredns_cache_size{job=\"coredns\",instance=~\"$instance\"}) by (type) or\nsum(coredns_cache_entries{job=\"coredns\",instance=~\"$instance\"}) by (type)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{type}}"}}", + "refId": "A", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cache (size)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 28 + }, + "hiddenSeries": false, + "id": 24, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "misses", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_cache_hits_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (type)", + "hide": false, + "intervalFactor": 2, + "legendFormat": "hits:{{"{{type}}"}}", + "refId": "A", + "step": 40 + }, + { + "expr": "sum(rate(coredns_cache_misses_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (type)", + "hide": false, + "intervalFactor": 2, + "legendFormat": "misses", + "refId": "B", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cache (hitrate)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "10s", + "schemaVersion": 26, + "style": "dark", + "tags": [ + "dns", + "coredns" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Data Source", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "definition": "label_values(up{job=\"coredns\"}, instance)", + "hide": 0, + "includeAll": true, + "label": "Instance", + "multi": false, + "name": "instance", + "options": [], + "query": "label_values(up{job=\"coredns\"}, instance)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 3, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "CoreDNS", + "uid": "vkQ0UHxik", + "version": 2 + } +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/k8s-resources-cluster.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/k8s-resources-cluster.yaml new file mode 100644 index 0000000000..581c4779a6 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/k8s-resources-cluster.yaml @@ -0,0 +1,3088 @@ +{{- /* +Generated from 'k8s-resources-cluster' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-cluster" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-cluster.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "100px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 1, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "cluster:node_cpu:ratio_rate5m{cluster=\"$cluster\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(namespace_cpu:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"cpu\",cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Requests Commitment", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(namespace_cpu:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"cpu\",cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Limits Commitment", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - sum(:node_memory_MemAvailable_bytes:sum{cluster=\"$cluster\"}) / sum(node_memory_MemTotal_bytes{job=\"node-exporter\",cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(namespace_memory:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"memory\",cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Requests Commitment", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(namespace_memory:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"memory\",cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Limits Commitment", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Headlines", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\"}) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Pods", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workloads", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to workloads", + "linkUrl": "d/a87fb0d919ec0ea5f6543124e16c42a5/k8s-resources-workloads-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(kube_pod_owner{job=\"kube-state-metrics\", cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "count(avg(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\"}) by (workload, namespace)) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(namespace_cpu:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\"}) by (namespace) / sum(namespace_cpu:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(namespace_cpu:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\"}) by (namespace) / sum(namespace_cpu:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_rss{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", container!=\"\"}) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage (w/o cache)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Pods", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workloads", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to workloads", + "linkUrl": "d/a87fb0d919ec0ea5f6543124e16c42a5/k8s-resources-workloads-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(kube_pod_owner{job=\"kube-state-metrics\", cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "count(avg(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\"}) by (workload, namespace)) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", container!=\"\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(namespace_memory:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", container!=\"\"}) by (namespace) / sum(namespace_memory:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(namespace_memory:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", container!=\"\"}) by (namespace) / sum(namespace_memory:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Requests by Namespace", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Requests", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Current Receive Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Transmit Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Network Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Network Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 14, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Namespace: Received", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 15, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Namespace: Transmitted", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Container Bandwidth by Namespace", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 16, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 17, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 18, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 19, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets Dropped", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": -1, + "fill": 10, + "id": 20, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ceil(sum by(namespace) (rate(container_fs_reads_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "IOPS(Reads+Writes)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 21, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(namespace) (rate(container_fs_reads_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "ThroughPut(Read+Write)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 22, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "sort": { + "col": 4, + "desc": true + }, + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "IOPS(Reads)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Reads + Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Throughput(Read)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Read + Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum by(namespace) (rate(container_fs_reads_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum by(namespace) (rate(container_fs_writes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum by(namespace) (rate(container_fs_reads_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum by(namespace) (rate(container_fs_reads_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum by(namespace) (rate(container_fs_writes_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum by(namespace) (rate(container_fs_reads_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Storage IO", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO - Distribution", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Compute Resources / Cluster", + "uid": "efa86fd1d0c121a26444b636a3f509a8", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/k8s-resources-namespace.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/k8s-resources-namespace.yaml new file mode 100644 index 0000000000..0c9c805fdf --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/k8s-resources-namespace.yaml @@ -0,0 +1,2797 @@ +{{- /* +Generated from 'k8s-resources-namespace' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-namespace" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-namespace.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "100px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 1, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) / sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation (from requests)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation (from limits)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) / sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation (from requests)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation (from limits)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Headlines", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "quota - requests", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "quota - limits", + "color": "#FF9830", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"requests.cpu\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"limits.cpu\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "quota - requests", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "quota - limits", + "color": "#FF9830", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"requests.memory\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"limits.memory\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage (w/o cache)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Usage (RSS)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Cache)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Swap)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(container_memory_cache{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sum(container_memory_swap{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Current Receive Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Transmit Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Network Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Network Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 14, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 15, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets Dropped", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": -1, + "fill": 10, + "id": 16, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ceil(sum by(pod) (rate(container_fs_reads_total{container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_total{container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "IOPS(Reads+Writes)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 17, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "ThroughPut(Read+Write)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 18, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "sort": { + "col": 4, + "desc": true + }, + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "IOPS(Reads)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Reads + Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Throughput(Read)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Read + Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum by(pod) (rate(container_fs_reads_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_writes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_reads_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_writes_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Storage IO", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO - Distribution", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"kube-state-metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_namespace_status_phase{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Compute Resources / Namespace (Pods)", + "uid": "85a562078cdf77779eaa1add43ccec1e", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/k8s-resources-node.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/k8s-resources-node.yaml new file mode 100644 index 0000000000..fc10e7f2d3 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/k8s-resources-node.yaml @@ -0,0 +1,1026 @@ +{{- /* +Generated from 'k8s-resources-node' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-node" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-node.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "max capacity", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_node_status_capacity{cluster=\"$cluster\", node=~\"$node\", resource=\"cpu\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "max capacity", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", node=~\"$node\"}) by (pod) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", node=~\"$node\"}) by (pod) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "max capacity", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_node_status_capacity{cluster=\"$cluster\", node=~\"$node\", resource=\"memory\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "max capacity", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\", container!=\"\"}) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage (w/o cache)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Usage (RSS)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Cache)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Swap)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_rss{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_cache{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_swap{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"kube-state-metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": true, + "name": "node", + "options": [ + + ], + "query": "label_values(kube_node_info{cluster=\"$cluster\"}, node)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Compute Resources / Node (Pods)", + "uid": "200ac8fdbfbb74b39aff88118e4d1c2c", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/k8s-resources-pod.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/k8s-resources-pod.yaml new file mode 100644 index 0000000000..881485e61e --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/k8s-resources-pod.yaml @@ -0,0 +1,2469 @@ +{{- /* +Generated from 'k8s-resources-pod' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-pod" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-pod.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "requests", + "color": "#F2495C", + "fill": 0, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "limits", + "color": "#FF9830", + "fill": 0, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{namespace=\"$namespace\", pod=\"$pod\", cluster=\"$cluster\"}) by (container)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}container{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(increase(container_cpu_cfs_throttled_periods_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", cluster=\"$cluster\"}[$__rate_interval])) by (container) /sum(increase(container_cpu_cfs_periods_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", cluster=\"$cluster\"}[$__rate_interval])) by (container)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}container{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 0.25, + "yaxis": "left" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Throttling", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Throttling", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Container", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "container", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "requests", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "limits", + "color": "#FF9830", + "dashes": true, + "fill": 0, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", image!=\"\"}) by (container)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}container{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"memory\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"memory\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage (WSS)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Memory Usage (WSS)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Usage (RSS)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Cache)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Swap)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Container", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "container", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", image!=\"\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", image!=\"\"}) by (container) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", image!=\"\"}) by (container) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(container_memory_cache{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sum(container_memory_swap{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets Dropped", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": -1, + "fill": 10, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ceil(sum by(pod) (rate(container_fs_reads_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Reads", + "legendLink": null, + "step": 10 + }, + { + "expr": "ceil(sum by(pod) (rate(container_fs_writes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)\", container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Writes", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "IOPS", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Reads", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_writes_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Writes", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "ThroughPut", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO - Distribution(Pod - Read & Writes)", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": -1, + "fill": 10, + "id": 14, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ceil(sum by(container) (rate(container_fs_reads_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}container{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "IOPS(Reads+Writes)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 15, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(container) (rate(container_fs_reads_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}container{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "ThroughPut(Read+Write)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO - Distribution(Containers)", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 16, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "sort": { + "col": 4, + "desc": true + }, + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "IOPS(Reads)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Reads + Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Throughput(Read)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Read + Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Container", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "container", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum by(container) (rate(container_fs_reads_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum by(container) (rate(container_fs_writes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\",device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum by(container) (rate(container_fs_reads_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum by(container) (rate(container_fs_reads_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum by(container) (rate(container_fs_writes_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum by(container) (rate(container_fs_reads_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Storage IO", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO - Distribution", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"kube-state-metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_namespace_status_phase{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "pod", + "options": [ + + ], + "query": "label_values(kube_pod_info{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\"}, pod)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Compute Resources / Pod", + "uid": "6581e46e4e5c7ba40a07646395ef7b23", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/k8s-resources-workload.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/k8s-resources-workload.yaml new file mode 100644 index 0000000000..7b19154c52 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/k8s-resources-workload.yaml @@ -0,0 +1,2024 @@ +{{- /* +Generated from 'k8s-resources-workload' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-workload" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-workload.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Current Receive Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Transmit Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "(sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Network Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Network Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(avg(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Pod: Received", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(avg(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Pod: Transmitted", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Container Bandwidth by Pod", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets Dropped", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"kube-state-metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_namespace_status_phase{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "type", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\"}, workload_type)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "workload", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}, workload)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Compute Resources / Workload", + "uid": "a164a7f0339f99e89cea5cb47e9be617", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/k8s-resources-workloads-namespace.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/k8s-resources-workloads-namespace.yaml new file mode 100644 index 0000000000..c4a15829d6 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/k8s-resources-workloads-namespace.yaml @@ -0,0 +1,2189 @@ +{{- /* +Generated from 'k8s-resources-workloads-namespace' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-workloads-namespace" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-workloads-namespace.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "quota - requests", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "quota - limits", + "color": "#FF9830", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}} - {{`{{`}}workload_type{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"requests.cpu\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"limits.cpu\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Running Pods", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Workload", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/a164a7f0339f99e89cea5cb47e9be617/k8s-resources-workload?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-workload=$__cell&var-type=$__cell_2", + "pattern": "workload", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workload Type", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "workload_type", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "count(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload, workload_type)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "quota - requests", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "quota - limits", + "color": "#FF9830", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}} - {{`{{`}}workload_type{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"requests.memory\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"limits.memory\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Running Pods", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Workload", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/a164a7f0339f99e89cea5cb47e9be617/k8s-resources-workload?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-workload=$__cell&var-type=$__cell_2", + "pattern": "workload", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workload Type", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "workload_type", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "count(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload, workload_type)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Current Receive Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Transmit Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Workload", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/a164a7f0339f99e89cea5cb47e9be617/k8s-resources-workload?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-workload=$__cell&var-type=$type", + "pattern": "workload", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workload Type", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "workload_type", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "(sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Network Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Network Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(avg(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Workload: Received", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(avg(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Workload: Transmitted", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Container Bandwidth by Workload", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets Dropped", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"kube-state-metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_pod_info{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "deployment", + "value": "deployment" + }, + "datasource": "$datasource", + "definition": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\"}, workload_type)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "type", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\"}, workload_type)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Compute Resources / Namespace (Workloads)", + "uid": "a87fb0d919ec0ea5f6543124e16c42a5", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/kubelet.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/kubelet.yaml new file mode 100644 index 0000000000..11c0934a39 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/kubelet.yaml @@ -0,0 +1,2256 @@ +{{- /* +Generated from 'kubelet' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +{{- if (include "exporter.kubelet.enabled" .) }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "kubelet" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + kubelet.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 0, + "y": 0 + }, + "id": 2, + "links": [ + + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(kubelet_node_name{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Running Kubelets", + "transparent": false, + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 4, + "y": 0 + }, + "id": 3, + "links": [ + + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(kubelet_running_pods{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}) OR sum(kubelet_running_pod_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "title": "Running Pods", + "transparent": false, + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 8, + "y": 0 + }, + "id": 4, + "links": [ + + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(kubelet_running_containers{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}) OR sum(kubelet_running_container_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "title": "Running Containers", + "transparent": false, + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 12, + "y": 0 + }, + "id": 5, + "links": [ + + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(volume_manager_total_volumes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\", state=\"actual_state_of_world\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "title": "Actual Volume Count", + "transparent": false, + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 16, + "y": 0 + }, + "id": 6, + "links": [ + + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(volume_manager_total_volumes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\",state=\"desired_state_of_world\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "title": "Desired Volume Count", + "transparent": false, + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 20, + "y": 0 + }, + "id": 7, + "links": [ + + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(rate(kubelet_node_config_error{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "title": "Config Error Count", + "transparent": false, + "type": "stat" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_runtime_operations_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (operation_type, instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_type{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Operation Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 9, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_runtime_operations_errors_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_type)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_type{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Operation Error Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 10, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_runtime_operations_duration_seconds_bucket{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_type, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_type{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Operation duration 99th quantile", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 11, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_pod_start_duration_seconds_count{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} pod", + "refId": "A" + }, + { + "expr": "sum(rate(kubelet_pod_worker_duration_seconds_count{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} worker", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Pod Start Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 21 + }, + "id": 12, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pod_start_duration_seconds_bucket{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} pod", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} worker", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Pod Start Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 28 + }, + "id": 13, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(storage_operation_duration_seconds_count{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_name, volume_plugin)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_name{{`}}`}} {{`{{`}}volume_plugin{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Storage Operation Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 28 + }, + "id": 14, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(storage_operation_errors_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_name, volume_plugin)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_name{{`}}`}} {{`{{`}}volume_plugin{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Storage Operation Error Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 35 + }, + "id": 15, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(storage_operation_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_name, volume_plugin, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_name{{`}}`}} {{`{{`}}volume_plugin{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Storage Operation Duration 99th quantile", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 42 + }, + "id": 16, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_cgroup_manager_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_type)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}operation_type{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Cgroup manager operation rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 42 + }, + "id": 17, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_cgroup_manager_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_type, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_type{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Cgroup manager 99th quantile", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Pod lifecycle event generator", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 49 + }, + "id": 18, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_pleg_relist_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "PLEG relist rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 49 + }, + "id": 19, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_interval_seconds_bucket{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "PLEG relist interval", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 56 + }, + "id": 20, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "PLEG relist duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 63 + }, + "id": 21, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\",code=~\"2..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "2xx", + "refId": "A" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\",code=~\"3..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "3xx", + "refId": "B" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\",code=~\"4..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "4xx", + "refId": "C" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\",code=~\"5..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5xx", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "RPC Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 70 + }, + "id": 22, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval])) by (instance, verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Request duration 99th quantile", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 77 + }, + "id": 23, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 77 + }, + "id": 24, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 77 + }, + "id": 25, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "instance", + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",cluster=\"$cluster\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Kubelet", + "uid": "3138fa155d5915769fbded898ac09fd9", + "version": 0 + } +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/namespace-by-pod.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/namespace-by-pod.yaml new file mode 100644 index 0000000000..e84fcae946 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/namespace-by-pod.yaml @@ -0,0 +1,1464 @@ +{{- /* +Generated from 'namespace-by-pod' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "namespace-by-pod" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + namespace-by-pod.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "time_series", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "height": 9, + "id": 3, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 12, + "nullPointMode": "connected", + "nullText": null, + "options": { + "fieldOptions": { + "calcs": [ + "last" + ], + "defaults": { + "max": 10000000000, + "min": 0, + "title": "$namespace", + "unit": "Bps" + }, + "mappings": [ + + ], + "override": { + + }, + "thresholds": [ + { + "color": "dark-green", + "index": 0, + "value": null + }, + { + "color": "dark-yellow", + "index": 1, + "value": 5000000000 + }, + { + "color": "dark-red", + "index": 2, + "value": 7000000000 + } + ], + "values": false + } + }, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 12, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution]))", + "format": "time_series", + "instant": null, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Received", + "type": "gauge", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "time_series", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "height": 9, + "id": 4, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 12, + "nullPointMode": "connected", + "nullText": null, + "options": { + "fieldOptions": { + "calcs": [ + "last" + ], + "defaults": { + "max": 10000000000, + "min": 0, + "title": "$namespace", + "unit": "Bps" + }, + "mappings": [ + + ], + "override": { + + }, + "thresholds": [ + { + "color": "dark-green", + "index": 0, + "value": null + }, + { + "color": "dark-yellow", + "index": 1, + "value": 5000000000 + }, + { + "color": "dark-red", + "index": 2, + "value": 7000000000 + } + ], + "values": false + } + }, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 12, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution]))", + "format": "time_series", + "instant": null, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Transmitted", + "type": "gauge", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "columns": [ + { + "text": "Time", + "value": "Time" + }, + { + "text": "Value #A", + "value": "Value #A" + }, + { + "text": "Value #B", + "value": "Value #B" + }, + { + "text": "Value #C", + "value": "Value #C" + }, + { + "text": "Value #D", + "value": "Value #D" + }, + { + "text": "Value #E", + "value": "Value #E" + }, + { + "text": "Value #F", + "value": "Value #F" + }, + { + "text": "pod", + "value": "pod" + } + ], + "datasource": "$datasource", + "fill": 1, + "fontSize": "100%", + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 5, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null as zero", + "renderer": "flot", + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": false + }, + "spaceLength": 10, + "span": 24, + "styles": [ + { + "alias": "Time", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Time", + "thresholds": [ + + ], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Bandwidth Received", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Bandwidth Transmitted", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "d/7a18067ce943a40ae25454675c19ff5c/kubernetes-networking-pod?orgId=1&refresh=30s&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Status", + "type": "table" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 6, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 20 + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 20 + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 9, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 30 + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 30 + }, + "id": 11, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Packets", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 12, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 14, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Errors", + "titleSize": "h6", + "type": "row" + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".+", + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "kube-system", + "value": "kube-system" + }, + "datasource": "$datasource", + "definition": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "resolution", + "options": [ + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "30s,5m,1h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Networking / Namespace (Pods)", + "uid": "8b7a8b326d7a6f1f04244066368c67af", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/namespace-by-workload.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/namespace-by-workload.yaml new file mode 100644 index 0000000000..5490fe7f02 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/namespace-by-workload.yaml @@ -0,0 +1,1736 @@ +{{- /* +Generated from 'namespace-by-workload' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "namespace-by-workload" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + namespace-by-workload.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 3, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} workload {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} workload {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "columns": [ + { + "text": "Time", + "value": "Time" + }, + { + "text": "Value #A", + "value": "Value #A" + }, + { + "text": "Value #B", + "value": "Value #B" + }, + { + "text": "Value #C", + "value": "Value #C" + }, + { + "text": "Value #D", + "value": "Value #D" + }, + { + "text": "Value #E", + "value": "Value #E" + }, + { + "text": "Value #F", + "value": "Value #F" + }, + { + "text": "Value #G", + "value": "Value #G" + }, + { + "text": "Value #H", + "value": "Value #H" + }, + { + "text": "workload", + "value": "workload" + } + ], + "datasource": "$datasource", + "fill": 1, + "fontSize": "90%", + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 5, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null as zero", + "renderer": "flot", + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": false + }, + "spaceLength": 10, + "span": 24, + "styles": [ + { + "alias": "Time", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Time", + "thresholds": [ + + ], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Current Bandwidth Received", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Bandwidth Transmitted", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Average Bandwidth Received", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Average Bandwidth Transmitted", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Workload", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "d/728bf77cc1166d2f3133bf25846876cc/kubernetes-networking-workload?orgId=1&refresh=30s&var-namespace=$namespace&var-type=$type&var-workload=$__cell", + "pattern": "workload", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Status", + "type": "table" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 6, + "panels": [ + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 20 + }, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} workload {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 20 + }, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} workload {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 9, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth HIstory", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 38 + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 38 + }, + "id": 11, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 39 + }, + "id": 12, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 14, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Packets", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 15, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 41 + }, + "id": 16, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 41 + }, + "id": 17, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Errors", + "titleSize": "h6", + "type": "row" + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "kube-system", + "value": "kube-system" + }, + "datasource": "$datasource", + "definition": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "deployment", + "value": "deployment" + }, + "datasource": "$datasource", + "definition": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\"}, workload_type)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "type", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\"}, workload_type)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "resolution", + "options": [ + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "30s,5m,1h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Networking / Namespace (Workload)", + "uid": "bbb2a765a623ae38130206c7d94a160f", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/node-cluster-rsrc-use.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/node-cluster-rsrc-use.yaml new file mode 100644 index 0000000000..ad688a398d --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/node-cluster-rsrc-use.yaml @@ -0,0 +1,1063 @@ +{{- /* +Generated from 'node-cluster-rsrc-use' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.nodeExporter.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "node-cluster-rsrc-use" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + node-cluster-rsrc-use.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 1, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "30s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "((\n instance:node_cpu_utilisation:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}\n *\n instance:node_num_cpu:sum{job=\"node-exporter\", cluster=\"$cluster\"}\n) != 0 )\n/ scalar(sum(instance:node_num_cpu:sum{job=\"node-exporter\", cluster=\"$cluster\"}))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} instance {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n instance:node_load1_per_cpu:ratio{job=\"node-exporter\", cluster=\"$cluster\"}\n / scalar(count(instance:node_load1_per_cpu:ratio{job=\"node-exporter\", cluster=\"$cluster\"}))\n) != 0\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Saturation (Load1 per CPU)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n instance:node_memory_utilisation:ratio{job=\"node-exporter\", cluster=\"$cluster\"}\n / scalar(count(instance:node_memory_utilisation:ratio{job=\"node-exporter\", cluster=\"$cluster\"}))\n) != 0\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_vmstat_pgmajfault:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Saturation (Major Page Faults)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "rds", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "rds", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/Receive/", + "stack": "A" + }, + { + "alias": "/Transmit/", + "stack": "B", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_network_receive_bytes_excluding_lo:rate5m{job=\"node-exporter\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Receive", + "refId": "A" + }, + { + "expr": "instance:node_network_transmit_bytes_excluding_lo:rate5m{job=\"node-exporter\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Transmit", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Utilisation (Bytes Receive/Transmit)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/ Receive/", + "stack": "A" + }, + { + "alias": "/ Transmit/", + "stack": "B", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_network_receive_drop_excluding_lo:rate5m{job=\"node-exporter\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Receive", + "refId": "A" + }, + { + "expr": "instance:node_network_transmit_drop_excluding_lo:rate5m{job=\"node-exporter\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Transmit", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Saturation (Drops Receive/Transmit)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n instance_device:node_disk_io_time_seconds:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}\n / scalar(count(instance_device:node_disk_io_time_seconds:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}))\n) != 0\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk IO Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n instance_device:node_disk_io_time_weighted_seconds:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}\n / scalar(count(instance_device:node_disk_io_time_weighted_seconds:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}))\n) != 0\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk IO Saturation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk IO", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum without (device) (\n max without (fstype, mountpoint) ((\n node_filesystem_size_bytes{job=\"node-exporter\", fstype!=\"\", cluster=\"$cluster\"}\n -\n node_filesystem_avail_bytes{job=\"node-exporter\", fstype!=\"\", cluster=\"$cluster\"}\n ) != 0)\n)\n/ scalar(sum(max without (fstype, mountpoint) (node_filesystem_size_bytes{job=\"node-exporter\", fstype!=\"\", cluster=\"$cluster\"})))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk Space Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk Space", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "node-exporter-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(node_time_seconds, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Node Exporter / USE Method / Cluster", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/node-rsrc-use.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/node-rsrc-use.yaml new file mode 100644 index 0000000000..561dcb93dd --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/node-rsrc-use.yaml @@ -0,0 +1,1089 @@ +{{- /* +Generated from 'node-rsrc-use' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.nodeExporter.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "node-rsrc-use" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + node-rsrc-use.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 1, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "30s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_cpu_utilisation:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Utilisation", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_load1_per_cpu:ratio{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Saturation", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Saturation (Load1 per CPU)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_memory_utilisation:ratio{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Utilisation", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_vmstat_pgmajfault:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Major page Faults", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Saturation (Major Page Faults)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "rds", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "rds", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/Receive/", + "stack": "A" + }, + { + "alias": "/Transmit/", + "stack": "B", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_network_receive_bytes_excluding_lo:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Receive", + "refId": "A" + }, + { + "expr": "instance:node_network_transmit_bytes_excluding_lo:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Transmit", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Utilisation (Bytes Receive/Transmit)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/ Receive/", + "stack": "A" + }, + { + "alias": "/ Transmit/", + "stack": "B", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_network_receive_drop_excluding_lo:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Receive", + "refId": "A" + }, + { + "expr": "instance:node_network_transmit_drop_excluding_lo:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Transmit", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Saturation (Drops Receive/Transmit)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance_device:node_disk_io_time_seconds:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk IO Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance_device:node_disk_io_time_weighted_seconds:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk IO Saturation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk IO", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(1 -\n (\n max without (mountpoint, fstype) (node_filesystem_avail_bytes{job=\"node-exporter\", fstype!=\"\", instance=\"$instance\", cluster=\"$cluster\"})\n /\n max without (mountpoint, fstype) (node_filesystem_size_bytes{job=\"node-exporter\", fstype!=\"\", instance=\"$instance\", cluster=\"$cluster\"})\n ) != 0\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk Space Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk Space", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "node-exporter-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(node_time_seconds, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(node_exporter_build_info{job=\"node-exporter\", cluster=\"$cluster\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Node Exporter / USE Method / Node", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/nodes-darwin.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/nodes-darwin.yaml new file mode 100644 index 0000000000..09bc5930a1 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/nodes-darwin.yaml @@ -0,0 +1,1073 @@ +{{- /* +Generated from 'nodes-darwin' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "kube-prometheus-stack-grafana.namespace" . }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "nodes-darwin" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + nodes-darwin.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 1, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "30s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n (1 - sum without (mode) (rate(node_cpu_seconds_total{job=\"node-exporter\", mode=~\"idle|iowait|steal\", instance=\"$instance\"}[$__rate_interval])))\n/ ignoring(cpu) group_left\n count without (cpu, mode) (node_cpu_seconds_total{job=\"node-exporter\", mode=\"idle\", instance=\"$instance\"})\n)\n", + "format": "time_series", + "intervalFactor": 5, + "legendFormat": "{{`{{`}}cpu{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_load1{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "1m load average", + "refId": "A" + }, + { + "expr": "node_load5{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5m load average", + "refId": "B" + }, + { + "expr": "node_load15{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "15m load average", + "refId": "C" + }, + { + "expr": "count(node_cpu_seconds_total{job=\"node-exporter\", instance=\"$instance\", mode=\"idle\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "logical cores", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Load Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 9, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_total_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Physical Memory", + "refId": "A" + }, + { + "expr": "(\n node_memory_internal_bytes{job=\"node-exporter\", instance=\"$instance\"} -\n node_memory_purgeable_bytes{job=\"node-exporter\", instance=\"$instance\"} +\n node_memory_wired_bytes{job=\"node-exporter\", instance=\"$instance\"} +\n node_memory_compressed_bytes{job=\"node-exporter\", instance=\"$instance\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Memory Used", + "refId": "B" + }, + { + "expr": "(\n node_memory_internal_bytes{job=\"node-exporter\", instance=\"$instance\"} -\n node_memory_purgeable_bytes{job=\"node-exporter\", instance=\"$instance\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "App Memory", + "refId": "C" + }, + { + "expr": "node_memory_wired_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Wired Memory", + "refId": "D" + }, + { + "expr": "node_memory_compressed_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Compressed", + "refId": "E" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)" + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 80 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 90 + } + ] + }, + "unit": "percent" + } + }, + "gridPos": { + + }, + "id": 5, + "span": 3, + "targets": [ + { + "expr": "(\n (\n avg(node_memory_internal_bytes{job=\"node-exporter\", instance=\"$instance\"}) -\n avg(node_memory_purgeable_bytes{job=\"node-exporter\", instance=\"$instance\"}) +\n avg(node_memory_wired_bytes{job=\"node-exporter\", instance=\"$instance\"}) +\n avg(node_memory_compressed_bytes{job=\"node-exporter\", instance=\"$instance\"})\n ) /\n avg(node_memory_total_bytes{job=\"node-exporter\", instance=\"$instance\"})\n)\n*\n100\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "" + } + ], + "title": "Memory Usage", + "transparent": false, + "type": "gauge" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/ read| written/", + "yaxis": 1 + }, + { + "alias": "/ io time/", + "yaxis": 2 + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_disk_read_bytes_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}} read", + "refId": "A" + }, + { + "expr": "rate(node_disk_written_bytes_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}} written", + "refId": "B" + }, + { + "expr": "rate(node_disk_io_time_seconds_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}} io time", + "refId": "C" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "yellow", + "value": 0.8 + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Mounted on" + }, + "properties": [ + { + "id": "custom.width", + "value": 260 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Size" + }, + "properties": [ + { + "id": "custom.width", + "value": 93 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Used" + }, + "properties": [ + { + "id": "custom.width", + "value": 72 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Available" + }, + "properties": [ + { + "id": "custom.width", + "value": 88 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Used, %" + }, + "properties": [ + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "custom.displayMode", + "value": "gradient-gauge" + }, + { + "id": "max", + "value": 1 + }, + { + "id": "min", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + + }, + "id": 7, + "span": 6, + "targets": [ + { + "expr": "max by (mountpoint) (node_filesystem_size_bytes{job=\"node-exporter\", instance=\"$instance\", fstype!=\"\"})\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "" + }, + { + "expr": "max by (mountpoint) (node_filesystem_avail_bytes{job=\"node-exporter\", instance=\"$instance\", fstype!=\"\"})\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "" + } + ], + "title": "Disk Space Usage", + "transformations": [ + { + "id": "groupBy", + "options": { + "fields": { + "Value #A": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "Value #B": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "mountpoint": { + "aggregations": [ + + ], + "operation": "groupby" + } + } + } + }, + { + "id": "merge", + "options": { + + } + }, + { + "id": "calculateField", + "options": { + "alias": "Used", + "binary": { + "left": "Value #A (lastNotNull)", + "operator": "-", + "reducer": "sum", + "right": "Value #B (lastNotNull)" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "calculateField", + "options": { + "alias": "Used, %", + "binary": { + "left": "Used", + "operator": "/", + "reducer": "sum", + "right": "Value #A (lastNotNull)" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + + }, + "indexByName": { + + }, + "renameByName": { + "Value #A (lastNotNull)": "Size", + "Value #B (lastNotNull)": "Available", + "mountpoint": "Mounted on" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": { + + }, + "sort": [ + { + "field": "Mounted on" + } + ] + } + } + ], + "transparent": false, + "type": "table" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Network received (bits/s)", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_receive_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!=\"lo\"}[$__rate_interval]) * 8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Received", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Network transmitted (bits/s)", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_transmit_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!=\"lo\"}[$__rate_interval]) * 8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Transmitted", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "node-exporter-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "Instance", + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(node_uname_info{job=\"node-exporter\", sysname=\"Darwin\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Node Exporter / MacOS", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/nodes.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/nodes.yaml new file mode 100644 index 0000000000..adbe3f02e2 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/nodes.yaml @@ -0,0 +1,1066 @@ +{{- /* +Generated from 'nodes' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.nodeExporter.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "nodes" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + nodes.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 1, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "30s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n (1 - sum without (mode) (rate(node_cpu_seconds_total{job=\"node-exporter\", mode=~\"idle|iowait|steal\", instance=\"$instance\"}[$__rate_interval])))\n/ ignoring(cpu) group_left\n count without (cpu, mode) (node_cpu_seconds_total{job=\"node-exporter\", mode=\"idle\", instance=\"$instance\"})\n)\n", + "format": "time_series", + "intervalFactor": 5, + "legendFormat": "{{`{{`}}cpu{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_load1{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "1m load average", + "refId": "A" + }, + { + "expr": "node_load5{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5m load average", + "refId": "B" + }, + { + "expr": "node_load15{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "15m load average", + "refId": "C" + }, + { + "expr": "count(node_cpu_seconds_total{job=\"node-exporter\", instance=\"$instance\", mode=\"idle\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "logical cores", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Load Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 9, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n node_memory_MemTotal_bytes{job=\"node-exporter\", instance=\"$instance\"}\n-\n node_memory_MemFree_bytes{job=\"node-exporter\", instance=\"$instance\"}\n-\n node_memory_Buffers_bytes{job=\"node-exporter\", instance=\"$instance\"}\n-\n node_memory_Cached_bytes{job=\"node-exporter\", instance=\"$instance\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory used", + "refId": "A" + }, + { + "expr": "node_memory_Buffers_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory buffers", + "refId": "B" + }, + { + "expr": "node_memory_Cached_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory cached", + "refId": "C" + }, + { + "expr": "node_memory_MemFree_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory free", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)" + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 80 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 90 + } + ] + }, + "unit": "percent" + } + }, + "gridPos": { + + }, + "id": 5, + "span": 3, + "targets": [ + { + "expr": "100 -\n(\n avg(node_memory_MemAvailable_bytes{job=\"node-exporter\", instance=\"$instance\"}) /\n avg(node_memory_MemTotal_bytes{job=\"node-exporter\", instance=\"$instance\"})\n* 100\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "" + } + ], + "title": "Memory Usage", + "transparent": false, + "type": "gauge" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/ read| written/", + "yaxis": 1 + }, + { + "alias": "/ io time/", + "yaxis": 2 + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_disk_read_bytes_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}} read", + "refId": "A" + }, + { + "expr": "rate(node_disk_written_bytes_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}} written", + "refId": "B" + }, + { + "expr": "rate(node_disk_io_time_seconds_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}} io time", + "refId": "C" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "yellow", + "value": 0.8 + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Mounted on" + }, + "properties": [ + { + "id": "custom.width", + "value": 260 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Size" + }, + "properties": [ + { + "id": "custom.width", + "value": 93 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Used" + }, + "properties": [ + { + "id": "custom.width", + "value": 72 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Available" + }, + "properties": [ + { + "id": "custom.width", + "value": 88 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Used, %" + }, + "properties": [ + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "custom.displayMode", + "value": "gradient-gauge" + }, + { + "id": "max", + "value": 1 + }, + { + "id": "min", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + + }, + "id": 7, + "span": 6, + "targets": [ + { + "expr": "max by (mountpoint) (node_filesystem_size_bytes{job=\"node-exporter\", instance=\"$instance\", fstype!=\"\"})\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "" + }, + { + "expr": "max by (mountpoint) (node_filesystem_avail_bytes{job=\"node-exporter\", instance=\"$instance\", fstype!=\"\"})\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "" + } + ], + "title": "Disk Space Usage", + "transformations": [ + { + "id": "groupBy", + "options": { + "fields": { + "Value #A": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "Value #B": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "mountpoint": { + "aggregations": [ + + ], + "operation": "groupby" + } + } + } + }, + { + "id": "merge", + "options": { + + } + }, + { + "id": "calculateField", + "options": { + "alias": "Used", + "binary": { + "left": "Value #A (lastNotNull)", + "operator": "-", + "reducer": "sum", + "right": "Value #B (lastNotNull)" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "calculateField", + "options": { + "alias": "Used, %", + "binary": { + "left": "Used", + "operator": "/", + "reducer": "sum", + "right": "Value #A (lastNotNull)" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + + }, + "indexByName": { + + }, + "renameByName": { + "Value #A (lastNotNull)": "Size", + "Value #B (lastNotNull)": "Available", + "mountpoint": "Mounted on" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": { + + }, + "sort": [ + { + "field": "Mounted on" + } + ] + } + } + ], + "transparent": false, + "type": "table" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Network received (bits/s)", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_receive_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!=\"lo\"}[$__rate_interval]) * 8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Received", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Network transmitted (bits/s)", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_transmit_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!=\"lo\"}[$__rate_interval]) * 8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Transmitted", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "node-exporter-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "Instance", + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(node_uname_info{job=\"node-exporter\", sysname!=\"Darwin\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Node Exporter / Nodes", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/persistentvolumesusage.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/persistentvolumesusage.yaml new file mode 100644 index 0000000000..7fa1bd135b --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/persistentvolumesusage.yaml @@ -0,0 +1,587 @@ +{{- /* +Generated from 'persistentvolumesusage' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "persistentvolumesusage" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + persistentvolumesusage.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 9, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n sum without(instance, node) (topk(1, (kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n -\n sum without(instance, node) (topk(1, (kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n)\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Used Space", + "refId": "A" + }, + { + "expr": "sum without(instance, node) (topk(1, (kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Free Space", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Volume Space Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "$datasource", + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "max without(instance,node) (\n(\n topk(1, kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n -\n topk(1, kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n)\n/\ntopk(1, kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n* 100)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "80, 90", + "title": "Volume Space Usage", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 9, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum without(instance, node) (topk(1, (kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Used inodes", + "refId": "A" + }, + { + "expr": "(\n sum without(instance, node) (topk(1, (kubelet_volume_stats_inodes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n -\n sum without(instance, node) (topk(1, (kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n)\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": " Free inodes", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Volume inodes Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "$datasource", + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "max without(instance,node) (\ntopk(1, kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n/\ntopk(1, kubelet_volume_stats_inodes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n* 100)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "80, 90", + "title": "Volume inodes Usage", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kubelet_volume_stats_capacity_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "Namespace", + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\"}, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "PersistentVolumeClaim", + "multi": false, + "name": "volume", + "options": [ + + ], + "query": "label_values(kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\"}, persistentvolumeclaim)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-7d", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Persistent Volumes", + "uid": "919b92a8e8041bd567af9edab12c840c", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/pod-total.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/pod-total.yaml new file mode 100644 index 0000000000..d4ce802fbd --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/pod-total.yaml @@ -0,0 +1,1228 @@ +{{- /* +Generated from 'pod-total' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "pod-total" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + pod-total.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "time_series", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "height": 9, + "id": 3, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 12, + "nullPointMode": "connected", + "nullText": null, + "options": { + "fieldOptions": { + "calcs": [ + "last" + ], + "defaults": { + "max": 10000000000, + "min": 0, + "title": "$namespace: $pod", + "unit": "Bps" + }, + "mappings": [ + + ], + "override": { + + }, + "thresholds": [ + { + "color": "dark-green", + "index": 0, + "value": null + }, + { + "color": "dark-yellow", + "index": 1, + "value": 5000000000 + }, + { + "color": "dark-red", + "index": 2, + "value": 7000000000 + } + ], + "values": false + } + }, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 12, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution]))", + "format": "time_series", + "instant": null, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Received", + "type": "gauge", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "time_series", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "height": 9, + "id": 4, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 12, + "nullPointMode": "connected", + "nullText": null, + "options": { + "fieldOptions": { + "calcs": [ + "last" + ], + "defaults": { + "max": 10000000000, + "min": 0, + "title": "$namespace: $pod", + "unit": "Bps" + }, + "mappings": [ + + ], + "override": { + + }, + "thresholds": [ + { + "color": "dark-green", + "index": 0, + "value": null + }, + { + "color": "dark-yellow", + "index": 1, + "value": 5000000000 + }, + { + "color": "dark-red", + "index": 2, + "value": 7000000000 + } + ], + "values": false + } + }, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 12, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution]))", + "format": "time_series", + "instant": null, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Transmitted", + "type": "gauge", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 5, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 8, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 21 + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Packets", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 11, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 12, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Errors", + "titleSize": "h6", + "type": "row" + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".+", + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "kube-system", + "value": "kube-system" + }, + "datasource": "$datasource", + "definition": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".+", + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "definition": "label_values(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}, pod)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "pod", + "options": [ + + ], + "query": "label_values(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}, pod)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "resolution", + "options": [ + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "30s,5m,1h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Networking / Pod", + "uid": "7a18067ce943a40ae25454675c19ff5c", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/prometheus-remote-write.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/prometheus-remote-write.yaml new file mode 100644 index 0000000000..082795801d --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/prometheus-remote-write.yaml @@ -0,0 +1,1674 @@ +{{- /* +Generated from 'prometheus-remote-write' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.prometheus.prometheusSpec.remoteWriteDashboards }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "prometheus-remote-write" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + prometheus-remote-write.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "60s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(\n prometheus_remote_storage_highest_timestamp_in_seconds{cluster=~\"$cluster\", instance=~\"$instance\"} \n- \n ignoring(remote_name, url) group_right(instance) (prometheus_remote_storage_queue_highest_sent_timestamp_seconds{cluster=~\"$cluster\", instance=~\"$instance\"} != 0)\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Highest Timestamp In vs. Highest Timestamp Sent", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "clamp_min(\n rate(prometheus_remote_storage_highest_timestamp_in_seconds{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) \n- \n ignoring (remote_name, url) group_right(instance) rate(prometheus_remote_storage_queue_highest_sent_timestamp_seconds{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])\n, 0)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate[5m]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Timestamps", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(\n prometheus_remote_storage_samples_in_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])\n- \n ignoring(remote_name, url) group_right(instance) (rate(prometheus_remote_storage_succeeded_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) or rate(prometheus_remote_storage_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]))\n- \n (rate(prometheus_remote_storage_dropped_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) or rate(prometheus_remote_storage_samples_dropped_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate, in vs. succeeded or dropped [5m]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Samples", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "minSpan": 6, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_shards{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Shards", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_shards_max{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Max Shards", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_shards_min{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Min Shards", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_shards_desired{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Desired Shards", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Shards", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_shard_capacity{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Shard Capacity", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_pending_samples{cluster=~\"$cluster\", instance=~\"$instance\"} or prometheus_remote_storage_samples_pending{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Pending Samples", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Shard Details", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 11, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_wal_segment_current{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "TSDB Current Segment", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 12, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_wal_watcher_current_segment{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}consumer{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Remote Write Current Segment", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Segments", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_remote_storage_dropped_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) or rate(prometheus_remote_storage_samples_dropped_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Dropped Samples", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 14, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_remote_storage_failed_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Failed Samples", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 15, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_remote_storage_retried_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) or rate(prometheus_remote_storage_samples_retried_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Retried Samples", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 16, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_remote_storage_enqueue_retries_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Enqueue Retries", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Misc. Rates", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "prometheus-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "value": { + "selected": true, + "text": "All", + "value": "$__all" + } + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": true, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kube_pod_container_info{image=~\".*prometheus.*\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "value": { + "selected": true, + "text": "All", + "value": "$__all" + } + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(prometheus_build_info{cluster=~\"$cluster\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "url", + "options": [ + + ], + "query": "label_values(prometheus_remote_storage_shards{cluster=~\"$cluster\", instance=~\"$instance\"}, url)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Prometheus / Remote Write", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/prometheus.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/prometheus.yaml new file mode 100644 index 0000000000..1fd0b99096 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/prometheus.yaml @@ -0,0 +1,1235 @@ +{{- /* +Generated from 'prometheus' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "prometheus" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + prometheus.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "60s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Count", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Uptime", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Instance", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "instance", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Job", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "job", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Version", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "version", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "count by (job, instance, version) (prometheus_build_info{job=~\"$job\", instance=~\"$instance\"})", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "max by (job, instance) (time() - process_start_time_seconds{job=~\"$job\", instance=~\"$instance\"})", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Prometheus Stats", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Prometheus Stats", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(prometheus_target_sync_length_seconds_sum{job=~\"$job\",instance=~\"$instance\"}[5m])) by (scrape_job) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}scrape_job{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Target Sync", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(prometheus_sd_discovered_targets{job=~\"$job\",instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Targets", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Targets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Discovery", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_target_interval_length_seconds_sum{job=~\"$job\",instance=~\"$instance\"}[5m]) / rate(prometheus_target_interval_length_seconds_count{job=~\"$job\",instance=~\"$instance\"}[5m]) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}interval{{`}}`}} configured", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Scrape Interval Duration", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (job) (rate(prometheus_target_scrapes_exceeded_body_size_limit_total[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "exceeded body size limit: {{`{{`}}job{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum by (job) (rate(prometheus_target_scrapes_exceeded_sample_limit_total[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "exceeded sample limit: {{`{{`}}job{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum by (job) (rate(prometheus_target_scrapes_sample_duplicate_timestamp_total[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "duplicate timestamp: {{`{{`}}job{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum by (job) (rate(prometheus_target_scrapes_sample_out_of_bounds_total[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "out of bounds: {{`{{`}}job{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum by (job) (rate(prometheus_target_scrapes_sample_out_of_order_total[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "out of order: {{`{{`}}job{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Scrape failures", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_head_samples_appended_total{job=~\"$job\",instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}job{{`}}`}} {{`{{`}}instance{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Appended Samples", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Retrieval", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_head_series{job=~\"$job\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}job{{`}}`}} {{`{{`}}instance{{`}}`}} head series", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Head Series", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_head_chunks{job=~\"$job\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}job{{`}}`}} {{`{{`}}instance{{`}}`}} head chunks", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Head Chunks", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_engine_query_duration_seconds_count{job=~\"$job\",instance=~\"$instance\",slice=\"inner_eval\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}job{{`}}`}} {{`{{`}}instance{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Query Rate", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "max by (slice) (prometheus_engine_query_duration_seconds{quantile=\"0.9\",job=~\"$job\",instance=~\"$instance\"}) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}slice{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Stage Duration", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Query", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "prometheus-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": ".+", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "job", + "multi": true, + "name": "job", + "options": [ + + ], + "query": "label_values(prometheus_build_info{job=\"prometheus-k8s\",namespace=\"monitoring\"}, job)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".+", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "instance", + "multi": true, + "name": "instance", + "options": [ + + ], + "query": "label_values(prometheus_build_info{job=~\"$job\"}, instance)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Prometheus / Overview", + "uid": "", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/proxy.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/proxy.yaml new file mode 100644 index 0000000000..77d4fdf9eb --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/proxy.yaml @@ -0,0 +1,1276 @@ +{{- /* +Generated from 'proxy' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +{{- if (include "exporter.kubeProxy.enabled" .)}} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "proxy" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + proxy.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + {{- if .Values.k3sServer.enabled }} + "expr": "sum(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", metrics_path=\"/metrics\"})", + {{- else }} + "expr": "sum(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\"})", + {{- end }} + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Up", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "min" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubeproxy_sync_proxy_rules_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "rate", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rules Sync Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99,rate(kubeproxy_sync_proxy_rules_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rule Sync Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubeproxy_network_programming_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "rate", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Programming Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubeproxy_network_programming_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Programming Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\",code=~\"2..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "2xx", + "refId": "A" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\",code=~\"3..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "3xx", + "refId": "B" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\",code=~\"4..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "4xx", + "refId": "C" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\",code=~\"5..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5xx", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Kube API Request Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 8, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\",instance=~\"$instance\",verb=\"POST\"}[$__rate_interval])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Post Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\", verb=\"GET\"}[$__rate_interval])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Get Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\",instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubeProxy.jobName" . }}\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubeProxy.jobName" . }}\", cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Proxy", + "uid": "632e265de029684c40b21cb76bca4f94", + "version": 0 + } +{{- end }}{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/scheduler.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/scheduler.yaml new file mode 100644 index 0000000000..b71a9d4e4b --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/scheduler.yaml @@ -0,0 +1,1118 @@ +{{- /* +Generated from 'scheduler' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +{{- if (include "exporter.kubeScheduler.enabled" .)}} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "scheduler" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + scheduler.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + {{- if .Values.k3sServer.enabled }} + "expr": "sum(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", metrics_path=\"/metrics\"})", + {{- else }} + "expr": "sum(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\"})", + {{- end }} + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Up", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "min" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(scheduler_e2e_scheduling_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} e2e", + "refId": "A" + }, + { + "expr": "sum(rate(scheduler_binding_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} binding", + "refId": "B" + }, + { + "expr": "sum(rate(scheduler_scheduling_algorithm_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} scheduling algorithm", + "refId": "C" + }, + { + "expr": "sum(rate(scheduler_volume_scheduling_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} volume", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Scheduling Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\",instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} e2e", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(scheduler_binding_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\",instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} binding", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\",instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} scheduling algorithm", + "refId": "C" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(scheduler_volume_scheduling_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\",instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} volume", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Scheduling latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\",code=~\"2..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "2xx", + "refId": "A" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\",code=~\"3..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "3xx", + "refId": "B" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\",code=~\"4..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "4xx", + "refId": "C" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\",code=~\"5..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5xx", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Kube API Request Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 8, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\", verb=\"POST\"}[$__rate_interval])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Post Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\", verb=\"GET\"}[$__rate_interval])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Get Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubeScheduler.jobName" . }}\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", cluster=\"$cluster\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Scheduler", + "uid": "2e6b6a3b4bddf1427b3a55aa1311c656", + "version": 0 + } +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/workload-total.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/workload-total.yaml new file mode 100644 index 0000000000..043af9af96 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/dashboards-1.14/workload-total.yaml @@ -0,0 +1,1438 @@ +{{- /* +Generated from 'workload-total' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "workload-total" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + workload-total.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 3, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} pod {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} pod {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 5, + "panels": [ + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 6, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} pod {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} pod {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 8, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth HIstory", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 11, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 22 + }, + "id": 12, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 22 + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Packets", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 14, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 23 + }, + "id": 15, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 23 + }, + "id": 16, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Errors", + "titleSize": "h6", + "type": "row" + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kube_pod_info{job=\"kube-state-metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".+", + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "kube-system", + "value": "kube-system" + }, + "datasource": "$datasource", + "definition": "label_values(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\"}, namespace)", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "definition": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\"}, workload)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "workload", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\"}, workload)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "deployment", + "value": "deployment" + }, + "datasource": "$datasource", + "definition": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\"}, workload_type)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "type", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\"}, workload_type)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "resolution", + "options": [ + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "30s,5m,1h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Networking / Workload", + "uid": "728bf77cc1166d2f3133bf25846876cc", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/namespaces.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/namespaces.yaml new file mode 100644 index 0000000000..39ed210ed4 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/grafana/namespaces.yaml @@ -0,0 +1,13 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled (not .Values.grafana.defaultDashboards.useExistingNamespace) }} +apiVersion: v1 +kind: Namespace +metadata: + name: {{ .Values.grafana.defaultDashboards.namespace }} + labels: + name: {{ .Values.grafana.defaultDashboards.namespace }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + annotations: +{{- if not .Values.grafana.defaultDashboards.cleanupOnUninstall }} + helm.sh/resource-policy: "keep" +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/admission-webhooks/job-patch/clusterrole.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/admission-webhooks/job-patch/clusterrole.yaml new file mode 100644 index 0000000000..8b3f15f0d1 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/admission-webhooks/job-patch/clusterrole.yaml @@ -0,0 +1,33 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission +{{- include "kube-prometheus-stack.labels" $ | indent 4 }} +rules: + - apiGroups: + - admissionregistration.k8s.io + resources: + - validatingwebhookconfigurations + - mutatingwebhookconfigurations + verbs: + - get + - update +{{- if .Values.global.cattle.psp.enabled }} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if semverCompare "> 1.15.0-0" $kubeTargetVersion }} + - apiGroups: ['policy'] +{{- else }} + - apiGroups: ['extensions'] +{{- end }} + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "kube-prometheus-stack.fullname" . }}-admission +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/admission-webhooks/job-patch/clusterrolebinding.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/admission-webhooks/job-patch/clusterrolebinding.yaml new file mode 100644 index 0000000000..b909d14ebd --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/admission-webhooks/job-patch/clusterrolebinding.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission +{{- include "kube-prometheus-stack.labels" $ | indent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.fullname" . }}-admission +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/admission-webhooks/job-patch/job-createSecret.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/admission-webhooks/job-patch/job-createSecret.yaml new file mode 100644 index 0000000000..cb1e59b3c3 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/admission-webhooks/job-patch/job-createSecret.yaml @@ -0,0 +1,69 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-create + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-create +{{- include "kube-prometheus-stack.labels" $ | indent 4 }} +spec: + {{- if .Capabilities.APIVersions.Has "batch/v1alpha1" }} + # Alpha feature since k8s 1.12 + ttlSecondsAfterFinished: 0 + {{- end }} + template: + metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-create +{{- with .Values.prometheusOperator.admissionWebhooks.patch.podAnnotations }} + annotations: +{{ toYaml . | indent 8 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-create +{{- include "kube-prometheus-stack.labels" $ | indent 8 }} + spec: + {{- if .Values.prometheusOperator.admissionWebhooks.patch.priorityClassName }} + priorityClassName: {{ .Values.prometheusOperator.admissionWebhooks.patch.priorityClassName }} + {{- end }} + containers: + - name: create + {{- if .Values.prometheusOperator.admissionWebhooks.patch.image.sha }} + image: {{ template "system_default_registry" . }}{{ .Values.prometheusOperator.admissionWebhooks.patch.image.repository }}:{{ .Values.prometheusOperator.admissionWebhooks.patch.image.tag }}@sha256:{{ .Values.prometheusOperator.admissionWebhooks.patch.image.sha }} + {{- else }} + image: {{ template "system_default_registry" . }}{{ .Values.prometheusOperator.admissionWebhooks.patch.image.repository }}:{{ .Values.prometheusOperator.admissionWebhooks.patch.image.tag }} + {{- end }} + imagePullPolicy: {{ .Values.prometheusOperator.admissionWebhooks.patch.image.pullPolicy }} + args: + - create + - --host={{ template "kube-prometheus-stack.operator.fullname" . }},{{ template "kube-prometheus-stack.operator.fullname" . }}.{{ template "kube-prometheus-stack.namespace" . }}.svc + - --namespace={{ template "kube-prometheus-stack.namespace" . }} + - --secret-name={{ template "kube-prometheus-stack.fullname" . }}-admission + {{- with .Values.prometheusOperator.admissionWebhooks.createSecretJob }} + securityContext: + {{ toYaml .securityContext | nindent 12 }} + {{- end }} + resources: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.patch.resources | indent 12 }} + restartPolicy: OnFailure + serviceAccountName: {{ template "kube-prometheus-stack.fullname" . }}-admission + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- with .Values.prometheusOperator.admissionWebhooks.patch.nodeSelector }} +{{ toYaml . | indent 8 }} +{{- end }} + {{- with .Values.prometheusOperator.admissionWebhooks.patch.affinity }} + affinity: +{{ toYaml . | indent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- with .Values.prometheusOperator.admissionWebhooks.patch.tolerations }} +{{ toYaml . | indent 8 }} + {{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.patch.securityContext }} + securityContext: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.patch.securityContext | indent 8 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/admission-webhooks/job-patch/job-patchWebhook.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/admission-webhooks/job-patch/job-patchWebhook.yaml new file mode 100644 index 0000000000..067507af7f --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/admission-webhooks/job-patch/job-patchWebhook.yaml @@ -0,0 +1,70 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-patch + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-patch +{{- include "kube-prometheus-stack.labels" $ | indent 4 }} +spec: + {{- if .Capabilities.APIVersions.Has "batch/v1alpha1" }} + # Alpha feature since k8s 1.12 + ttlSecondsAfterFinished: 0 + {{- end }} + template: + metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-patch +{{- with .Values.prometheusOperator.admissionWebhooks.patch.podAnnotations }} + annotations: +{{ toYaml . | indent 8 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-patch +{{- include "kube-prometheus-stack.labels" $ | indent 8 }} + spec: + {{- if .Values.prometheusOperator.admissionWebhooks.patch.priorityClassName }} + priorityClassName: {{ .Values.prometheusOperator.admissionWebhooks.patch.priorityClassName }} + {{- end }} + containers: + - name: patch + {{- if .Values.prometheusOperator.admissionWebhooks.patch.image.sha }} + image: {{ template "system_default_registry" . }}{{ .Values.prometheusOperator.admissionWebhooks.patch.image.repository }}:{{ .Values.prometheusOperator.admissionWebhooks.patch.image.tag }}@sha256:{{ .Values.prometheusOperator.admissionWebhooks.patch.image.sha }} + {{- else }} + image: {{ template "system_default_registry" . }}{{ .Values.prometheusOperator.admissionWebhooks.patch.image.repository }}:{{ .Values.prometheusOperator.admissionWebhooks.patch.image.tag }} + {{- end }} + imagePullPolicy: {{ .Values.prometheusOperator.admissionWebhooks.patch.image.pullPolicy }} + args: + - patch + - --webhook-name={{ template "kube-prometheus-stack.fullname" . }}-admission + - --namespace={{ template "kube-prometheus-stack.namespace" . }} + - --secret-name={{ template "kube-prometheus-stack.fullname" . }}-admission + - --patch-failure-policy={{ .Values.prometheusOperator.admissionWebhooks.failurePolicy }} + {{- with .Values.prometheusOperator.admissionWebhooks.patchWebhookJob }} + securityContext: + {{ toYaml .securityContext | nindent 12 }} + {{- end }} + resources: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.patch.resources | indent 12 }} + restartPolicy: OnFailure + serviceAccountName: {{ template "kube-prometheus-stack.fullname" . }}-admission + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- with .Values.prometheusOperator.admissionWebhooks.patch.nodeSelector }} +{{ toYaml . | indent 8 }} +{{- end }} + {{- with .Values.prometheusOperator.admissionWebhooks.patch.affinity }} + affinity: +{{ toYaml . | indent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- with .Values.prometheusOperator.admissionWebhooks.patch.tolerations }} +{{ toYaml . | indent 8 }} + {{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.patch.securityContext }} + securityContext: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.patch.securityContext | indent 8 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/admission-webhooks/job-patch/psp.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/admission-webhooks/job-patch/psp.yaml new file mode 100644 index 0000000000..cd1ee7e477 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/admission-webhooks/job-patch/psp.yaml @@ -0,0 +1,47 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create .Values.global.cattle.psp.enabled (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +{{- if .Values.global.rbac.pspAnnotations }} +{{ toYaml .Values.global.rbac.pspAnnotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-admission +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + privileged: false + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + # Permits the container to run with root privileges as well. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/admission-webhooks/job-patch/role.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/admission-webhooks/job-patch/role.yaml new file mode 100644 index 0000000000..a64e982a3d --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/admission-webhooks/job-patch/role.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission +{{- include "kube-prometheus-stack.labels" $ | indent 4 }} +rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - create +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/admission-webhooks/job-patch/rolebinding.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/admission-webhooks/job-patch/rolebinding.yaml new file mode 100644 index 0000000000..d713629834 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/admission-webhooks/job-patch/rolebinding.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission +{{- include "kube-prometheus-stack.labels" $ | indent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ template "kube-prometheus-stack.fullname" . }}-admission +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/admission-webhooks/job-patch/serviceaccount.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/admission-webhooks/job-patch/serviceaccount.yaml new file mode 100644 index 0000000000..4fd52ae0a9 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/admission-webhooks/job-patch/serviceaccount.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission +{{- include "kube-prometheus-stack.labels" $ | indent 4 }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{ include "kube-prometheus-stack.imagePullSecrets" . | trim | indent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/admission-webhooks/mutatingWebhookConfiguration.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/admission-webhooks/mutatingWebhookConfiguration.yaml new file mode 100644 index 0000000000..7a12754ecc --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/admission-webhooks/mutatingWebhookConfiguration.yaml @@ -0,0 +1,42 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled }} +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission +{{- if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }} + annotations: + certmanager.k8s.io/inject-ca-from: {{ printf "%s/%s-admission" .Release.Namespace (include "kube-prometheus-stack.fullname" .) | quote }} + cert-manager.io/inject-ca-from: {{ printf "%s/%s-admission" .Release.Namespace (include "kube-prometheus-stack.fullname" .) | quote }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission +{{- include "kube-prometheus-stack.labels" $ | indent 4 }} +webhooks: + - name: prometheusrulemutate.monitoring.coreos.com + {{- if .Values.prometheusOperator.admissionWebhooks.patch.enabled }} + failurePolicy: Ignore + {{- else }} + failurePolicy: {{ .Values.prometheusOperator.admissionWebhooks.failurePolicy }} + {{- end }} + rules: + - apiGroups: + - monitoring.coreos.com + apiVersions: + - "*" + resources: + - prometheusrules + operations: + - CREATE + - UPDATE + clientConfig: + service: + namespace: {{ template "kube-prometheus-stack.namespace" . }} + name: {{ template "kube-prometheus-stack.operator.fullname" $ }} + path: /admission-prometheusrules/mutate + {{- if and .Values.prometheusOperator.admissionWebhooks.caBundle (not .Values.prometheusOperator.admissionWebhooks.patch.enabled) (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} + caBundle: {{ .Values.prometheusOperator.admissionWebhooks.caBundle }} + {{- end }} + timeoutSeconds: {{ .Values.prometheusOperator.admissionWebhooks.timeoutSeconds }} + admissionReviewVersions: ["v1", "v1beta1"] + sideEffects: None +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/admission-webhooks/validatingWebhookConfiguration.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/admission-webhooks/validatingWebhookConfiguration.yaml new file mode 100644 index 0000000000..9242659416 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/admission-webhooks/validatingWebhookConfiguration.yaml @@ -0,0 +1,41 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled }} +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission +{{- if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }} + annotations: + certmanager.k8s.io/inject-ca-from: {{ printf "%s/%s-admission" .Release.Namespace (include "kube-prometheus-stack.fullname" .) | quote }} + cert-manager.io/inject-ca-from: {{ printf "%s/%s-admission" .Release.Namespace (include "kube-prometheus-stack.fullname" .) | quote }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission +{{- include "kube-prometheus-stack.labels" $ | indent 4 }} +webhooks: + - name: prometheusrulemutate.monitoring.coreos.com + {{- if .Values.prometheusOperator.admissionWebhooks.patch.enabled }} + failurePolicy: Ignore + {{- else }} + failurePolicy: {{ .Values.prometheusOperator.admissionWebhooks.failurePolicy }} + {{- end }} + rules: + - apiGroups: + - monitoring.coreos.com + apiVersions: + - "*" + resources: + - prometheusrules + operations: + - CREATE + - UPDATE + clientConfig: + service: + namespace: {{ template "kube-prometheus-stack.namespace" . }} + name: {{ template "kube-prometheus-stack.operator.fullname" $ }} + path: /admission-prometheusrules/validate + {{- if and .Values.prometheusOperator.admissionWebhooks.caBundle (not .Values.prometheusOperator.admissionWebhooks.patch.enabled) (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} + caBundle: {{ .Values.prometheusOperator.admissionWebhooks.caBundle }} + {{- end }} + admissionReviewVersions: ["v1", "v1beta1"] + sideEffects: None +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/certmanager.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/certmanager.yaml new file mode 100644 index 0000000000..a1e06aec46 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/certmanager.yaml @@ -0,0 +1,57 @@ +{{- if .Values.prometheusOperator.admissionWebhooks.certManager.enabled -}} +{{- if not .Values.prometheusOperator.admissionWebhooks.certManager.issuerRef -}} +# Create a selfsigned Issuer, in order to create a root CA certificate for +# signing webhook serving certificates +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-self-signed-issuer + namespace: {{ template "kube-prometheus-stack.namespace" . }} +spec: + selfSigned: {} +--- +# Generate a CA Certificate used to sign certificates for the webhook +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-root-cert + namespace: {{ template "kube-prometheus-stack.namespace" . }} +spec: + secretName: {{ template "kube-prometheus-stack.fullname" . }}-root-cert + duration: {{ .Values.prometheusOperator.admissionWebhooks.certManager.rootCert.duration | default "43800h0m0s" | quote }} + issuerRef: + name: {{ template "kube-prometheus-stack.fullname" . }}-self-signed-issuer + commonName: "ca.webhook.kube-prometheus-stack" + isCA: true +--- +# Create an Issuer that uses the above generated CA certificate to issue certs +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-root-issuer + namespace: {{ template "kube-prometheus-stack.namespace" . }} +spec: + ca: + secretName: {{ template "kube-prometheus-stack.fullname" . }}-root-cert +{{- end }} +--- +# generate a server certificate for the apiservices to use +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + namespace: {{ template "kube-prometheus-stack.namespace" . }} +spec: + secretName: {{ template "kube-prometheus-stack.fullname" . }}-admission + duration: {{ .Values.prometheusOperator.admissionWebhooks.certManager.admissionCert.duration | default "8760h0m0s" | quote }} + issuerRef: + {{- if .Values.prometheusOperator.admissionWebhooks.certManager.issuerRef }} + {{- toYaml .Values.prometheusOperator.admissionWebhooks.certManager.issuerRef | nindent 4 }} + {{- else }} + name: {{ template "kube-prometheus-stack.fullname" . }}-root-issuer + {{- end }} + dnsNames: + - {{ template "kube-prometheus-stack.operator.fullname" . }} + - {{ template "kube-prometheus-stack.operator.fullname" . }}.{{ template "kube-prometheus-stack.namespace" . }} + - {{ template "kube-prometheus-stack.operator.fullname" . }}.{{ template "kube-prometheus-stack.namespace" . }}.svc +{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/clusterrole.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/clusterrole.yaml new file mode 100644 index 0000000000..300956a1da --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/clusterrole.yaml @@ -0,0 +1,81 @@ +{{- if and .Values.prometheusOperator.enabled .Values.global.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-operator + labels: + app: {{ template "kube-prometheus-stack.name" . }}-operator +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +rules: +- apiGroups: + - monitoring.coreos.com + resources: + - alertmanagers + - alertmanagers/finalizers + - alertmanagerconfigs + - prometheuses + - prometheuses/status + - prometheuses/finalizers + - thanosrulers + - thanosrulers/finalizers + - servicemonitors + - podmonitors + - probes + - prometheusrules + verbs: + - '*' +- apiGroups: + - apps + resources: + - statefulsets + verbs: + - '*' +- apiGroups: + - "" + resources: + - configmaps + - secrets + verbs: + - '*' +- apiGroups: + - "" + resources: + - pods + verbs: + - list + - delete +- apiGroups: + - "" + resources: + - services + - services/finalizers + - endpoints + verbs: + - get + - create + - update + - delete +- apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - namespaces + verbs: + - get + - list + - watch +- apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - get + - list + - watch +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/clusterrolebinding.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/clusterrolebinding.yaml new file mode 100644 index 0000000000..c9ab0ab872 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/clusterrolebinding.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.prometheusOperator.enabled .Values.global.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-operator + labels: + app: {{ template "kube-prometheus-stack.name" . }}-operator +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.fullname" . }}-operator +subjects: +- kind: ServiceAccount + name: {{ template "kube-prometheus-stack.operator.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/deployment.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/deployment.yaml new file mode 100644 index 0000000000..058d6801bb --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/deployment.yaml @@ -0,0 +1,164 @@ +{{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} +{{- $defaultKubeletSvcName := printf "%s-kubelet" (include "kube-prometheus-stack.fullname" .) }} +{{- if .Values.prometheusOperator.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-operator + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-operator +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheusOperator.annotations }} + annotations: +{{ toYaml .Values.prometheusOperator.annotations | indent 4 }} +{{- end }} +spec: + replicas: 1 + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-operator + release: {{ $.Release.Name | quote }} + template: + metadata: + labels: + app: {{ template "kube-prometheus-stack.name" . }}-operator +{{ include "kube-prometheus-stack.labels" . | indent 8 }} +{{- if .Values.prometheusOperator.podLabels }} +{{ toYaml .Values.prometheusOperator.podLabels | indent 8 }} +{{- end }} +{{- if .Values.prometheusOperator.podAnnotations }} + annotations: +{{ toYaml .Values.prometheusOperator.podAnnotations | indent 8 }} +{{- end }} + spec: + {{- if .Values.prometheusOperator.priorityClassName }} + priorityClassName: {{ .Values.prometheusOperator.priorityClassName }} + {{- end }} + containers: + - name: {{ template "kube-prometheus-stack.name" . }} + {{- if .Values.prometheusOperator.image.sha }} + image: "{{ template "system_default_registry" . }}{{ .Values.prometheusOperator.image.repository }}:{{ .Values.prometheusOperator.image.tag }}@sha256:{{ .Values.prometheusOperator.image.sha }}" + {{- else }} + image: "{{ template "system_default_registry" . }}{{ .Values.prometheusOperator.image.repository }}:{{ .Values.prometheusOperator.image.tag }}" + {{- end }} + imagePullPolicy: "{{ .Values.prometheusOperator.image.pullPolicy }}" + args: + {{- if .Values.prometheusOperator.kubeletService.enabled }} + - --kubelet-service={{ .Values.prometheusOperator.kubeletService.namespace }}/{{ default $defaultKubeletSvcName .Values.prometheusOperator.kubeletService.name }} + {{- end }} + {{- if .Values.prometheusOperator.logFormat }} + - --log-format={{ .Values.prometheusOperator.logFormat }} + {{- end }} + {{- if .Values.prometheusOperator.logLevel }} + - --log-level={{ .Values.prometheusOperator.logLevel }} + {{- end }} + {{- if .Values.prometheusOperator.denyNamespaces }} + - --deny-namespaces={{ tpl (.Values.prometheusOperator.denyNamespaces | join ",") $ }} + {{- end }} + {{- with $.Values.prometheusOperator.namespaces }} + {{- $namespaces := list }} + {{- if .releaseNamespace }} + {{- $namespaces = append $namespaces $namespace }} + {{- end }} + {{- if .additional }} + {{- range $ns := .additional }} + {{- $namespaces = append $namespaces (tpl $ns $) }} + {{- end }} + {{- end }} + - --namespaces={{ $namespaces | mustUniq | join "," }} + {{- end }} + - --localhost=127.0.0.1 + {{- if .Values.prometheusOperator.prometheusDefaultBaseImage }} + - --prometheus-default-base-image={{ template "system_default_registry" . }}{{ .Values.prometheusOperator.prometheusDefaultBaseImage }} + {{- end }} + {{- if .Values.prometheusOperator.alertmanagerDefaultBaseImage }} + - --alertmanager-default-base-image={{ template "system_default_registry" . }}{{ .Values.prometheusOperator.alertmanagerDefaultBaseImage }} + {{- end }} + {{- if .Values.prometheusOperator.prometheusConfigReloader.image.sha }} + - --prometheus-config-reloader={{ template "system_default_registry" . }}{{ .Values.prometheusOperator.prometheusConfigReloader.image.repository }}:{{ .Values.prometheusOperator.prometheusConfigReloader.image.tag }}@sha256:{{ .Values.prometheusOperator.prometheusConfigReloader.image.sha }} + {{- else }} + - --prometheus-config-reloader={{ template "system_default_registry" . }}{{ .Values.prometheusOperator.prometheusConfigReloader.image.repository }}:{{ .Values.prometheusOperator.prometheusConfigReloader.image.tag }} + {{- end }} + - --config-reloader-cpu-request={{ .Values.prometheusOperator.prometheusConfigReloader.resources.requests.cpu }} + - --config-reloader-cpu-limit={{ .Values.prometheusOperator.prometheusConfigReloader.resources.limits.cpu }} + - --config-reloader-memory-request={{ .Values.prometheusOperator.prometheusConfigReloader.resources.requests.memory }} + - --config-reloader-memory-limit={{ .Values.prometheusOperator.prometheusConfigReloader.resources.limits.memory }} + {{- if .Values.prometheusOperator.alertmanagerInstanceNamespaces }} + - --alertmanager-instance-namespaces={{ .Values.prometheusOperator.alertmanagerInstanceNamespaces | join "," }} + {{- end }} + {{- if .Values.prometheusOperator.alertmanagerConfigNamespaces }} + - --alertmanager-config-namespaces={{ .Values.prometheusOperator.alertmanagerConfigNamespaces | join "," }} + {{- end }} + {{- if .Values.prometheusOperator.prometheusInstanceNamespaces }} + - --prometheus-instance-namespaces={{ .Values.prometheusOperator.prometheusInstanceNamespaces | join "," }} + {{- end }} + {{- if .Values.prometheusOperator.thanosImage.sha }} + - --thanos-default-base-image={{ template "system_default_registry" . }}{{ .Values.prometheusOperator.thanosImage.repository }}:{{ .Values.prometheusOperator.thanosImage.tag }}@sha256:{{ .Values.prometheusOperator.thanosImage.sha }} + {{- else }} + - --thanos-default-base-image={{ template "system_default_registry" . }}{{ .Values.prometheusOperator.thanosImage.repository }}:{{ .Values.prometheusOperator.thanosImage.tag }} + {{- end }} + {{- if .Values.prometheusOperator.thanosRulerInstanceNamespaces }} + - --thanos-ruler-instance-namespaces={{ .Values.prometheusOperator.thanosRulerInstanceNamespaces | join "," }} + {{- end }} + {{- if .Values.prometheusOperator.secretFieldSelector }} + - --secret-field-selector={{ .Values.prometheusOperator.secretFieldSelector }} + {{- end }} + {{- if .Values.prometheusOperator.clusterDomain }} + - --cluster-domain={{ .Values.prometheusOperator.clusterDomain }} + {{- end }} + {{- if .Values.prometheusOperator.tls.enabled }} + - --web.enable-tls=true + - --web.cert-file=/cert/{{ if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }}tls.crt{{ else }}cert{{ end }} + - --web.key-file=/cert/{{ if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }}tls.key{{ else }}key{{ end }} + - --web.listen-address=:{{ .Values.prometheusOperator.tls.internalPort }} + - --web.tls-min-version={{ .Values.prometheusOperator.tls.tlsMinVersion }} + ports: + - containerPort: {{ .Values.prometheusOperator.tls.internalPort }} + name: https + {{- else }} + ports: + - containerPort: 8080 + name: http + {{- end }} + resources: +{{ toYaml .Values.prometheusOperator.resources | indent 12 }} + securityContext: +{{ toYaml .Values.prometheusOperator.containerSecurityContext | indent 12 }} +{{- if .Values.prometheusOperator.tls.enabled }} + volumeMounts: + - name: tls-secret + mountPath: /cert + readOnly: true + volumes: + - name: tls-secret + secret: + defaultMode: 420 + secretName: {{ template "kube-prometheus-stack.fullname" . }}-admission +{{- end }} + {{- with .Values.prometheusOperator.dnsConfig }} + dnsConfig: +{{ toYaml . | indent 8 }} + {{- end }} +{{- if .Values.prometheusOperator.securityContext }} + securityContext: +{{ toYaml .Values.prometheusOperator.securityContext | indent 8 }} +{{- end }} + serviceAccountName: {{ template "kube-prometheus-stack.operator.serviceAccountName" . }} +{{- if .Values.prometheusOperator.hostNetwork }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet +{{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- with .Values.prometheusOperator.nodeSelector }} +{{ toYaml . | indent 8 }} +{{- end }} + {{- with .Values.prometheusOperator.affinity }} + affinity: +{{ toYaml . | indent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- with .Values.prometheusOperator.tolerations }} +{{ toYaml . | indent 8 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/psp-clusterrole.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/psp-clusterrole.yaml new file mode 100644 index 0000000000..9d9019a430 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/psp-clusterrole.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.prometheusOperator.enabled .Values.global.rbac.create .Values.global.cattle.psp.enabled }} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-operator-psp + labels: + app: {{ template "kube-prometheus-stack.name" . }}-operator +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +rules: +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if semverCompare "> 1.15.0-0" $kubeTargetVersion }} +- apiGroups: ['policy'] +{{- else }} +- apiGroups: ['extensions'] +{{- end }} + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "kube-prometheus-stack.fullname" . }}-operator +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/psp-clusterrolebinding.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/psp-clusterrolebinding.yaml new file mode 100644 index 0000000000..2dea3e36cb --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/psp-clusterrolebinding.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.prometheusOperator.enabled .Values.global.rbac.create .Values.global.cattle.psp.enabled }} +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-operator-psp + labels: + app: {{ template "kube-prometheus-stack.name" . }}-operator +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.fullname" . }}-operator-psp +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.operator.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/psp.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/psp.yaml new file mode 100644 index 0000000000..5d9408d748 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/psp.yaml @@ -0,0 +1,45 @@ +{{- if and .Values.prometheusOperator.enabled .Values.global.rbac.create .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-operator + labels: + app: {{ template "kube-prometheus-stack.name" . }}-operator +{{- if .Values.global.rbac.pspAnnotations }} + annotations: +{{ toYaml .Values.global.rbac.pspAnnotations | indent 4 }} +{{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + privileged: false + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' + hostNetwork: {{ .Values.prometheusOperator.hostNetwork }} + hostIPC: false + hostPID: false + runAsUser: + # Permits the container to run with root privileges as well. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/service.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/service.yaml new file mode 100644 index 0000000000..b5ef5b93d5 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/service.yaml @@ -0,0 +1,58 @@ +{{- if .Values.prometheusOperator.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-operator + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-operator +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheusOperator.service.labels }} +{{ toYaml .Values.prometheusOperator.service.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheusOperator.service.annotations }} + annotations: +{{ toYaml .Values.prometheusOperator.service.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.prometheusOperator.service.clusterIP }} + clusterIP: {{ .Values.prometheusOperator.service.clusterIP }} +{{- end }} +{{- if .Values.prometheusOperator.service.externalIPs }} + externalIPs: +{{ toYaml .Values.prometheusOperator.service.externalIPs | indent 4 }} +{{- end }} +{{- if .Values.prometheusOperator.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.prometheusOperator.service.loadBalancerIP }} +{{- end }} +{{- if .Values.prometheusOperator.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.prometheusOperator.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if ne .Values.prometheusOperator.service.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.prometheusOperator.service.externalTrafficPolicy }} +{{- end }} + ports: + {{- if not .Values.prometheusOperator.tls.enabled }} + - name: http + {{- if eq .Values.prometheusOperator.service.type "NodePort" }} + nodePort: {{ .Values.prometheusOperator.service.nodePort }} + {{- end }} + port: 8080 + targetPort: http + {{- end }} + {{- if .Values.prometheusOperator.tls.enabled }} + - name: https + {{- if eq .Values.prometheusOperator.service.type "NodePort"}} + nodePort: {{ .Values.prometheusOperator.service.nodePortTls }} + {{- end }} + port: 443 + targetPort: https + {{- end }} + selector: + app: {{ template "kube-prometheus-stack.name" . }}-operator + release: {{ $.Release.Name | quote }} + type: "{{ .Values.prometheusOperator.service.type }}" +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/serviceaccount.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/serviceaccount.yaml new file mode 100644 index 0000000000..781975f32e --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/serviceaccount.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "kube-prometheus-stack.operator.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-operator + app.kubernetes.io/name: {{ template "kube-prometheus-stack.name" . }}-prometheus-operator + app.kubernetes.io/component: prometheus-operator +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{ include "kube-prometheus-stack.imagePullSecrets" . | trim | indent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/servicemonitor.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/servicemonitor.yaml new file mode 100644 index 0000000000..3af46529cb --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus-operator/servicemonitor.yaml @@ -0,0 +1,54 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.serviceMonitor.selfMonitor }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-operator + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-operator +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + endpoints: + {{- if .Values.prometheusOperator.tls.enabled }} + - port: https + scheme: https + tlsConfig: + serverName: {{ template "kube-prometheus-stack.operator.fullname" . }} + ca: + secret: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + key: {{ if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }}ca.crt{{ else }}ca{{ end }} + optional: false + {{- else }} + - port: http + {{- end }} + honorLabels: true + {{- if .Values.prometheusOperator.serviceMonitor.interval }} + interval: {{ .Values.prometheusOperator.serviceMonitor.interval }} + {{- end }} + metricRelabelings: + {{- if .Values.prometheusOperator.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.prometheusOperator.serviceMonitor.metricRelabelings | indent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.prometheusOperator.serviceMonitor.relabelings }} + relabelings: +{{ toYaml .Values.prometheusOperator.serviceMonitor.relabelings | indent 6 }} +{{- end }} + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-operator + release: {{ $.Release.Name | quote }} + namespaceSelector: + matchNames: + - {{ printf "%s" (include "kube-prometheus-stack.namespace" .) | quote }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/_rules.tpl b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/_rules.tpl new file mode 100644 index 0000000000..e8baf98e47 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/_rules.tpl @@ -0,0 +1,36 @@ +{{- /* +Generated file. Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- define "rules.names" }} +rules: + - "alertmanager.rules" + - "config-reloaders" + - "etcd" + - "general.rules" + - "k8s.rules" + - "kube-apiserver-availability.rules" + - "kube-apiserver-burnrate.rules" + - "kube-apiserver-histogram.rules" + - "kube-apiserver-slos" + - "kube-prometheus-general.rules" + - "kube-prometheus-node-recording.rules" + - "kube-scheduler.rules" + - "kube-state-metrics" + - "kubelet.rules" + - "kubernetes-apps" + - "kubernetes-resources" + - "kubernetes-storage" + - "kubernetes-system" + - "kubernetes-system-kube-proxy" + - "kubernetes-system-apiserver" + - "kubernetes-system-kubelet" + - "kubernetes-system-controller-manager" + - "kubernetes-system-scheduler" + - "node-exporter.rules" + - "node-exporter" + - "node.rules" + - "node-network" + - "prometheus-operator" + - "prometheus" +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/additionalAlertRelabelConfigs.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/additionalAlertRelabelConfigs.yaml new file mode 100644 index 0000000000..bff930981a --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/additionalAlertRelabelConfigs.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.prometheusSpec.additionalAlertRelabelConfigs }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-am-relabel-confg + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.prometheus.prometheusSpec.additionalPrometheusSecretsAnnotations }} + annotations: +{{ toYaml .Values.prometheus.prometheusSpec.additionalPrometheusSecretsAnnotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus-am-relabel-confg +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: + additional-alert-relabel-configs.yaml: {{ toYaml .Values.prometheus.prometheusSpec.additionalAlertRelabelConfigs | b64enc | quote }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/additionalAlertmanagerConfigs.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/additionalAlertmanagerConfigs.yaml new file mode 100644 index 0000000000..2fe8fdb816 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/additionalAlertmanagerConfigs.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.prometheusSpec.additionalAlertManagerConfigs }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-am-confg + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.prometheus.prometheusSpec.additionalPrometheusSecretsAnnotations }} + annotations: +{{ toYaml .Values.prometheus.prometheusSpec.additionalPrometheusSecretsAnnotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus-am-confg +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: + additional-alertmanager-configs.yaml: {{ tpl (toYaml .Values.prometheus.prometheusSpec.additionalAlertManagerConfigs) . | b64enc | quote }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/additionalPrometheusRules.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/additionalPrometheusRules.yaml new file mode 100644 index 0000000000..cb4aabaa7b --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/additionalPrometheusRules.yaml @@ -0,0 +1,43 @@ +{{- if or .Values.additionalPrometheusRules .Values.additionalPrometheusRulesMap}} +apiVersion: v1 +kind: List +metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-additional-prometheus-rules + namespace: {{ template "kube-prometheus-stack.namespace" . }} +items: +{{- if .Values.additionalPrometheusRulesMap }} +{{- range $prometheusRuleName, $prometheusRule := .Values.additionalPrometheusRulesMap }} + - apiVersion: monitoring.coreos.com/v1 + kind: PrometheusRule + metadata: + name: {{ template "kube-prometheus-stack.name" $ }}-{{ $prometheusRuleName }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }} +{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if $prometheusRule.additionalLabels }} +{{ toYaml $prometheusRule.additionalLabels | indent 8 }} + {{- end }} + spec: + groups: +{{ toYaml $prometheusRule.groups| indent 8 }} +{{- end }} +{{- else }} +{{- range .Values.additionalPrometheusRules }} + - apiVersion: monitoring.coreos.com/v1 + kind: PrometheusRule + metadata: + name: {{ template "kube-prometheus-stack.name" $ }}-{{ .name }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }} +{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if .additionalLabels }} +{{ toYaml .additionalLabels | indent 8 }} + {{- end }} + spec: + groups: +{{ toYaml .groups| indent 8 }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/additionalScrapeConfigs.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/additionalScrapeConfigs.yaml new file mode 100644 index 0000000000..ebdf766fde --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/additionalScrapeConfigs.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.prometheusSpec.additionalScrapeConfigs }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-scrape-confg + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.prometheus.prometheusSpec.additionalPrometheusSecretsAnnotations }} + annotations: +{{ toYaml .Values.prometheus.prometheusSpec.additionalPrometheusSecretsAnnotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus-scrape-confg +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: +{{- if eq ( typeOf .Values.prometheus.prometheusSpec.additionalScrapeConfigs ) "string" }} + additional-scrape-configs.yaml: {{ tpl .Values.prometheus.prometheusSpec.additionalScrapeConfigs $ | b64enc | quote }} +{{- else }} + additional-scrape-configs.yaml: {{ tpl (toYaml .Values.prometheus.prometheusSpec.additionalScrapeConfigs) $ | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/clusterrole.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/clusterrole.yaml new file mode 100644 index 0000000000..3585b5db11 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/clusterrole.yaml @@ -0,0 +1,30 @@ +{{- if and .Values.prometheus.enabled .Values.global.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +rules: +# This permission are not in the kube-prometheus repo +# they're grabbed from https://github.com/prometheus/prometheus/blob/master/documentation/examples/rbac-setup.yml +- apiGroups: [""] + resources: + - nodes + - nodes/metrics + - services + - endpoints + - pods + verbs: ["get", "list", "watch"] +- apiGroups: + - "networking.k8s.io" + resources: + - ingresses + verbs: ["get", "list", "watch"] +- nonResourceURLs: ["/metrics", "/metrics/cadvisor"] + verbs: ["get"] +{{- if .Values.prometheus.additionalRulesForClusterRole }} +{{ toYaml .Values.prometheus.additionalRulesForClusterRole | indent 0 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/clusterrolebinding.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/clusterrolebinding.yaml new file mode 100644 index 0000000000..9fc4f65da4 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/clusterrolebinding.yaml @@ -0,0 +1,18 @@ +{{- if and .Values.prometheus.enabled .Values.global.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.prometheus.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} + diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/csi-secret.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/csi-secret.yaml new file mode 100644 index 0000000000..89399cec85 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/csi-secret.yaml @@ -0,0 +1,12 @@ +{{- if .Values.prometheus.prometheusSpec.thanos.secretProviderClass }} +--- +apiVersion: secrets-store.csi.x-k8s.io/v1alpha1 +kind: SecretProviderClass +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +spec: +{{ toYaml .Values.prometheus.prometheusSpec.thanos.secretProviderClass | indent 2 }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/extrasecret.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/extrasecret.yaml new file mode 100644 index 0000000000..17f3478a46 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/extrasecret.yaml @@ -0,0 +1,20 @@ +{{- if .Values.prometheus.extraSecret.data -}} +{{- $secretName := printf "prometheus-%s-extra" (include "kube-prometheus-stack.fullname" . ) -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ default $secretName .Values.prometheus.extraSecret.name }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.prometheus.extraSecret.annotations }} + annotations: +{{ toYaml .Values.prometheus.extraSecret.annotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + app.kubernetes.io/component: prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: +{{- range $key, $val := .Values.prometheus.extraSecret.data }} + {{ $key }}: {{ $val | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/ingress.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/ingress.yaml new file mode 100644 index 0000000000..91fadf905f --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/ingress.yaml @@ -0,0 +1,77 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.ingress.enabled -}} + {{- $pathType := .Values.prometheus.ingress.pathType | default "ImplementationSpecific" -}} + {{- $serviceName := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "prometheus" -}} + {{- $servicePort := .Values.prometheus.ingress.servicePort | default .Values.prometheus.service.port -}} + {{- $routePrefix := list .Values.prometheus.prometheusSpec.routePrefix -}} + {{- $paths := .Values.prometheus.ingress.paths | default $routePrefix -}} + {{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} + {{- $ingressSupportsPathType := eq (include "kube-prometheus-stack.ingress.supportsPathType" .) "true" -}} +apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" . }} +kind: Ingress +metadata: +{{- if .Values.prometheus.ingress.annotations }} + annotations: +{{ toYaml .Values.prometheus.ingress.annotations | indent 4 }} +{{- end }} + name: {{ $serviceName }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.ingress.labels }} +{{ toYaml .Values.prometheus.ingress.labels | indent 4 }} +{{- end }} +spec: + {{- if $apiIsStable }} + {{- if .Values.prometheus.ingress.ingressClassName }} + ingressClassName: {{ .Values.prometheus.ingress.ingressClassName }} + {{- end }} + {{- end }} + rules: + {{- if .Values.prometheus.ingress.hosts }} + {{- range $host := .Values.prometheus.ingress.hosts }} + - host: {{ tpl $host $ }} + http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $serviceName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $serviceName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- else }} + - http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $serviceName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $serviceName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- if .Values.prometheus.ingress.tls }} + tls: +{{ tpl (toYaml .Values.prometheus.ingress.tls | indent 4) . }} + {{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/ingressThanosSidecar.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/ingressThanosSidecar.yaml new file mode 100644 index 0000000000..7a338597b0 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/ingressThanosSidecar.yaml @@ -0,0 +1,76 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.thanosIngress.enabled }} +{{- $pathType := .Values.prometheus.thanosIngress.pathType | default "" }} +{{- $serviceName := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "prometheus" }} +{{- $thanosPort := .Values.prometheus.thanosIngress.servicePort -}} +{{- $routePrefix := list .Values.prometheus.prometheusSpec.routePrefix }} +{{- $paths := .Values.prometheus.thanosIngress.paths | default $routePrefix -}} +{{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "kube-prometheus-stack.ingress.supportsPathType" .) "true" -}} +apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" . }} +kind: Ingress +metadata: +{{- if .Values.prometheus.thanosIngress.annotations }} + annotations: +{{ toYaml .Values.prometheus.thanosIngress.annotations | indent 4 }} +{{- end }} + name: {{ template "kube-prometheus-stack.fullname" . }}-thanos-gateway + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.thanosIngress.labels }} +{{ toYaml .Values.prometheus.thanosIngress.labels | indent 4 }} +{{- end }} +spec: + {{- if $apiIsStable }} + {{- if .Values.prometheus.thanosIngress.ingressClassName }} + ingressClassName: {{ .Values.prometheus.thanosIngress.ingressClassName }} + {{- end }} + {{- end }} + rules: + {{- if .Values.prometheus.thanosIngress.hosts }} + {{- range $host := .Values.prometheus.thanosIngress.hosts }} + - host: {{ tpl $host $ }} + http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $serviceName }} + port: + number: {{ $thanosPort }} + {{- else }} + serviceName: {{ $serviceName }} + servicePort: {{ $thanosPort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- else }} + - http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $serviceName }} + port: + number: {{ $thanosPort }} + {{- else }} + serviceName: {{ $serviceName }} + servicePort: {{ $thanosPort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- if .Values.prometheus.thanosIngress.tls }} + tls: +{{ tpl (toYaml .Values.prometheus.thanosIngress.tls | indent 4) . }} + {{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/ingressperreplica.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/ingressperreplica.yaml new file mode 100644 index 0000000000..df631993ba --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/ingressperreplica.yaml @@ -0,0 +1,67 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.servicePerReplica.enabled .Values.prometheus.ingressPerReplica.enabled }} +{{- $pathType := .Values.prometheus.ingressPerReplica.pathType | default "" }} +{{- $count := .Values.prometheus.prometheusSpec.replicas | int -}} +{{- $servicePort := .Values.prometheus.servicePerReplica.port -}} +{{- $ingressValues := .Values.prometheus.ingressPerReplica -}} +{{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "kube-prometheus-stack.ingress.supportsPathType" .) "true" -}} +apiVersion: v1 +kind: List +metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-ingressperreplica + namespace: {{ template "kube-prometheus-stack.namespace" $ }} +items: +{{ range $i, $e := until $count }} + - kind: Ingress + apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" $ }} + metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-{{ $i }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ include "kube-prometheus-stack.name" $ }}-prometheus + {{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if $ingressValues.labels }} +{{ toYaml $ingressValues.labels | indent 8 }} + {{- end }} + {{- if $ingressValues.annotations }} + annotations: +{{ toYaml $ingressValues.annotations | indent 8 }} + {{- end }} + spec: + {{- if $apiIsStable }} + {{- if $ingressValues.ingressClassName }} + ingressClassName: {{ $ingressValues.ingressClassName }} + {{- end }} + {{- end }} + rules: + - host: {{ $ingressValues.hostPrefix }}-{{ $i }}.{{ $ingressValues.hostDomain }} + http: + paths: + {{- range $p := $ingressValues.paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-{{ $i }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-{{ $i }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- if or $ingressValues.tlsSecretName $ingressValues.tlsSecretPerReplica.enabled }} + tls: + - hosts: + - {{ $ingressValues.hostPrefix }}-{{ $i }}.{{ $ingressValues.hostDomain }} + {{- if $ingressValues.tlsSecretPerReplica.enabled }} + secretName: {{ $ingressValues.tlsSecretPerReplica.prefix }}-{{ $i }} + {{- else }} + secretName: {{ $ingressValues.tlsSecretName }} + {{- end }} + {{- end }} +{{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/nginx-config.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/nginx-config.yaml new file mode 100644 index 0000000000..e4d91f9a9e --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/nginx-config.yaml @@ -0,0 +1,68 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-nginx-proxy-config + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.annotations }} + annotations: +{{ toYaml .Values.prometheus.annotations | indent 4 }} +{{- end }} +data: + nginx.conf: |- + worker_processes auto; + error_log /dev/stdout warn; + pid /var/cache/nginx/nginx.pid; + + events { + worker_connections 1024; + } + + http { + include /etc/nginx/mime.types; + log_format main '[$time_local - $status] $remote_addr - $remote_user $request ($http_referer)'; + + proxy_connect_timeout 10; + proxy_read_timeout 180; + proxy_send_timeout 5; + proxy_buffering off; + proxy_cache_path /var/cache/nginx/cache levels=1:2 keys_zone=my_zone:100m inactive=1d max_size=10g; + + server { + listen 8081; + access_log off; + + gzip on; + gzip_min_length 1k; + gzip_comp_level 2; + gzip_types text/plain application/javascript application/x-javascript text/css application/xml text/javascript image/jpeg image/gif image/png; + gzip_vary on; + gzip_disable "MSIE [1-6]\."; + + proxy_set_header Host $host; + + location / { + proxy_cache my_zone; + proxy_cache_valid 200 302 1d; + proxy_cache_valid 301 30d; + proxy_cache_valid any 5m; + proxy_cache_bypass $http_cache_control; + add_header X-Proxy-Cache $upstream_cache_status; + add_header Cache-Control "public"; + + proxy_pass http://localhost:9090/; + + sub_filter_once off; + sub_filter 'var PATH_PREFIX = "";' 'var PATH_PREFIX = ".";'; + + if ($request_filename ~ .*\.(?:js|css|jpg|jpeg|gif|png|ico|cur|gz|svg|svgz|mp4|ogg|ogv|webm)$) { + expires 90d; + } + + rewrite ^/k8s/clusters/.*/proxy(.*) /$1 break; + + } + } + } diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/podDisruptionBudget.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/podDisruptionBudget.yaml new file mode 100644 index 0000000000..02a320eff4 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/podDisruptionBudget.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.podDisruptionBudget.enabled }} +apiVersion: {{ include "kube-prometheus-stack.pdb.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- if .Values.prometheus.podDisruptionBudget.minAvailable }} + minAvailable: {{ .Values.prometheus.podDisruptionBudget.minAvailable }} + {{- end }} + {{- if .Values.prometheus.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ .Values.prometheus.podDisruptionBudget.maxUnavailable }} + {{- end }} + selector: + matchLabels: + app.kubernetes.io/name: prometheus + prometheus: {{ template "kube-prometheus-stack.prometheus.crname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/podmonitors.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/podmonitors.yaml new file mode 100644 index 0000000000..95d568e13d --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/podmonitors.yaml @@ -0,0 +1,37 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.additionalPodMonitors }} +apiVersion: v1 +kind: List +items: +{{- range .Values.prometheus.additionalPodMonitors }} + - apiVersion: monitoring.coreos.com/v1 + kind: PodMonitor + metadata: + name: {{ .name }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-prometheus +{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if .additionalLabels }} +{{ toYaml .additionalLabels | indent 8 }} + {{- end }} + spec: + podMetricsEndpoints: +{{ toYaml .podMetricsEndpoints | indent 8 }} + {{- if .jobLabel }} + jobLabel: {{ .jobLabel }} + {{- end }} + {{- if .namespaceSelector }} + namespaceSelector: +{{ toYaml .namespaceSelector | indent 8 }} + {{- end }} + selector: +{{ toYaml .selector | indent 8 }} + {{- if .podTargetLabels }} + podTargetLabels: +{{ toYaml .podTargetLabels | indent 8 }} + {{- end }} + {{- if .sampleLimit }} + sampleLimit: {{ .sampleLimit }} + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/prometheus.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/prometheus.yaml new file mode 100644 index 0000000000..ffa00b8d97 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/prometheus.yaml @@ -0,0 +1,388 @@ +{{- if .Values.prometheus.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: Prometheus +metadata: + name: {{ template "kube-prometheus-stack.prometheus.crname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.annotations }} + annotations: +{{ toYaml .Values.prometheus.annotations | indent 4 }} +{{- end }} +spec: + alerting: + alertmanagers: +{{- if .Values.prometheus.prometheusSpec.alertingEndpoints }} +{{ toYaml .Values.prometheus.prometheusSpec.alertingEndpoints | indent 6 }} +{{- else if .Values.alertmanager.enabled }} + - namespace: {{ template "kube-prometheus-stack.namespace" . }} + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + port: {{ .Values.alertmanager.alertmanagerSpec.portName }} + {{- if .Values.alertmanager.alertmanagerSpec.routePrefix }} + pathPrefix: "{{ .Values.alertmanager.alertmanagerSpec.routePrefix }}" + {{- end }} + apiVersion: {{ .Values.alertmanager.apiVersion }} +{{- else }} + [] +{{- end }} +{{- if .Values.prometheus.prometheusSpec.apiserverConfig }} + apiserverConfig: +{{ toYaml .Values.prometheus.prometheusSpec.apiserverConfig | indent 4}} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.image }} + {{- if and .Values.prometheus.prometheusSpec.image.tag .Values.prometheus.prometheusSpec.image.sha }} + image: "{{ template "system_default_registry" . }}{{ .Values.prometheus.prometheusSpec.image.repository }}:{{ .Values.prometheus.prometheusSpec.image.tag }}@sha256:{{ .Values.prometheus.prometheusSpec.image.sha }}" + {{- else if .Values.prometheus.prometheusSpec.image.sha }} + image: "{{ template "system_default_registry" . }}{{ .Values.prometheus.prometheusSpec.image.repository }}@sha256:{{ .Values.prometheus.prometheusSpec.image.sha }}" + {{- else if .Values.prometheus.prometheusSpec.image.tag }} + image: "{{ template "system_default_registry" . }}{{ .Values.prometheus.prometheusSpec.image.repository }}:{{ .Values.prometheus.prometheusSpec.image.tag }}" + {{- else }} + image: "{{ template "system_default_registry" . }}{{ .Values.prometheus.prometheusSpec.image.repository }}" + {{- end }} + version: {{ .Values.prometheus.prometheusSpec.image.tag }} + {{- if .Values.prometheus.prometheusSpec.image.sha }} + sha: {{ .Values.prometheus.prometheusSpec.image.sha }} + {{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.externalLabels }} + externalLabels: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.externalLabels | indent 4) . }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.prometheusExternalLabelNameClear }} + prometheusExternalLabelName: "" +{{- else if .Values.prometheus.prometheusSpec.prometheusExternalLabelName }} + prometheusExternalLabelName: "{{ .Values.prometheus.prometheusSpec.prometheusExternalLabelName }}" +{{- end }} +{{- if .Values.prometheus.prometheusSpec.replicaExternalLabelNameClear }} + replicaExternalLabelName: "" +{{- else if .Values.prometheus.prometheusSpec.replicaExternalLabelName }} + replicaExternalLabelName: "{{ .Values.prometheus.prometheusSpec.replicaExternalLabelName }}" +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enableRemoteWriteReceiver }} + enableRemoteWriteReceiver: {{ .Values.prometheus.prometheusSpec.enableRemoteWriteReceiver }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.externalUrl }} + externalUrl: "{{ tpl .Values.prometheus.prometheusSpec.externalUrl . }}" +{{- else if and .Values.prometheus.ingress.enabled .Values.prometheus.ingress.hosts }} + externalUrl: "http://{{ tpl (index .Values.prometheus.ingress.hosts 0) . }}{{ .Values.prometheus.prometheusSpec.routePrefix }}" +{{- else if not (or (kindIs "invalid" .Values.global.cattle.url) (kindIs "invalid" .Values.global.cattle.clusterId)) }} + externalUrl: "{{ .Values.global.cattle.url }}/k8s/clusters/{{ .Values.global.cattle.clusterId }}/api/v1/namespaces/{{ template "kube-prometheus-stack.namespace" . }}/services/http:{{ template "kube-prometheus-stack.fullname" . }}-prometheus:{{ .Values.prometheus.service.port }}/proxy" +{{- else }} + externalUrl: http://{{ template "kube-prometheus-stack.fullname" . }}-prometheus.{{ template "kube-prometheus-stack.namespace" . }}:{{ .Values.prometheus.service.port }} +{{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 4 }} +{{- if .Values.prometheus.prometheusSpec.nodeSelector }} +{{ toYaml .Values.prometheus.prometheusSpec.nodeSelector | indent 4 }} +{{- end }} + paused: {{ .Values.prometheus.prometheusSpec.paused }} + replicas: {{ .Values.prometheus.prometheusSpec.replicas }} + shards: {{ .Values.prometheus.prometheusSpec.shards }} + logLevel: {{ .Values.prometheus.prometheusSpec.logLevel }} + logFormat: {{ .Values.prometheus.prometheusSpec.logFormat }} + listenLocal: {{ .Values.prometheus.prometheusSpec.listenLocal }} + enableAdminAPI: {{ .Values.prometheus.prometheusSpec.enableAdminAPI }} +{{- if .Values.prometheus.prometheusSpec.web }} + web: +{{ toYaml .Values.prometheus.prometheusSpec.web | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.exemplars }} + exemplars: + {{ toYaml .Values.prometheus.prometheusSpec.exemplars | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enableFeatures }} + enableFeatures: +{{- range $enableFeatures := .Values.prometheus.prometheusSpec.enableFeatures }} + - {{ tpl $enableFeatures $ }} +{{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.scrapeInterval }} + scrapeInterval: {{ .Values.prometheus.prometheusSpec.scrapeInterval }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.scrapeTimeout }} + scrapeTimeout: {{ .Values.prometheus.prometheusSpec.scrapeTimeout }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.evaluationInterval }} + evaluationInterval: {{ .Values.prometheus.prometheusSpec.evaluationInterval }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.resources }} + resources: +{{ toYaml .Values.prometheus.prometheusSpec.resources | indent 4 }} +{{- end }} + retention: {{ .Values.prometheus.prometheusSpec.retention | quote }} +{{- if .Values.prometheus.prometheusSpec.retentionSize }} + retentionSize: {{ .Values.prometheus.prometheusSpec.retentionSize | quote }} +{{- end }} +{{- if eq .Values.prometheus.prometheusSpec.walCompression false }} + walCompression: false +{{ else }} + walCompression: true +{{- end }} +{{- if .Values.prometheus.prometheusSpec.routePrefix }} + routePrefix: {{ .Values.prometheus.prometheusSpec.routePrefix | quote }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.secrets }} + secrets: +{{ toYaml .Values.prometheus.prometheusSpec.secrets | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.configMaps }} + configMaps: +{{ toYaml .Values.prometheus.prometheusSpec.configMaps | indent 4 }} +{{- end }} + serviceAccountName: {{ template "kube-prometheus-stack.prometheus.serviceAccountName" . }} +{{- if .Values.prometheus.prometheusSpec.serviceMonitorSelector }} + serviceMonitorSelector: +{{ toYaml .Values.prometheus.prometheusSpec.serviceMonitorSelector | indent 4 }} +{{ else if .Values.prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues }} + serviceMonitorSelector: + matchLabels: + release: {{ $.Release.Name | quote }} +{{ else }} + serviceMonitorSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.serviceMonitorNamespaceSelector }} + serviceMonitorNamespaceSelector: +{{ toYaml .Values.prometheus.prometheusSpec.serviceMonitorNamespaceSelector | indent 4 }} +{{ else }} + serviceMonitorNamespaceSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.podMonitorSelector }} + podMonitorSelector: +{{ toYaml .Values.prometheus.prometheusSpec.podMonitorSelector | indent 4 }} +{{ else if .Values.prometheus.prometheusSpec.podMonitorSelectorNilUsesHelmValues }} + podMonitorSelector: + matchLabels: + release: {{ $.Release.Name | quote }} +{{ else }} + podMonitorSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.podMonitorNamespaceSelector }} + podMonitorNamespaceSelector: +{{ toYaml .Values.prometheus.prometheusSpec.podMonitorNamespaceSelector | indent 4 }} +{{ else }} + podMonitorNamespaceSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.probeSelector }} + probeSelector: +{{ toYaml .Values.prometheus.prometheusSpec.probeSelector | indent 4 }} +{{ else if .Values.prometheus.prometheusSpec.probeSelectorNilUsesHelmValues }} + probeSelector: + matchLabels: + release: {{ $.Release.Name | quote }} +{{ else }} + probeSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.probeNamespaceSelector }} + probeNamespaceSelector: +{{ toYaml .Values.prometheus.prometheusSpec.probeNamespaceSelector | indent 4 }} +{{ else }} + probeNamespaceSelector: {} +{{- end }} +{{- if (or .Values.prometheus.prometheusSpec.remoteRead .Values.prometheus.prometheusSpec.additionalRemoteRead) }} + remoteRead: +{{- if .Values.prometheus.prometheusSpec.remoteRead }} +{{ tpl (toYaml .Values.prometheus.prometheusSpec.remoteRead | indent 4) . }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalRemoteRead }} +{{ toYaml .Values.prometheus.prometheusSpec.additionalRemoteRead | indent 4 }} +{{- end }} +{{- end }} +{{- if (or .Values.prometheus.prometheusSpec.remoteWrite .Values.prometheus.prometheusSpec.additionalRemoteWrite) }} + remoteWrite: +{{- if .Values.prometheus.prometheusSpec.remoteWrite }} +{{ tpl (toYaml .Values.prometheus.prometheusSpec.remoteWrite | indent 4) . }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalRemoteWrite }} +{{ toYaml .Values.prometheus.prometheusSpec.additionalRemoteWrite | indent 4 }} +{{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.securityContext }} + securityContext: +{{ toYaml .Values.prometheus.prometheusSpec.securityContext | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.ruleNamespaceSelector }} + ruleNamespaceSelector: +{{ toYaml .Values.prometheus.prometheusSpec.ruleNamespaceSelector | indent 4 }} +{{ else }} + ruleNamespaceSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.ruleSelector }} + ruleSelector: +{{ toYaml .Values.prometheus.prometheusSpec.ruleSelector | indent 4}} +{{- else if .Values.prometheus.prometheusSpec.ruleSelectorNilUsesHelmValues }} + ruleSelector: + matchLabels: + release: {{ $.Release.Name | quote }} +{{ else }} + ruleSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.storageSpec }} + storage: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.storageSpec | indent 4) . }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.podMetadata }} + podMetadata: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.podMetadata | indent 4) . }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.query }} + query: +{{ toYaml .Values.prometheus.prometheusSpec.query | indent 4}} +{{- end }} +{{- if or .Values.prometheus.prometheusSpec.podAntiAffinity .Values.prometheus.prometheusSpec.affinity }} + affinity: +{{- if .Values.prometheus.prometheusSpec.affinity }} +{{ toYaml .Values.prometheus.prometheusSpec.affinity | indent 4 }} +{{- end }} +{{- if eq .Values.prometheus.prometheusSpec.podAntiAffinity "hard" }} + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: {{ .Values.prometheus.prometheusSpec.podAntiAffinityTopologyKey }} + labelSelector: + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [prometheus]} + - {key: prometheus, operator: In, values: [{{ template "kube-prometheus-stack.fullname" . }}-prometheus]} +{{- else if eq .Values.prometheus.prometheusSpec.podAntiAffinity "soft" }} + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: {{ .Values.prometheus.prometheusSpec.podAntiAffinityTopologyKey }} + labelSelector: + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [prometheus]} + - {key: prometheus, operator: In, values: [{{ template "kube-prometheus-stack.fullname" . }}-prometheus]} +{{- end }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 4 }} +{{- if .Values.prometheus.prometheusSpec.tolerations }} +{{ toYaml .Values.prometheus.prometheusSpec.tolerations | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.topologySpreadConstraints }} + topologySpreadConstraints: +{{ toYaml .Values.prometheus.prometheusSpec.topologySpreadConstraints | indent 4 }} +{{- end }} +{{- if .Values.global.imagePullSecrets }} + imagePullSecrets: +{{ include "kube-prometheus-stack.imagePullSecrets" . | trim | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalScrapeConfigs }} + additionalScrapeConfigs: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-scrape-confg + key: additional-scrape-configs.yaml +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalScrapeConfigsSecret.enabled }} + additionalScrapeConfigs: + name: {{ .Values.prometheus.prometheusSpec.additionalScrapeConfigsSecret.name }} + key: {{ .Values.prometheus.prometheusSpec.additionalScrapeConfigsSecret.key }} +{{- end }} +{{- if or .Values.prometheus.prometheusSpec.additionalAlertManagerConfigs .Values.prometheus.prometheusSpec.additionalAlertManagerConfigsSecret }} + additionalAlertManagerConfigs: +{{- if .Values.prometheus.prometheusSpec.additionalAlertManagerConfigs }} + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-am-confg + key: additional-alertmanager-configs.yaml +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalAlertManagerConfigsSecret }} + name: {{ .Values.prometheus.prometheusSpec.additionalAlertManagerConfigsSecret.name }} + key: {{ .Values.prometheus.prometheusSpec.additionalAlertManagerConfigsSecret.key }} + {{- if hasKey .Values.prometheus.prometheusSpec.additionalAlertManagerConfigsSecret "optional" }} + optional: {{ .Values.prometheus.prometheusSpec.additionalAlertManagerConfigsSecret.optional }} + {{- end }} +{{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalAlertRelabelConfigs }} + additionalAlertRelabelConfigs: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-am-relabel-confg + key: additional-alert-relabel-configs.yaml +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalAlertRelabelConfigsSecret }} + additionalAlertRelabelConfigs: + name: {{ .Values.prometheus.prometheusSpec.additionalAlertRelabelConfigsSecret.name }} + key: {{ .Values.prometheus.prometheusSpec.additionalAlertRelabelConfigsSecret.key }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.containers }} + containers: +{{ tpl .Values.prometheus.prometheusSpec.containers $ | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.initContainers }} + initContainers: +{{ toYaml .Values.prometheus.prometheusSpec.initContainers | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.priorityClassName }} + priorityClassName: {{ .Values.prometheus.prometheusSpec.priorityClassName }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.thanos }} + thanos: +{{ toYaml .Values.prometheus.prometheusSpec.thanos | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.disableCompaction }} + disableCompaction: {{ .Values.prometheus.prometheusSpec.disableCompaction }} +{{- end }} + portName: {{ .Values.prometheus.prometheusSpec.portName }} +{{- if .Values.prometheus.prometheusSpec.volumes }} + volumes: +{{ toYaml .Values.prometheus.prometheusSpec.volumes | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.volumeMounts }} + volumeMounts: +{{ toYaml .Values.prometheus.prometheusSpec.volumeMounts | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.arbitraryFSAccessThroughSMs }} + arbitraryFSAccessThroughSMs: +{{ toYaml .Values.prometheus.prometheusSpec.arbitraryFSAccessThroughSMs | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.overrideHonorLabels }} + overrideHonorLabels: {{ .Values.prometheus.prometheusSpec.overrideHonorLabels }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.overrideHonorTimestamps }} + overrideHonorTimestamps: {{ .Values.prometheus.prometheusSpec.overrideHonorTimestamps }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + ignoreNamespaceSelectors: {{ .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedNamespaceLabel }} + enforcedNamespaceLabel: {{ .Values.prometheus.prometheusSpec.enforcedNamespaceLabel }} +{{- $prometheusDefaultRulesExcludedFromEnforce := (include "rules.names" .) | fromYaml }} + prometheusRulesExcludedFromEnforce: +{{- range $prometheusDefaultRulesExcludedFromEnforce.rules }} + - ruleNamespace: "{{ template "kube-prometheus-stack.namespace" $ }}" + ruleName: "{{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) . | trunc 63 | trimSuffix "-" }}" +{{- end }} +{{- if .Values.prometheus.prometheusSpec.prometheusRulesExcludedFromEnforce }} +{{ toYaml .Values.prometheus.prometheusSpec.prometheusRulesExcludedFromEnforce | indent 4 }} +{{- end }} + excludedFromEnforcement: +{{- range $prometheusDefaultRulesExcludedFromEnforce.rules }} + - resource: prometheusrules + namespace: "{{ template "kube-prometheus-stack.namespace" $ }}" + name: "{{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) . | trunc 63 | trimSuffix "-" }}" +{{- end }} +{{- if .Values.prometheus.prometheusSpec.excludedFromEnforcement }} +{{ tpl (toYaml .Values.prometheus.prometheusSpec.excludedFromEnforcement | indent 4) . }} +{{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.queryLogFile }} + queryLogFile: {{ .Values.prometheus.prometheusSpec.queryLogFile }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedSampleLimit }} + enforcedSampleLimit: {{ .Values.prometheus.prometheusSpec.enforcedSampleLimit }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedTargetLimit }} + enforcedTargetLimit: {{ .Values.prometheus.prometheusSpec.enforcedTargetLimit }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedLabelLimit }} + enforcedLabelLimit: {{ .Values.prometheus.prometheusSpec.enforcedLabelLimit }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedLabelNameLengthLimit }} + enforcedLabelNameLengthLimit: {{ .Values.prometheus.prometheusSpec.enforcedLabelNameLengthLimit }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedLabelValueLengthLimit}} + enforcedLabelValueLengthLimit: {{ .Values.prometheus.prometheusSpec.enforcedLabelValueLengthLimit }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.allowOverlappingBlocks }} + allowOverlappingBlocks: {{ .Values.prometheus.prometheusSpec.allowOverlappingBlocks }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.minReadySeconds }} + minReadySeconds: {{ .Values.prometheus.prometheusSpec.minReadySeconds }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/psp-clusterrole.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/psp-clusterrole.yaml new file mode 100644 index 0000000000..0eb974eb4c --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/psp-clusterrole.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.prometheus.enabled .Values.global.rbac.create .Values.global.cattle.psp.enabled }} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-psp + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +rules: +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if semverCompare "> 1.15.0-0" $kubeTargetVersion }} +- apiGroups: ['policy'] +{{- else }} +- apiGroups: ['extensions'] +{{- end }} + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "kube-prometheus-stack.fullname" . }}-prometheus +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/psp-clusterrolebinding.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/psp-clusterrolebinding.yaml new file mode 100644 index 0000000000..ce11e5f62d --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/psp-clusterrolebinding.yaml @@ -0,0 +1,18 @@ +{{- if and .Values.prometheus.enabled .Values.global.rbac.create .Values.global.cattle.psp.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-psp + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-psp +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.prometheus.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} + diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/psp.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/psp.yaml new file mode 100644 index 0000000000..9a60d8ec38 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/psp.yaml @@ -0,0 +1,56 @@ +{{- if and .Values.prometheus.enabled .Values.global.rbac.create .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{- if .Values.global.rbac.pspAnnotations }} + annotations: +{{ toYaml .Values.global.rbac.pspAnnotations | indent 4 }} +{{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + privileged: false + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' +{{- if .Values.prometheus.podSecurityPolicy.volumes }} +{{ toYaml .Values.prometheus.podSecurityPolicy.volumes | indent 4 }} +{{- end }} + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + # Permits the container to run with root privileges as well. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + readOnlyRootFilesystem: false +{{- if .Values.prometheus.podSecurityPolicy.allowedCapabilities }} + allowedCapabilities: +{{ toYaml .Values.prometheus.podSecurityPolicy.allowedCapabilities | indent 4 }} +{{- end }} +{{- if .Values.prometheus.podSecurityPolicy.allowedHostPaths }} + allowedHostPaths: +{{ toYaml .Values.prometheus.podSecurityPolicy.allowedHostPaths | indent 4 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/alertmanager.rules.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/alertmanager.rules.yaml new file mode 100644 index 0000000000..5e7c548f61 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/alertmanager.rules.yaml @@ -0,0 +1,217 @@ +{{- /* +Generated from 'alertmanager.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/alertmanager-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.alertmanager }} +{{- $alertmanagerJob := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "alertmanager" }} +{{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} +{{- if and .Values.alertmanager.enabled .Values.alertmanager.serviceMonitor.selfMonitor }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "alertmanager.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: alertmanager.rules + rules: +{{- if not (.Values.defaultRules.disabled.AlertmanagerFailedReload | default false) }} + - alert: AlertmanagerFailedReload + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Configuration has failed to load for {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod{{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerfailedreload + summary: Reloading an Alertmanager configuration has failed. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + max_over_time(alertmanager_config_last_reload_successful{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m]) == 0 + for: 10m + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerMembersInconsistent | default false) }} + - alert: AlertmanagerMembersInconsistent + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Alertmanager {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod{{`}}`}} has only found {{`{{`}} $value {{`}}`}} members of the {{`{{`}}$labels.job{{`}}`}} cluster. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagermembersinconsistent + summary: A member of an Alertmanager cluster has not found all other cluster members. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + max_over_time(alertmanager_cluster_members{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m]) + < on (namespace,service) group_left + count by (namespace,service) (max_over_time(alertmanager_cluster_members{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m])) + for: 15m + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerFailedToSendAlerts | default false) }} + - alert: AlertmanagerFailedToSendAlerts + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Alertmanager {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod{{`}}`}} failed to send {{`{{`}} $value | humanizePercentage {{`}}`}} of notifications to {{`{{`}} $labels.integration {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerfailedtosendalerts + summary: An Alertmanager instance failed to send notifications. + expr: |- + ( + rate(alertmanager_notifications_failed_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m]) + / + rate(alertmanager_notifications_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m]) + ) + > 0.01 + for: 5m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerClusterFailedToSendAlerts | default false) }} + - alert: AlertmanagerClusterFailedToSendAlerts + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: The minimum notification failure rate to {{`{{`}} $labels.integration {{`}}`}} sent from any instance in the {{`{{`}}$labels.job{{`}}`}} cluster is {{`{{`}} $value | humanizePercentage {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerclusterfailedtosendalerts + summary: All Alertmanager instances in a cluster failed to send notifications to a critical integration. + expr: |- + min by (namespace,service, integration) ( + rate(alertmanager_notifications_failed_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}", integration=~`.*`}[5m]) + / + rate(alertmanager_notifications_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}", integration=~`.*`}[5m]) + ) + > 0.01 + for: 5m + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerClusterFailedToSendAlerts | default false) }} + - alert: AlertmanagerClusterFailedToSendAlerts + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: The minimum notification failure rate to {{`{{`}} $labels.integration {{`}}`}} sent from any instance in the {{`{{`}}$labels.job{{`}}`}} cluster is {{`{{`}} $value | humanizePercentage {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerclusterfailedtosendalerts + summary: All Alertmanager instances in a cluster failed to send notifications to a non-critical integration. + expr: |- + min by (namespace,service, integration) ( + rate(alertmanager_notifications_failed_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}", integration!~`.*`}[5m]) + / + rate(alertmanager_notifications_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}", integration!~`.*`}[5m]) + ) + > 0.01 + for: 5m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerConfigInconsistent | default false) }} + - alert: AlertmanagerConfigInconsistent + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Alertmanager instances within the {{`{{`}}$labels.job{{`}}`}} cluster have different configurations. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerconfiginconsistent + summary: Alertmanager instances within the same cluster have different configurations. + expr: |- + count by (namespace,service) ( + count_values by (namespace,service) ("config_hash", alertmanager_config_hash{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}) + ) + != 1 + for: 20m + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerClusterDown | default false) }} + - alert: AlertmanagerClusterDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: '{{`{{`}} $value | humanizePercentage {{`}}`}} of Alertmanager instances within the {{`{{`}}$labels.job{{`}}`}} cluster have been up for less than half of the last 5m.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerclusterdown + summary: Half or more of the Alertmanager instances within the same cluster are down. + expr: |- + ( + count by (namespace,service) ( + avg_over_time(up{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m]) < 0.5 + ) + / + count by (namespace,service) ( + up{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"} + ) + ) + >= 0.5 + for: 5m + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerClusterCrashlooping | default false) }} + - alert: AlertmanagerClusterCrashlooping + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: '{{`{{`}} $value | humanizePercentage {{`}}`}} of Alertmanager instances within the {{`{{`}}$labels.job{{`}}`}} cluster have restarted at least 5 times in the last 10m.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerclustercrashlooping + summary: Half or more of the Alertmanager instances within the same cluster are crashlooping. + expr: |- + ( + count by (namespace,service) ( + changes(process_start_time_seconds{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[10m]) > 4 + ) + / + count by (namespace,service) ( + up{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"} + ) + ) + >= 0.5 + for: 5m + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/config-reloaders.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/config-reloaders.yaml new file mode 100644 index 0000000000..37109eb0be --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/config-reloaders.yaml @@ -0,0 +1,46 @@ +{{- /* +Generated from 'config-reloaders' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/prometheusOperator-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.configReloaders }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "config-reloaders" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: config-reloaders + rules: +{{- if not (.Values.defaultRules.disabled.ConfigReloaderSidecarErrors | default false) }} + - alert: ConfigReloaderSidecarErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: 'Errors encountered while the {{`{{`}}$labels.pod{{`}}`}} config-reloader sidecar attempts to sync config in {{`{{`}}$labels.namespace{{`}}`}} namespace. + + As a result, configuration for service running in {{`{{`}}$labels.pod{{`}}`}} may be stale and cannot be updated anymore.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/configreloadersidecarerrors + summary: config-reloader sidecar has not had a successful reload for 10m + expr: max_over_time(reloader_last_reload_successful{namespace=~".+"}[5m]) == 0 + for: 10m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/etcd.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/etcd.yaml new file mode 100644 index 0000000000..1caa193959 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/etcd.yaml @@ -0,0 +1,296 @@ +{{- /* +Generated from 'etcd' group from https://raw.githubusercontent.com/etcd-io/etcd/main/contrib/mixin/mixin.libsonnet +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.etcd }} +{{- if (include "exporter.kubeEtcd.enabled" .)}} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "etcd" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: etcd + rules: +{{- if not (.Values.defaultRules.disabled.etcdMembersDown | default false) }} + - alert: etcdMembersDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": members are down ({{`{{`}} $value {{`}}`}}).' + summary: etcd cluster members are down. + expr: |- + max without (endpoint) ( + sum without (instance) (up{job=~".*etcd.*"} == bool 0) + or + count without (To) ( + sum without (instance) (rate(etcd_network_peer_sent_failures_total{job=~".*etcd.*"}[120s])) > 0.01 + ) + ) + > 0 + for: 10m + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdInsufficientMembers | default false) }} + - alert: etcdInsufficientMembers + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": insufficient members ({{`{{`}} $value {{`}}`}}).' + summary: etcd cluster has insufficient number of members. + expr: sum(up{job=~".*etcd.*"} == bool 1) without (instance) < ((count(up{job=~".*etcd.*"}) without (instance) + 1) / 2) + for: 3m + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdNoLeader | default false) }} + - alert: etcdNoLeader + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": member {{`{{`}} $labels.instance {{`}}`}} has no leader.' + summary: etcd cluster has no leader. + expr: etcd_server_has_leader{job=~".*etcd.*"} == 0 + for: 1m + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighNumberOfLeaderChanges | default false) }} + - alert: etcdHighNumberOfLeaderChanges + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}} leader changes within the last 15 minutes. Frequent elections may be a sign of insufficient resources, high network latency, or disruptions by other components and should be investigated.' + summary: etcd cluster has high number of leader changes. + expr: increase((max without (instance) (etcd_server_leader_changes_seen_total{job=~".*etcd.*"}) or 0*absent(etcd_server_leader_changes_seen_total{job=~".*etcd.*"}))[15m:1m]) >= 4 + for: 5m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighNumberOfFailedGRPCRequests | default false) }} + - alert: etcdHighNumberOfFailedGRPCRequests + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}}% of requests for {{`{{`}} $labels.grpc_method {{`}}`}} failed on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster has high number of failed grpc requests. + expr: |- + 100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) without (grpc_type, grpc_code) + / + sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) without (grpc_type, grpc_code) + > 1 + for: 10m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighNumberOfFailedGRPCRequests | default false) }} + - alert: etcdHighNumberOfFailedGRPCRequests + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}}% of requests for {{`{{`}} $labels.grpc_method {{`}}`}} failed on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster has high number of failed grpc requests. + expr: |- + 100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) without (grpc_type, grpc_code) + / + sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) without (grpc_type, grpc_code) + > 5 + for: 5m + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdGRPCRequestsSlow | default false) }} + - alert: etcdGRPCRequestsSlow + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": 99th percentile of gRPC requests is {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}} for {{`{{`}} $labels.grpc_method {{`}}`}} method.' + summary: etcd grpc requests are slow + expr: |- + histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job=~".*etcd.*", grpc_method!="Defragment", grpc_type="unary"}[5m])) without(grpc_type)) + > 0.15 + for: 10m + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdMemberCommunicationSlow | default false) }} + - alert: etcdMemberCommunicationSlow + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": member communication with {{`{{`}} $labels.To {{`}}`}} is taking {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster member communication is slow. + expr: |- + histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket{job=~".*etcd.*"}[5m])) + > 0.15 + for: 10m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighNumberOfFailedProposals | default false) }} + - alert: etcdHighNumberOfFailedProposals + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}} proposal failures within the last 30 minutes on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster has high number of proposal failures. + expr: rate(etcd_server_proposals_failed_total{job=~".*etcd.*"}[15m]) > 5 + for: 15m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighFsyncDurations | default false) }} + - alert: etcdHighFsyncDurations + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": 99th percentile fsync durations are {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster 99th percentile fsync durations are too high. + expr: |- + histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m])) + > 0.5 + for: 10m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighFsyncDurations | default false) }} + - alert: etcdHighFsyncDurations + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": 99th percentile fsync durations are {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster 99th percentile fsync durations are too high. + expr: |- + histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m])) + > 1 + for: 10m + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighCommitDurations | default false) }} + - alert: etcdHighCommitDurations + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": 99th percentile commit durations {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster 99th percentile commit durations are too high. + expr: |- + histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket{job=~".*etcd.*"}[5m])) + > 0.25 + for: 10m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdDatabaseQuotaLowSpace | default false) }} + - alert: etcdDatabaseQuotaLowSpace + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": database size exceeds the defined quota on etcd instance {{`{{`}} $labels.instance {{`}}`}}, please defrag or increase the quota as the writes to etcd will be disabled when it is full.' + summary: etcd cluster database is running full. + expr: (last_over_time(etcd_mvcc_db_total_size_in_bytes[5m]) / last_over_time(etcd_server_quota_backend_bytes[5m]))*100 > 95 + for: 10m + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdExcessiveDatabaseGrowth | default false) }} + - alert: etcdExcessiveDatabaseGrowth + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": Predicting running out of disk space in the next four hours, based on write observations within the past four hours on etcd instance {{`{{`}} $labels.instance {{`}}`}}, please check as it might be disruptive.' + summary: etcd cluster database growing very fast. + expr: predict_linear(etcd_mvcc_db_total_size_in_bytes[4h], 4*60*60) > etcd_server_quota_backend_bytes + for: 10m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdDatabaseHighFragmentationRatio | default false) }} + - alert: etcdDatabaseHighFragmentationRatio + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": database size in use on instance {{`{{`}} $labels.instance {{`}}`}} is {{`{{`}} $value | humanizePercentage {{`}}`}} of the actual allocated disk space, please run defragmentation (e.g. etcdctl defrag) to retrieve the unused fragmented disk space.' + runbook_url: https://etcd.io/docs/v3.5/op-guide/maintenance/#defragmentation + summary: etcd database size in use is less than 50% of the actual allocated storage. + expr: (last_over_time(etcd_mvcc_db_total_size_in_use_in_bytes[5m]) / last_over_time(etcd_mvcc_db_total_size_in_bytes[5m])) < 0.5 + for: 10m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/general.rules.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/general.rules.yaml new file mode 100644 index 0000000000..7ab648bc02 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/general.rules.yaml @@ -0,0 +1,98 @@ +{{- /* +Generated from 'general.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kubePrometheus-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.general }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "general.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: general.rules + rules: +{{- if not (.Values.defaultRules.disabled.TargetDown | default false) }} + - alert: TargetDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: '{{`{{`}} printf "%.4g" $value {{`}}`}}% of the {{`{{`}} $labels.job {{`}}`}}/{{`{{`}} $labels.service {{`}}`}} targets in {{`{{`}} $labels.namespace {{`}}`}} namespace are down.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/general/targetdown + summary: One or more targets are unreachable. + expr: 100 * (count(up == 0) BY (job, namespace, service) / count(up) BY (job, namespace, service)) > 10 + for: 10m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.Watchdog | default false) }} + - alert: Watchdog + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: 'This is an alert meant to ensure that the entire alerting pipeline is functional. + + This alert is always firing, therefore it should always be firing in Alertmanager + + and always fire against a receiver. There are integrations with various notification + + mechanisms that send a notification when this alert is not firing. For example the + + "DeadMansSnitch" integration in PagerDuty. + + ' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/general/watchdog + summary: An alert that should always be firing to certify that Alertmanager is working properly. + expr: vector(1) + labels: + severity: none +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.InfoInhibitor | default false) }} + - alert: InfoInhibitor + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: 'This is an alert that is used to inhibit info alerts. + + By themselves, the info-level alerts are sometimes very noisy, but they are relevant when combined with + + other alerts. + + This alert fires whenever there''s a severity="info" alert, and stops firing when another alert with a + + severity of ''warning'' or ''critical'' starts firing on the same namespace. + + This alert should be routed to a null receiver and configured to inhibit alerts with severity="info". + + ' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/general/infoinhibitor + summary: Info-level alert inhibition. + expr: ALERTS{severity = "info"} == 1 unless on(namespace) ALERTS{alertname != "InfoInhibitor", severity =~ "warning|critical", alertstate="firing"} == 1 + labels: + severity: none +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/k8s.rules.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/k8s.rules.yaml new file mode 100644 index 0000000000..c3e97b66ef --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/k8s.rules.yaml @@ -0,0 +1,173 @@ +{{- /* +Generated from 'k8s.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8s }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules + rules: + - expr: |- + sum by (cluster, namespace, pod, container) ( + irate(container_cpu_usage_seconds_total{job="{{ include "exporter.kubelet.jobName" . }}", metrics_path="/metrics/cadvisor", image!=""}[5m]) + ) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( + 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate + - expr: |- + container_memory_working_set_bytes{job="{{ include "exporter.kubelet.jobName" . }}", metrics_path="/metrics/cadvisor", image!=""} + * on (namespace, pod) group_left(node) topk by(namespace, pod) (1, + max by(namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_working_set_bytes + - expr: |- + container_memory_rss{job="{{ include "exporter.kubelet.jobName" . }}", metrics_path="/metrics/cadvisor", image!=""} + * on (namespace, pod) group_left(node) topk by(namespace, pod) (1, + max by(namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_rss + - expr: |- + container_memory_cache{job="{{ include "exporter.kubelet.jobName" . }}", metrics_path="/metrics/cadvisor", image!=""} + * on (namespace, pod) group_left(node) topk by(namespace, pod) (1, + max by(namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_cache + - expr: |- + container_memory_swap{job="{{ include "exporter.kubelet.jobName" . }}", metrics_path="/metrics/cadvisor", image!=""} + * on (namespace, pod) group_left(node) topk by(namespace, pod) (1, + max by(namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_swap + - expr: |- + kube_pod_container_resource_requests{resource="memory",job="kube-state-metrics"} * on (namespace, pod, cluster) + group_left() max by (namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_memory:active:kube_pod_container_resource_requests + - expr: |- + sum by (namespace, cluster) ( + sum by (namespace, pod, cluster) ( + max by (namespace, pod, container, cluster) ( + kube_pod_container_resource_requests{resource="memory",job="kube-state-metrics"} + ) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_memory:kube_pod_container_resource_requests:sum + - expr: |- + kube_pod_container_resource_requests{resource="cpu",job="kube-state-metrics"} * on (namespace, pod, cluster) + group_left() max by (namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests + - expr: |- + sum by (namespace, cluster) ( + sum by (namespace, pod, cluster) ( + max by (namespace, pod, container, cluster) ( + kube_pod_container_resource_requests{resource="cpu",job="kube-state-metrics"} + ) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_cpu:kube_pod_container_resource_requests:sum + - expr: |- + kube_pod_container_resource_limits{resource="memory",job="kube-state-metrics"} * on (namespace, pod, cluster) + group_left() max by (namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_memory:active:kube_pod_container_resource_limits + - expr: |- + sum by (namespace, cluster) ( + sum by (namespace, pod, cluster) ( + max by (namespace, pod, container, cluster) ( + kube_pod_container_resource_limits{resource="memory",job="kube-state-metrics"} + ) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_memory:kube_pod_container_resource_limits:sum + - expr: |- + kube_pod_container_resource_limits{resource="cpu",job="kube-state-metrics"} * on (namespace, pod, cluster) + group_left() max by (namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits + - expr: |- + sum by (namespace, cluster) ( + sum by (namespace, pod, cluster) ( + max by (namespace, pod, container, cluster) ( + kube_pod_container_resource_limits{resource="cpu",job="kube-state-metrics"} + ) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_cpu:kube_pod_container_resource_limits:sum + - expr: |- + max by (cluster, namespace, workload, pod) ( + label_replace( + label_replace( + kube_pod_owner{job="kube-state-metrics", owner_kind="ReplicaSet"}, + "replicaset", "$1", "owner_name", "(.*)" + ) * on(replicaset, namespace) group_left(owner_name) topk by(replicaset, namespace) ( + 1, max by (replicaset, namespace, owner_name) ( + kube_replicaset_owner{job="kube-state-metrics"} + ) + ), + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: deployment + record: namespace_workload_pod:kube_pod_owner:relabel + - expr: |- + max by (cluster, namespace, workload, pod) ( + label_replace( + kube_pod_owner{job="kube-state-metrics", owner_kind="DaemonSet"}, + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: daemonset + record: namespace_workload_pod:kube_pod_owner:relabel + - expr: |- + max by (cluster, namespace, workload, pod) ( + label_replace( + kube_pod_owner{job="kube-state-metrics", owner_kind="StatefulSet"}, + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: statefulset + record: namespace_workload_pod:kube_pod_owner:relabel + - expr: |- + max by (cluster, namespace, workload, pod) ( + label_replace( + kube_pod_owner{job="kube-state-metrics", owner_kind="Job"}, + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: job + record: namespace_workload_pod:kube_pod_owner:relabel +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kube-apiserver-availability.rules.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kube-apiserver-availability.rules.yaml new file mode 100644 index 0000000000..aa648b437e --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kube-apiserver-availability.rules.yaml @@ -0,0 +1,136 @@ +{{- /* +Generated from 'kube-apiserver-availability.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeApiServer.enabled .Values.defaultRules.rules.kubeApiserverAvailability }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-apiserver-availability.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - interval: 3m + name: kube-apiserver-availability.rules + rules: + - expr: avg_over_time(code_verb:apiserver_request_total:increase1h[30d]) * 24 * 30 + record: code_verb:apiserver_request_total:increase30d + - expr: sum by (cluster, code) (code_verb:apiserver_request_total:increase30d{verb=~"LIST|GET"}) + labels: + verb: read + record: code:apiserver_request_total:increase30d + - expr: sum by (cluster, code) (code_verb:apiserver_request_total:increase30d{verb=~"POST|PUT|PATCH|DELETE"}) + labels: + verb: write + record: code:apiserver_request_total:increase30d + - expr: sum by (cluster, verb, scope) (increase(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count[1h])) + record: cluster_verb_scope:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count:increase1h + - expr: sum by (cluster, verb, scope) (avg_over_time(cluster_verb_scope:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count:increase1h[30d]) * 24 * 30) + record: cluster_verb_scope:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count:increase30d + - expr: sum by (cluster, verb, scope, le) (increase(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket[1h])) + record: cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase1h + - expr: sum by (cluster, verb, scope, le) (avg_over_time(cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase1h[30d]) * 24 * 30) + record: cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d + - expr: |- + 1 - ( + ( + # write too slow + sum by (cluster) (cluster_verb_scope:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count:increase30d{verb=~"POST|PUT|PATCH|DELETE"}) + - + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"POST|PUT|PATCH|DELETE",le="1"}) + ) + + ( + # read too slow + sum by (cluster) (cluster_verb_scope:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count:increase30d{verb=~"LIST|GET"}) + - + ( + ( + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope=~"resource|",le="1"}) + or + vector(0) + ) + + + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="namespace",le="5"}) + + + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="cluster",le="30"}) + ) + ) + + # errors + sum by (cluster) (code:apiserver_request_total:increase30d{code=~"5.."} or vector(0)) + ) + / + sum by (cluster) (code:apiserver_request_total:increase30d) + labels: + verb: all + record: apiserver_request:availability30d + - expr: |- + 1 - ( + sum by (cluster) (cluster_verb_scope:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count:increase30d{verb=~"LIST|GET"}) + - + ( + # too slow + ( + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope=~"resource|",le="1"}) + or + vector(0) + ) + + + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="namespace",le="5"}) + + + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="cluster",le="30"}) + ) + + + # errors + sum by (cluster) (code:apiserver_request_total:increase30d{verb="read",code=~"5.."} or vector(0)) + ) + / + sum by (cluster) (code:apiserver_request_total:increase30d{verb="read"}) + labels: + verb: read + record: apiserver_request:availability30d + - expr: |- + 1 - ( + ( + # too slow + sum by (cluster) (cluster_verb_scope:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count:increase30d{verb=~"POST|PUT|PATCH|DELETE"}) + - + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"POST|PUT|PATCH|DELETE",le="1"}) + ) + + + # errors + sum by (cluster) (code:apiserver_request_total:increase30d{verb="write",code=~"5.."} or vector(0)) + ) + / + sum by (cluster) (code:apiserver_request_total:increase30d{verb="write"}) + labels: + verb: write + record: apiserver_request:availability30d + - expr: sum by (cluster,code,resource) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[5m])) + labels: + verb: read + record: code_resource:apiserver_request_total:rate5m + - expr: sum by (cluster,code,resource) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[5m])) + labels: + verb: write + record: code_resource:apiserver_request_total:rate5m + - expr: sum by (cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb=~"LIST|GET|POST|PUT|PATCH|DELETE",code=~"2.."}[1h])) + record: code_verb:apiserver_request_total:increase1h + - expr: sum by (cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb=~"LIST|GET|POST|PUT|PATCH|DELETE",code=~"3.."}[1h])) + record: code_verb:apiserver_request_total:increase1h + - expr: sum by (cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb=~"LIST|GET|POST|PUT|PATCH|DELETE",code=~"4.."}[1h])) + record: code_verb:apiserver_request_total:increase1h + - expr: sum by (cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb=~"LIST|GET|POST|PUT|PATCH|DELETE",code=~"5.."}[1h])) + record: code_verb:apiserver_request_total:increase1h +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kube-apiserver-burnrate.rules.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kube-apiserver-burnrate.rules.yaml new file mode 100644 index 0000000000..1cac2da6e0 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kube-apiserver-burnrate.rules.yaml @@ -0,0 +1,328 @@ +{{- /* +Generated from 'kube-apiserver-burnrate.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeApiServer.enabled .Values.defaultRules.rules.kubeApiserverBurnrate }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-apiserver-burnrate.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-apiserver-burnrate.rules + rules: + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[1d])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[1d])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[1d])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[1d])) + ) + ) + + + # errors + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[1d])) + ) + / + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[1d])) + labels: + verb: read + record: apiserver_request:burnrate1d + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[1h])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[1h])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[1h])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[1h])) + ) + ) + + + # errors + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[1h])) + ) + / + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[1h])) + labels: + verb: read + record: apiserver_request:burnrate1h + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[2h])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[2h])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[2h])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[2h])) + ) + ) + + + # errors + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[2h])) + ) + / + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[2h])) + labels: + verb: read + record: apiserver_request:burnrate2h + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[30m])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[30m])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[30m])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[30m])) + ) + ) + + + # errors + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[30m])) + ) + / + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[30m])) + labels: + verb: read + record: apiserver_request:burnrate30m + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[3d])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[3d])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[3d])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[3d])) + ) + ) + + + # errors + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[3d])) + ) + / + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[3d])) + labels: + verb: read + record: apiserver_request:burnrate3d + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[5m])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[5m])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[5m])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[5m])) + ) + ) + + + # errors + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[5m])) + ) + / + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[5m])) + labels: + verb: read + record: apiserver_request:burnrate5m + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[6h])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[6h])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[6h])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[6h])) + ) + ) + + + # errors + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[6h])) + ) + / + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[6h])) + labels: + verb: read + record: apiserver_request:burnrate6h + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[1d])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[1d])) + ) + + + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[1d])) + ) + / + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1d])) + labels: + verb: write + record: apiserver_request:burnrate1d + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[1h])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[1h])) + ) + + + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[1h])) + ) + / + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1h])) + labels: + verb: write + record: apiserver_request:burnrate1h + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[2h])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[2h])) + ) + + + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[2h])) + ) + / + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[2h])) + labels: + verb: write + record: apiserver_request:burnrate2h + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[30m])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[30m])) + ) + + + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[30m])) + ) + / + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[30m])) + labels: + verb: write + record: apiserver_request:burnrate30m + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[3d])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[3d])) + ) + + + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[3d])) + ) + / + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[3d])) + labels: + verb: write + record: apiserver_request:burnrate3d + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[5m])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[5m])) + ) + + + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[5m])) + ) + / + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[5m])) + labels: + verb: write + record: apiserver_request:burnrate5m + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[6h])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[6h])) + ) + + + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[6h])) + ) + / + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[6h])) + labels: + verb: write + record: apiserver_request:burnrate6h +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kube-apiserver-histogram.rules.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kube-apiserver-histogram.rules.yaml new file mode 100644 index 0000000000..9d2ea4d1e0 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kube-apiserver-histogram.rules.yaml @@ -0,0 +1,37 @@ +{{- /* +Generated from 'kube-apiserver-histogram.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeApiServer.enabled .Values.defaultRules.rules.kubeApiserverHistogram }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-apiserver-histogram.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-apiserver-histogram.rules + rules: + - expr: histogram_quantile(0.99, sum by (cluster, le, resource) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[5m]))) > 0 + labels: + quantile: '0.99' + verb: read + record: cluster_quantile:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.99, sum by (cluster, le, resource) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[5m]))) > 0 + labels: + quantile: '0.99' + verb: write + record: cluster_quantile:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds:histogram_quantile +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kube-apiserver-slos.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kube-apiserver-slos.yaml new file mode 100644 index 0000000000..867fe20db4 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kube-apiserver-slos.yaml @@ -0,0 +1,115 @@ +{{- /* +Generated from 'kube-apiserver-slos' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeApiServer.enabled .Values.defaultRules.rules.kubeApiserverSlos }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-apiserver-slos" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-apiserver-slos + rules: +{{- if not (.Values.defaultRules.disabled.KubeAPIErrorBudgetBurn | default false) }} + - alert: KubeAPIErrorBudgetBurn + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: The API server is burning too much error budget. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeapierrorbudgetburn + summary: The API server is burning too much error budget. + expr: |- + sum(apiserver_request:burnrate1h) > (14.40 * 0.01000) + and + sum(apiserver_request:burnrate5m) > (14.40 * 0.01000) + for: 2m + labels: + long: 1h + severity: critical + short: 5m +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeAPIErrorBudgetBurn | default false) }} + - alert: KubeAPIErrorBudgetBurn + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: The API server is burning too much error budget. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeapierrorbudgetburn + summary: The API server is burning too much error budget. + expr: |- + sum(apiserver_request:burnrate6h) > (6.00 * 0.01000) + and + sum(apiserver_request:burnrate30m) > (6.00 * 0.01000) + for: 15m + labels: + long: 6h + severity: critical + short: 30m +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeAPIErrorBudgetBurn | default false) }} + - alert: KubeAPIErrorBudgetBurn + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: The API server is burning too much error budget. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeapierrorbudgetburn + summary: The API server is burning too much error budget. + expr: |- + sum(apiserver_request:burnrate1d) > (3.00 * 0.01000) + and + sum(apiserver_request:burnrate2h) > (3.00 * 0.01000) + for: 1h + labels: + long: 1d + severity: warning + short: 2h +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeAPIErrorBudgetBurn | default false) }} + - alert: KubeAPIErrorBudgetBurn + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: The API server is burning too much error budget. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeapierrorbudgetburn + summary: The API server is burning too much error budget. + expr: |- + sum(apiserver_request:burnrate3d) > (1.00 * 0.01000) + and + sum(apiserver_request:burnrate6h) > (1.00 * 0.01000) + for: 3h + labels: + long: 3d + severity: warning + short: 6h +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kube-prometheus-general.rules.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kube-prometheus-general.rules.yaml new file mode 100644 index 0000000000..78a3db1cfc --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kube-prometheus-general.rules.yaml @@ -0,0 +1,31 @@ +{{- /* +Generated from 'kube-prometheus-general.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kubePrometheus-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubePrometheusGeneral }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-prometheus-general.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-prometheus-general.rules + rules: + - expr: count without(instance, pod, node) (up == 1) + record: count:up1 + - expr: count without(instance, pod, node) (up == 0) + record: count:up0 +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kube-prometheus-node-recording.rules.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kube-prometheus-node-recording.rules.yaml new file mode 100644 index 0000000000..0cd0ba5bf8 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kube-prometheus-node-recording.rules.yaml @@ -0,0 +1,39 @@ +{{- /* +Generated from 'kube-prometheus-node-recording.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kubePrometheus-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubePrometheusNodeRecording }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-prometheus-node-recording.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-prometheus-node-recording.rules + rules: + - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal"}[3m])) BY (instance) + record: instance:node_cpu:rate:sum + - expr: sum(rate(node_network_receive_bytes_total[3m])) BY (instance) + record: instance:node_network_receive_bytes:rate:sum + - expr: sum(rate(node_network_transmit_bytes_total[3m])) BY (instance) + record: instance:node_network_transmit_bytes:rate:sum + - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal"}[5m])) WITHOUT (cpu, mode) / ON(instance) GROUP_LEFT() count(sum(node_cpu_seconds_total) BY (instance, cpu)) BY (instance) + record: instance:node_cpu:ratio + - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal"}[5m])) + record: cluster:node_cpu:sum_rate5m + - expr: cluster:node_cpu:sum_rate5m / count(sum(node_cpu_seconds_total) BY (instance, cpu)) + record: cluster:node_cpu:ratio +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kube-scheduler.rules.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kube-scheduler.rules.yaml new file mode 100644 index 0000000000..efa1593d6a --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kube-scheduler.rules.yaml @@ -0,0 +1,65 @@ +{{- /* +Generated from 'kube-scheduler.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubeScheduler }} +{{- if (include "exporter.kubeScheduler.enabled" .)}} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-scheduler.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-scheduler.rules + rules: + - expr: histogram_quantile(0.99, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.99' + record: cluster_quantile:scheduler_e2e_scheduling_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.99, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.99' + record: cluster_quantile:scheduler_scheduling_algorithm_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.99, sum(rate(scheduler_binding_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.99' + record: cluster_quantile:scheduler_binding_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.9, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.9' + record: cluster_quantile:scheduler_e2e_scheduling_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.9, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.9' + record: cluster_quantile:scheduler_scheduling_algorithm_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.9, sum(rate(scheduler_binding_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.9' + record: cluster_quantile:scheduler_binding_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.5, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.5' + record: cluster_quantile:scheduler_e2e_scheduling_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.5, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.5' + record: cluster_quantile:scheduler_scheduling_algorithm_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.5, sum(rate(scheduler_binding_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.5' + record: cluster_quantile:scheduler_binding_duration_seconds:histogram_quantile +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kube-state-metrics.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kube-state-metrics.yaml new file mode 100644 index 0000000000..7547436a77 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kube-state-metrics.yaml @@ -0,0 +1,107 @@ +{{- /* +Generated from 'kube-state-metrics' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kubeStateMetrics-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubeStateMetrics }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-state-metrics" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-state-metrics + rules: +{{- if not (.Values.defaultRules.disabled.KubeStateMetricsListErrors | default false) }} + - alert: KubeStateMetricsListErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: kube-state-metrics is experiencing errors at an elevated rate in list operations. This is likely causing it to not be able to expose metrics about Kubernetes objects correctly or at all. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kube-state-metrics/kubestatemetricslisterrors + summary: kube-state-metrics is experiencing errors in list operations. + expr: |- + (sum(rate(kube_state_metrics_list_total{job="kube-state-metrics",result="error"}[5m])) + / + sum(rate(kube_state_metrics_list_total{job="kube-state-metrics"}[5m]))) + > 0.01 + for: 15m + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeStateMetricsWatchErrors | default false) }} + - alert: KubeStateMetricsWatchErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: kube-state-metrics is experiencing errors at an elevated rate in watch operations. This is likely causing it to not be able to expose metrics about Kubernetes objects correctly or at all. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kube-state-metrics/kubestatemetricswatcherrors + summary: kube-state-metrics is experiencing errors in watch operations. + expr: |- + (sum(rate(kube_state_metrics_watch_total{job="kube-state-metrics",result="error"}[5m])) + / + sum(rate(kube_state_metrics_watch_total{job="kube-state-metrics"}[5m]))) + > 0.01 + for: 15m + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeStateMetricsShardingMismatch | default false) }} + - alert: KubeStateMetricsShardingMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: kube-state-metrics pods are running with different --total-shards configuration, some Kubernetes objects may be exposed multiple times or not exposed at all. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kube-state-metrics/kubestatemetricsshardingmismatch + summary: kube-state-metrics sharding is misconfigured. + expr: stdvar (kube_state_metrics_total_shards{job="kube-state-metrics"}) != 0 + for: 15m + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeStateMetricsShardsMissing | default false) }} + - alert: KubeStateMetricsShardsMissing + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: kube-state-metrics shards are missing, some Kubernetes objects are not being exposed. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kube-state-metrics/kubestatemetricsshardsmissing + summary: kube-state-metrics shards are missing. + expr: |- + 2^max(kube_state_metrics_total_shards{job="kube-state-metrics"}) - 1 + - + sum( 2 ^ max by (shard_ordinal) (kube_state_metrics_shard_ordinal{job="kube-state-metrics"}) ) + != 0 + for: 15m + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kubelet.rules.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kubelet.rules.yaml new file mode 100644 index 0000000000..613c68c91a --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kubelet.rules.yaml @@ -0,0 +1,41 @@ +{{- /* +Generated from 'kubelet.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubelet }} +{{- if (include "exporter.kubelet.enabled" .)}} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubelet.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubelet.rules + rules: + - expr: histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_duration_seconds_bucket[5m])) by (cluster, instance, le) * on(cluster, instance) group_left(node) kubelet_node_name{job="{{ include "exporter.kubelet.jobName" . }}", metrics_path="/metrics"}) + labels: + quantile: '0.99' + record: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.9, sum(rate(kubelet_pleg_relist_duration_seconds_bucket[5m])) by (cluster, instance, le) * on(cluster, instance) group_left(node) kubelet_node_name{job="{{ include "exporter.kubelet.jobName" . }}", metrics_path="/metrics"}) + labels: + quantile: '0.9' + record: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.5, sum(rate(kubelet_pleg_relist_duration_seconds_bucket[5m])) by (cluster, instance, le) * on(cluster, instance) group_left(node) kubelet_node_name{job="{{ include "exporter.kubelet.jobName" . }}", metrics_path="/metrics"}) + labels: + quantile: '0.5' + record: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kubernetes-apps.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kubernetes-apps.yaml new file mode 100644 index 0000000000..d13185e33c --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kubernetes-apps.yaml @@ -0,0 +1,375 @@ +{{- /* +Generated from 'kubernetes-apps' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesApps }} +{{- $targetNamespace := .Values.defaultRules.appNamespacesTarget }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-apps" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-apps + rules: +{{- if not (.Values.defaultRules.disabled.KubePodCrashLooping | default false) }} + - alert: KubePodCrashLooping + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: 'Pod {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}} ({{`{{`}} $labels.container {{`}}`}}) is in waiting state (reason: "CrashLoopBackOff").' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepodcrashlooping + summary: Pod is crash looping. + expr: max_over_time(kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff", job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}[5m]) >= 1 + for: 15m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubePodNotReady | default false) }} + - alert: KubePodNotReady + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Pod {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}} has been in a non-ready state for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepodnotready + summary: Pod has been in a non-ready state for more than 15 minutes. + expr: |- + sum by (namespace, pod, cluster) ( + max by(namespace, pod, cluster) ( + kube_pod_status_phase{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}", phase=~"Pending|Unknown"} + ) * on(namespace, pod, cluster) group_left(owner_kind) topk by(namespace, pod, cluster) ( + 1, max by(namespace, pod, owner_kind, cluster) (kube_pod_owner{owner_kind!="Job"}) + ) + ) > 0 + for: 15m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeDeploymentGenerationMismatch | default false) }} + - alert: KubeDeploymentGenerationMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Deployment generation for {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.deployment {{`}}`}} does not match, this indicates that the Deployment has failed but has not been rolled back. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubedeploymentgenerationmismatch + summary: Deployment generation mismatch due to possible roll-back + expr: |- + kube_deployment_status_observed_generation{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + != + kube_deployment_metadata_generation{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + for: 15m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeDeploymentReplicasMismatch | default false) }} + - alert: KubeDeploymentReplicasMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Deployment {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.deployment {{`}}`}} has not matched the expected number of replicas for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubedeploymentreplicasmismatch + summary: Deployment has not matched the expected number of replicas. + expr: |- + ( + kube_deployment_spec_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + > + kube_deployment_status_replicas_available{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + ) and ( + changes(kube_deployment_status_replicas_updated{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}[10m]) + == + 0 + ) + for: 15m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeStatefulSetReplicasMismatch | default false) }} + - alert: KubeStatefulSetReplicasMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: StatefulSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.statefulset {{`}}`}} has not matched the expected number of replicas for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubestatefulsetreplicasmismatch + summary: Deployment has not matched the expected number of replicas. + expr: |- + ( + kube_statefulset_status_replicas_ready{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + != + kube_statefulset_status_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + ) and ( + changes(kube_statefulset_status_replicas_updated{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}[10m]) + == + 0 + ) + for: 15m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeStatefulSetGenerationMismatch | default false) }} + - alert: KubeStatefulSetGenerationMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: StatefulSet generation for {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.statefulset {{`}}`}} does not match, this indicates that the StatefulSet has failed but has not been rolled back. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubestatefulsetgenerationmismatch + summary: StatefulSet generation mismatch due to possible roll-back + expr: |- + kube_statefulset_status_observed_generation{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + != + kube_statefulset_metadata_generation{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + for: 15m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeStatefulSetUpdateNotRolledOut | default false) }} + - alert: KubeStatefulSetUpdateNotRolledOut + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: StatefulSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.statefulset {{`}}`}} update has not been rolled out. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubestatefulsetupdatenotrolledout + summary: StatefulSet update has not been rolled out. + expr: |- + ( + max without (revision) ( + kube_statefulset_status_current_revision{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + unless + kube_statefulset_status_update_revision{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + ) + * + ( + kube_statefulset_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + != + kube_statefulset_status_replicas_updated{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + ) + ) and ( + changes(kube_statefulset_status_replicas_updated{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}[5m]) + == + 0 + ) + for: 15m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeDaemonSetRolloutStuck | default false) }} + - alert: KubeDaemonSetRolloutStuck + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: DaemonSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.daemonset {{`}}`}} has not finished or progressed for at least 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubedaemonsetrolloutstuck + summary: DaemonSet rollout is stuck. + expr: |- + ( + ( + kube_daemonset_status_current_number_scheduled{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + != + kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + ) or ( + kube_daemonset_status_number_misscheduled{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + != + 0 + ) or ( + kube_daemonset_status_updated_number_scheduled{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + != + kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + ) or ( + kube_daemonset_status_number_available{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + != + kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + ) + ) and ( + changes(kube_daemonset_status_updated_number_scheduled{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}[5m]) + == + 0 + ) + for: 15m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeContainerWaiting | default false) }} + - alert: KubeContainerWaiting + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: pod/{{`{{`}} $labels.pod {{`}}`}} in namespace {{`{{`}} $labels.namespace {{`}}`}} on container {{`{{`}} $labels.container{{`}}`}} has been in waiting state for longer than 1 hour. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubecontainerwaiting + summary: Pod container waiting longer than 1 hour + expr: sum by (namespace, pod, container, cluster) (kube_pod_container_status_waiting_reason{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}) > 0 + for: 1h + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeDaemonSetNotScheduled | default false) }} + - alert: KubeDaemonSetNotScheduled + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: '{{`{{`}} $value {{`}}`}} Pods of DaemonSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.daemonset {{`}}`}} are not scheduled.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubedaemonsetnotscheduled + summary: DaemonSet pods are not scheduled. + expr: |- + kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + - + kube_daemonset_status_current_number_scheduled{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} > 0 + for: 10m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeDaemonSetMisScheduled | default false) }} + - alert: KubeDaemonSetMisScheduled + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: '{{`{{`}} $value {{`}}`}} Pods of DaemonSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.daemonset {{`}}`}} are running where they are not supposed to run.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubedaemonsetmisscheduled + summary: DaemonSet pods are misscheduled. + expr: kube_daemonset_status_number_misscheduled{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} > 0 + for: 15m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeJobNotCompleted | default false) }} + - alert: KubeJobNotCompleted + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Job {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.job_name {{`}}`}} is taking more than {{`{{`}} "43200" | humanizeDuration {{`}}`}} to complete. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubejobnotcompleted + summary: Job did not complete in time + expr: |- + time() - max by(namespace, job_name, cluster) (kube_job_status_start_time{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + and + kube_job_status_active{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} > 0) > 43200 + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeJobFailed | default false) }} + - alert: KubeJobFailed + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Job {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.job_name {{`}}`}} failed to complete. Removing failed job after investigation should clear this alert. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubejobfailed + summary: Job failed to complete. + expr: kube_job_failed{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} > 0 + for: 15m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeHpaReplicasMismatch | default false) }} + - alert: KubeHpaReplicasMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: HPA {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.horizontalpodautoscaler {{`}}`}} has not matched the desired number of replicas for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubehpareplicasmismatch + summary: HPA has not matched desired number of replicas. + expr: |- + (kube_horizontalpodautoscaler_status_desired_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + != + kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}) + and + (kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + > + kube_horizontalpodautoscaler_spec_min_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}) + and + (kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + < + kube_horizontalpodautoscaler_spec_max_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}) + and + changes(kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}[15m]) == 0 + for: 15m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeHpaMaxedOut | default false) }} + - alert: KubeHpaMaxedOut + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: HPA {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.horizontalpodautoscaler {{`}}`}} has been running at max replicas for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubehpamaxedout + summary: HPA is running at max replicas + expr: |- + kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + == + kube_horizontalpodautoscaler_spec_max_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + for: 15m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kubernetes-resources.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kubernetes-resources.yaml new file mode 100644 index 0000000000..5fab8d7a3f --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kubernetes-resources.yaml @@ -0,0 +1,193 @@ +{{- /* +Generated from 'kubernetes-resources' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesResources }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-resources" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-resources + rules: +{{- if not (.Values.defaultRules.disabled.KubeCPUOvercommit | default false) }} + - alert: KubeCPUOvercommit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Cluster has overcommitted CPU resource requests for Pods by {{`{{`}} $value {{`}}`}} CPU shares and cannot tolerate node failure. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubecpuovercommit + summary: Cluster has overcommitted CPU resource requests. + expr: |- + sum(namespace_cpu:kube_pod_container_resource_requests:sum{}) - (sum(kube_node_status_allocatable{resource="cpu"}) - max(kube_node_status_allocatable{resource="cpu"})) > 0 + and + (sum(kube_node_status_allocatable{resource="cpu"}) - max(kube_node_status_allocatable{resource="cpu"})) > 0 + for: 10m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeMemoryOvercommit | default false) }} + - alert: KubeMemoryOvercommit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Cluster has overcommitted memory resource requests for Pods by {{`{{`}} $value | humanize {{`}}`}} bytes and cannot tolerate node failure. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubememoryovercommit + summary: Cluster has overcommitted memory resource requests. + expr: |- + sum(namespace_memory:kube_pod_container_resource_requests:sum{}) - (sum(kube_node_status_allocatable{resource="memory"}) - max(kube_node_status_allocatable{resource="memory"})) > 0 + and + (sum(kube_node_status_allocatable{resource="memory"}) - max(kube_node_status_allocatable{resource="memory"})) > 0 + for: 10m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeCPUQuotaOvercommit | default false) }} + - alert: KubeCPUQuotaOvercommit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Cluster has overcommitted CPU resource requests for Namespaces. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubecpuquotaovercommit + summary: Cluster has overcommitted CPU resource requests. + expr: |- + sum(min without(resource) (kube_resourcequota{job="kube-state-metrics", type="hard", resource=~"(cpu|requests.cpu)"})) + / + sum(kube_node_status_allocatable{resource="cpu", job="kube-state-metrics"}) + > 1.5 + for: 5m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeMemoryQuotaOvercommit | default false) }} + - alert: KubeMemoryQuotaOvercommit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Cluster has overcommitted memory resource requests for Namespaces. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubememoryquotaovercommit + summary: Cluster has overcommitted memory resource requests. + expr: |- + sum(min without(resource) (kube_resourcequota{job="kube-state-metrics", type="hard", resource=~"(memory|requests.memory)"})) + / + sum(kube_node_status_allocatable{resource="memory", job="kube-state-metrics"}) + > 1.5 + for: 5m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeQuotaAlmostFull | default false) }} + - alert: KubeQuotaAlmostFull + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Namespace {{`{{`}} $labels.namespace {{`}}`}} is using {{`{{`}} $value | humanizePercentage {{`}}`}} of its {{`{{`}} $labels.resource {{`}}`}} quota. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubequotaalmostfull + summary: Namespace quota is going to be full. + expr: |- + kube_resourcequota{job="kube-state-metrics", type="used"} + / ignoring(instance, job, type) + (kube_resourcequota{job="kube-state-metrics", type="hard"} > 0) + > 0.9 < 1 + for: 15m + labels: + severity: info +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeQuotaFullyUsed | default false) }} + - alert: KubeQuotaFullyUsed + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Namespace {{`{{`}} $labels.namespace {{`}}`}} is using {{`{{`}} $value | humanizePercentage {{`}}`}} of its {{`{{`}} $labels.resource {{`}}`}} quota. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubequotafullyused + summary: Namespace quota is fully used. + expr: |- + kube_resourcequota{job="kube-state-metrics", type="used"} + / ignoring(instance, job, type) + (kube_resourcequota{job="kube-state-metrics", type="hard"} > 0) + == 1 + for: 15m + labels: + severity: info +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeQuotaExceeded | default false) }} + - alert: KubeQuotaExceeded + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Namespace {{`{{`}} $labels.namespace {{`}}`}} is using {{`{{`}} $value | humanizePercentage {{`}}`}} of its {{`{{`}} $labels.resource {{`}}`}} quota. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubequotaexceeded + summary: Namespace quota has exceeded the limits. + expr: |- + kube_resourcequota{job="kube-state-metrics", type="used"} + / ignoring(instance, job, type) + (kube_resourcequota{job="kube-state-metrics", type="hard"} > 0) + > 1 + for: 15m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.CPUThrottlingHigh | default false) }} + - alert: CPUThrottlingHigh + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: '{{`{{`}} $value | humanizePercentage {{`}}`}} throttling of CPU in namespace {{`{{`}} $labels.namespace {{`}}`}} for container {{`{{`}} $labels.container {{`}}`}} in pod {{`{{`}} $labels.pod {{`}}`}}.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/cputhrottlinghigh + summary: Processes experience elevated CPU throttling. + expr: |- + sum(increase(container_cpu_cfs_throttled_periods_total{container!="", }[5m])) by (container, pod, namespace) + / + sum(increase(container_cpu_cfs_periods_total{}[5m])) by (container, pod, namespace) + > ( 25 / 100 ) + for: 15m + labels: + severity: info +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kubernetes-storage.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kubernetes-storage.yaml new file mode 100644 index 0000000000..7620061dad --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kubernetes-storage.yaml @@ -0,0 +1,160 @@ +{{- /* +Generated from 'kubernetes-storage' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesStorage }} +{{- $targetNamespace := .Values.defaultRules.appNamespacesTarget }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-storage" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-storage + rules: +{{- if not (.Values.defaultRules.disabled.KubePersistentVolumeFillingUp | default false) }} + - alert: KubePersistentVolumeFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: The PersistentVolume claimed by {{`{{`}} $labels.persistentvolumeclaim {{`}}`}} in Namespace {{`{{`}} $labels.namespace {{`}}`}} is only {{`{{`}} $value | humanizePercentage {{`}}`}} free. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepersistentvolumefillingup + summary: PersistentVolume is filling up. + expr: |- + kubelet_volume_stats_available_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + / + kubelet_volume_stats_capacity_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + < 0.03 + and + kubelet_volume_stats_used_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} > 0 + unless on(namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1 + unless on(namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1 + for: 1m + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubePersistentVolumeFillingUp | default false) }} + - alert: KubePersistentVolumeFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Based on recent sampling, the PersistentVolume claimed by {{`{{`}} $labels.persistentvolumeclaim {{`}}`}} in Namespace {{`{{`}} $labels.namespace {{`}}`}} is expected to fill up within four days. Currently {{`{{`}} $value | humanizePercentage {{`}}`}} is available. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepersistentvolumefillingup + summary: PersistentVolume is filling up. + expr: |- + ( + kubelet_volume_stats_available_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + / + kubelet_volume_stats_capacity_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + ) < 0.15 + and + kubelet_volume_stats_used_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} > 0 + and + predict_linear(kubelet_volume_stats_available_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"}[6h], 4 * 24 * 3600) < 0 + unless on(namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1 + unless on(namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1 + for: 1h + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubePersistentVolumeInodesFillingUp | default false) }} + - alert: KubePersistentVolumeInodesFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: The PersistentVolume claimed by {{`{{`}} $labels.persistentvolumeclaim {{`}}`}} in Namespace {{`{{`}} $labels.namespace {{`}}`}} only has {{`{{`}} $value | humanizePercentage {{`}}`}} free inodes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepersistentvolumeinodesfillingup + summary: PersistentVolumeInodes are filling up. + expr: |- + ( + kubelet_volume_stats_inodes_free{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + / + kubelet_volume_stats_inodes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + ) < 0.03 + and + kubelet_volume_stats_inodes_used{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} > 0 + unless on(namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1 + unless on(namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1 + for: 1m + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubePersistentVolumeInodesFillingUp | default false) }} + - alert: KubePersistentVolumeInodesFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Based on recent sampling, the PersistentVolume claimed by {{`{{`}} $labels.persistentvolumeclaim {{`}}`}} in Namespace {{`{{`}} $labels.namespace {{`}}`}} is expected to run out of inodes within four days. Currently {{`{{`}} $value | humanizePercentage {{`}}`}} of its inodes are free. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepersistentvolumeinodesfillingup + summary: PersistentVolumeInodes are filling up. + expr: |- + ( + kubelet_volume_stats_inodes_free{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + / + kubelet_volume_stats_inodes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + ) < 0.15 + and + kubelet_volume_stats_inodes_used{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} > 0 + and + predict_linear(kubelet_volume_stats_inodes_free{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"}[6h], 4 * 24 * 3600) < 0 + unless on(namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1 + unless on(namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1 + for: 1h + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubePersistentVolumeErrors | default false) }} + - alert: KubePersistentVolumeErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: The persistent volume {{`{{`}} $labels.persistentvolume {{`}}`}} has status {{`{{`}} $labels.phase {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepersistentvolumeerrors + summary: PersistentVolume is having issues with provisioning. + expr: kube_persistentvolume_status_phase{phase=~"Failed|Pending",job="kube-state-metrics"} > 0 + for: 5m + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml new file mode 100644 index 0000000000..aff07d6a66 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml @@ -0,0 +1,128 @@ +{{- /* +Generated from 'kubernetes-system-apiserver' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesSystem }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-system-apiserver" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-system-apiserver + rules: +{{- if not (.Values.defaultRules.disabled.KubeClientCertificateExpiration | default false) }} + - alert: KubeClientCertificateExpiration + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: A client certificate used to authenticate to kubernetes apiserver is expiring in less than 7.0 days. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeclientcertificateexpiration + summary: Client certificate is about to expire. + expr: apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and on(job) histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 604800 + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeClientCertificateExpiration | default false) }} + - alert: KubeClientCertificateExpiration + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: A client certificate used to authenticate to kubernetes apiserver is expiring in less than 24.0 hours. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeclientcertificateexpiration + summary: Client certificate is about to expire. + expr: apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and on(job) histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 86400 + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeAggregatedAPIErrors | default false) }} + - alert: KubeAggregatedAPIErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Kubernetes aggregated API {{`{{`}} $labels.name {{`}}`}}/{{`{{`}} $labels.namespace {{`}}`}} has reported errors. It has appeared unavailable {{`{{`}} $value | humanize {{`}}`}} times averaged over the past 10m. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeaggregatedapierrors + summary: Kubernetes aggregated API has reported errors. + expr: sum by(name, namespace, cluster)(increase(aggregator_unavailable_apiservice_total[10m])) > 4 + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeAggregatedAPIDown | default false) }} + - alert: KubeAggregatedAPIDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Kubernetes aggregated API {{`{{`}} $labels.name {{`}}`}}/{{`{{`}} $labels.namespace {{`}}`}} has been only {{`{{`}} $value | humanize {{`}}`}}% available over the last 10m. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeaggregatedapidown + summary: Kubernetes aggregated API is down. + expr: (1 - max by(name, namespace, cluster)(avg_over_time(aggregator_unavailable_apiservice[10m]))) * 100 < 85 + for: 5m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if .Values.kubeApiServer.enabled }} +{{- if not (.Values.defaultRules.disabled.KubeAPIDown | default false) }} + - alert: KubeAPIDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: KubeAPI has disappeared from Prometheus target discovery. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeapidown + summary: Target disappeared from Prometheus target discovery. + expr: absent(up{job="apiserver"} == 1) + for: 15m + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeAPITerminatedRequests | default false) }} + - alert: KubeAPITerminatedRequests + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: The kubernetes apiserver has terminated {{`{{`}} $value | humanizePercentage {{`}}`}} of its incoming requests. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeapiterminatedrequests + summary: The kubernetes apiserver has terminated {{`{{`}} $value | humanizePercentage {{`}}`}} of its incoming requests. + expr: sum(rate(apiserver_request_terminations_total{job="apiserver"}[10m])) / ( sum(rate(apiserver_request_total{job="apiserver"}[10m])) + sum(rate(apiserver_request_terminations_total{job="apiserver"}[10m])) ) > 0.20 + for: 5m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kubernetes-system-controller-manager.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kubernetes-system-controller-manager.yaml new file mode 100644 index 0000000000..0639ef0ebb --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kubernetes-system-controller-manager.yaml @@ -0,0 +1,47 @@ +{{- /* +Generated from 'kubernetes-system-controller-manager' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubeControllerManager }} +{{- if (include "exporter.kubeControllerManager.enabled" .)}} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-system-controller-manager" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-system-controller-manager + rules: +{{- if not (.Values.defaultRules.disabled.KubeControllerManagerDown | default false) }} + - alert: KubeControllerManagerDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: KubeControllerManager has disappeared from Prometheus target discovery. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubecontrollermanagerdown + summary: Target disappeared from Prometheus target discovery. + expr: absent(up{job="{{ include "exporter.kubeControllerManager.jobName" . }}"} == 1) + for: 15m + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} + diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kubernetes-system-kube-proxy.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kubernetes-system-kube-proxy.yaml new file mode 100644 index 0000000000..9b64455677 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kubernetes-system-kube-proxy.yaml @@ -0,0 +1,46 @@ +{{- /* +Generated from 'kubernetes-system-kube-proxy' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubeProxy }} +{{- if (include "exporter.kubeProxy.enabled" .)}} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-system-kube-proxy" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-system-kube-proxy + rules: +{{- if not (.Values.defaultRules.disabled.KubeProxyDown | default false) }} + - alert: KubeProxyDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: KubeProxy has disappeared from Prometheus target discovery. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeproxydown + summary: Target disappeared from Prometheus target discovery. + expr: absent(up{job="{{ include "exporter.kubeProxy.jobName" . }}"} == 1) + for: 15m + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml new file mode 100644 index 0000000000..b9036c6a46 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml @@ -0,0 +1,253 @@ +{{- /* +Generated from 'kubernetes-system-kubelet' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesSystem }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-system-kubelet" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-system-kubelet + rules: +{{- if not (.Values.defaultRules.disabled.KubeNodeNotReady | default false) }} + - alert: KubeNodeNotReady + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: '{{`{{`}} $labels.node {{`}}`}} has been unready for more than 15 minutes.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubenodenotready + summary: Node is not ready. + expr: kube_node_status_condition{job="kube-state-metrics",condition="Ready",status="true"} == 0 + for: 15m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeNodeUnreachable | default false) }} + - alert: KubeNodeUnreachable + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: '{{`{{`}} $labels.node {{`}}`}} is unreachable and some workloads may be rescheduled.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubenodeunreachable + summary: Node is unreachable. + expr: (kube_node_spec_taint{job="kube-state-metrics",key="node.kubernetes.io/unreachable",effect="NoSchedule"} unless ignoring(key,value) kube_node_spec_taint{job="kube-state-metrics",key=~"ToBeDeletedByClusterAutoscaler|cloud.google.com/impending-node-termination|aws-node-termination-handler/spot-itn"}) == 1 + for: 15m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletTooManyPods | default false) }} + - alert: KubeletTooManyPods + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Kubelet '{{`{{`}} $labels.node {{`}}`}}' is running at {{`{{`}} $value | humanizePercentage {{`}}`}} of its Pod capacity. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubelettoomanypods + summary: Kubelet is running at capacity. + expr: |- + count by(cluster, node) ( + (kube_pod_status_phase{job="kube-state-metrics",phase="Running"} == 1) * on(instance,pod,namespace,cluster) group_left(node) topk by(instance,pod,namespace,cluster) (1, kube_pod_info{job="kube-state-metrics"}) + ) + / + max by(cluster, node) ( + kube_node_status_capacity{job="kube-state-metrics",resource="pods"} != 1 + ) > 0.95 + for: 15m + labels: + severity: info +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeNodeReadinessFlapping | default false) }} + - alert: KubeNodeReadinessFlapping + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: The readiness status of node {{`{{`}} $labels.node {{`}}`}} has changed {{`{{`}} $value {{`}}`}} times in the last 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubenodereadinessflapping + summary: Node readiness status is flapping. + expr: sum(changes(kube_node_status_condition{status="true",condition="Ready"}[15m])) by (cluster, node) > 2 + for: 15m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletPlegDurationHigh | default false) }} + - alert: KubeletPlegDurationHigh + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: The Kubelet Pod Lifecycle Event Generator has a 99th percentile duration of {{`{{`}} $value {{`}}`}} seconds on node {{`{{`}} $labels.node {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletplegdurationhigh + summary: Kubelet Pod Lifecycle Event Generator is taking too long to relist. + expr: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile{quantile="0.99"} >= 10 + for: 5m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletPodStartUpLatencyHigh | default false) }} + - alert: KubeletPodStartUpLatencyHigh + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Kubelet Pod startup 99th percentile latency is {{`{{`}} $value {{`}}`}} seconds on node {{`{{`}} $labels.node {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletpodstartuplatencyhigh + summary: Kubelet Pod startup latency is too high. + expr: histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{job="{{ include "exporter.kubelet.jobName" . }}", metrics_path="/metrics"}[5m])) by (cluster, instance, le)) * on(cluster, instance) group_left(node) kubelet_node_name{job="{{ include "exporter.kubelet.jobName" . }}", metrics_path="/metrics"} > 60 + for: 15m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletClientCertificateExpiration | default false) }} + - alert: KubeletClientCertificateExpiration + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Client certificate for Kubelet on node {{`{{`}} $labels.node {{`}}`}} expires in {{`{{`}} $value | humanizeDuration {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletclientcertificateexpiration + summary: Kubelet client certificate is about to expire. + expr: kubelet_certificate_manager_client_ttl_seconds < 604800 + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletClientCertificateExpiration | default false) }} + - alert: KubeletClientCertificateExpiration + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Client certificate for Kubelet on node {{`{{`}} $labels.node {{`}}`}} expires in {{`{{`}} $value | humanizeDuration {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletclientcertificateexpiration + summary: Kubelet client certificate is about to expire. + expr: kubelet_certificate_manager_client_ttl_seconds < 86400 + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletServerCertificateExpiration | default false) }} + - alert: KubeletServerCertificateExpiration + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Server certificate for Kubelet on node {{`{{`}} $labels.node {{`}}`}} expires in {{`{{`}} $value | humanizeDuration {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletservercertificateexpiration + summary: Kubelet server certificate is about to expire. + expr: kubelet_certificate_manager_server_ttl_seconds < 604800 + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletServerCertificateExpiration | default false) }} + - alert: KubeletServerCertificateExpiration + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Server certificate for Kubelet on node {{`{{`}} $labels.node {{`}}`}} expires in {{`{{`}} $value | humanizeDuration {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletservercertificateexpiration + summary: Kubelet server certificate is about to expire. + expr: kubelet_certificate_manager_server_ttl_seconds < 86400 + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletClientCertificateRenewalErrors | default false) }} + - alert: KubeletClientCertificateRenewalErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Kubelet on node {{`{{`}} $labels.node {{`}}`}} has failed to renew its client certificate ({{`{{`}} $value | humanize {{`}}`}} errors in the last 5 minutes). + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletclientcertificaterenewalerrors + summary: Kubelet has failed to renew its client certificate. + expr: increase(kubelet_certificate_manager_client_expiration_renew_errors[5m]) > 0 + for: 15m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletServerCertificateRenewalErrors | default false) }} + - alert: KubeletServerCertificateRenewalErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Kubelet on node {{`{{`}} $labels.node {{`}}`}} has failed to renew its server certificate ({{`{{`}} $value | humanize {{`}}`}} errors in the last 5 minutes). + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletservercertificaterenewalerrors + summary: Kubelet has failed to renew its server certificate. + expr: increase(kubelet_server_expiration_renew_errors[5m]) > 0 + for: 15m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if (include "exporter.kubelet.enabled" .)}} +{{- if not (.Values.defaultRules.disabled.KubeletDown | default false) }} + - alert: KubeletDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Kubelet has disappeared from Prometheus target discovery. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletdown + summary: Target disappeared from Prometheus target discovery. + expr: absent(up{job="{{ include "exporter.kubelet.jobName" . }}", metrics_path="/metrics"} == 1) + for: 15m + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kubernetes-system-scheduler.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kubernetes-system-scheduler.yaml new file mode 100644 index 0000000000..283429cfd8 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kubernetes-system-scheduler.yaml @@ -0,0 +1,46 @@ +{{- /* +Generated from 'kubernetes-system-scheduler' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubeScheduler }} +{{- if (include "exporter.kubeScheduler.enabled" .)}} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-system-scheduler" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-system-scheduler + rules: +{{- if not (.Values.defaultRules.disabled.KubeSchedulerDown | default false) }} + - alert: KubeSchedulerDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: KubeScheduler has disappeared from Prometheus target discovery. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeschedulerdown + summary: Target disappeared from Prometheus target discovery. + expr: absent(up{job="{{ include "exporter.kubeScheduler.jobName" . }}"} == 1) + for: 15m + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kubernetes-system.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kubernetes-system.yaml new file mode 100644 index 0000000000..32605926c9 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/kubernetes-system.yaml @@ -0,0 +1,65 @@ +{{- /* +Generated from 'kubernetes-system' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesSystem }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-system" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-system + rules: +{{- if not (.Values.defaultRules.disabled.KubeVersionMismatch | default false) }} + - alert: KubeVersionMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: There are {{`{{`}} $value {{`}}`}} different semantic versions of Kubernetes components running. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeversionmismatch + summary: Different semantic versions of Kubernetes components running. + expr: count by (cluster) (count by (git_version, cluster) (label_replace(kubernetes_build_info{job!~"kube-dns|coredns"},"git_version","$1","git_version","(v[0-9]*.[0-9]*).*"))) > 1 + for: 15m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeClientErrors | default false) }} + - alert: KubeClientErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Kubernetes API server client '{{`{{`}} $labels.job {{`}}`}}/{{`{{`}} $labels.instance {{`}}`}}' is experiencing {{`{{`}} $value | humanizePercentage {{`}}`}} errors.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeclienterrors + summary: Kubernetes API server client is experiencing errors. + expr: |- + (sum(rate(rest_client_requests_total{code=~"5.."}[5m])) by (cluster, instance, job, namespace) + / + sum(rate(rest_client_requests_total[5m])) by (cluster, instance, job, namespace)) + > 0.01 + for: 15m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/node-exporter.rules.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/node-exporter.rules.yaml new file mode 100644 index 0000000000..c3cfe36ca2 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/node-exporter.rules.yaml @@ -0,0 +1,89 @@ +{{- /* +Generated from 'node-exporter.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/nodeExporter-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.nodeExporterRecording }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "node-exporter.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: node-exporter.rules + rules: + - expr: |- + count without (cpu, mode) ( + node_cpu_seconds_total{job="node-exporter",mode="idle"} + ) + record: instance:node_num_cpu:sum + - expr: |- + 1 - avg without (cpu) ( + sum without (mode) (rate(node_cpu_seconds_total{job="node-exporter", mode=~"idle|iowait|steal"}[5m])) + ) + record: instance:node_cpu_utilisation:rate5m + - expr: |- + ( + node_load1{job="node-exporter"} + / + instance:node_num_cpu:sum{job="node-exporter"} + ) + record: instance:node_load1_per_cpu:ratio + - expr: |- + 1 - ( + ( + node_memory_MemAvailable_bytes{job="node-exporter"} + or + ( + node_memory_Buffers_bytes{job="node-exporter"} + + + node_memory_Cached_bytes{job="node-exporter"} + + + node_memory_MemFree_bytes{job="node-exporter"} + + + node_memory_Slab_bytes{job="node-exporter"} + ) + ) + / + node_memory_MemTotal_bytes{job="node-exporter"} + ) + record: instance:node_memory_utilisation:ratio + - expr: rate(node_vmstat_pgmajfault{job="node-exporter"}[5m]) + record: instance:node_vmstat_pgmajfault:rate5m + - expr: rate(node_disk_io_time_seconds_total{job="node-exporter", device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)"}[5m]) + record: instance_device:node_disk_io_time_seconds:rate5m + - expr: rate(node_disk_io_time_weighted_seconds_total{job="node-exporter", device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)"}[5m]) + record: instance_device:node_disk_io_time_weighted_seconds:rate5m + - expr: |- + sum without (device) ( + rate(node_network_receive_bytes_total{job="node-exporter", device!="lo"}[5m]) + ) + record: instance:node_network_receive_bytes_excluding_lo:rate5m + - expr: |- + sum without (device) ( + rate(node_network_transmit_bytes_total{job="node-exporter", device!="lo"}[5m]) + ) + record: instance:node_network_transmit_bytes_excluding_lo:rate5m + - expr: |- + sum without (device) ( + rate(node_network_receive_drop_total{job="node-exporter", device!="lo"}[5m]) + ) + record: instance:node_network_receive_drop_excluding_lo:rate5m + - expr: |- + sum without (device) ( + rate(node_network_transmit_drop_total{job="node-exporter", device!="lo"}[5m]) + ) + record: instance:node_network_transmit_drop_excluding_lo:rate5m +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/node-exporter.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/node-exporter.yaml new file mode 100644 index 0000000000..2fa7e28d3c --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/node-exporter.yaml @@ -0,0 +1,398 @@ +{{- /* +Generated from 'node-exporter' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/nodeExporter-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.nodeExporterAlerting }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "node-exporter" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: node-exporter + rules: +{{- if not (.Values.defaultRules.disabled.NodeFilesystemSpaceFillingUp | default false) }} + - alert: NodeFilesystemSpaceFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left and is filling up. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemspacefillingup + summary: Filesystem is predicted to run out of space within the next 24 hours. + expr: |- + ( + node_filesystem_avail_bytes{job="node-exporter",fstype!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!=""} * 100 < 15 + and + predict_linear(node_filesystem_avail_bytes{job="node-exporter",fstype!=""}[6h], 24*60*60) < 0 + and + node_filesystem_readonly{job="node-exporter",fstype!=""} == 0 + ) + for: 1h + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemSpaceFillingUp | default false) }} + - alert: NodeFilesystemSpaceFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left and is filling up fast. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemspacefillingup + summary: Filesystem is predicted to run out of space within the next 4 hours. + expr: |- + ( + node_filesystem_avail_bytes{job="node-exporter",fstype!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!=""} * 100 < 10 + and + predict_linear(node_filesystem_avail_bytes{job="node-exporter",fstype!=""}[6h], 4*60*60) < 0 + and + node_filesystem_readonly{job="node-exporter",fstype!=""} == 0 + ) + for: 1h + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemAlmostOutOfSpace | default false) }} + - alert: NodeFilesystemAlmostOutOfSpace + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemalmostoutofspace + summary: Filesystem has less than 5% space left. + expr: |- + ( + node_filesystem_avail_bytes{job="node-exporter",fstype!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!=""} * 100 < 5 + and + node_filesystem_readonly{job="node-exporter",fstype!=""} == 0 + ) + for: 30m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemAlmostOutOfSpace | default false) }} + - alert: NodeFilesystemAlmostOutOfSpace + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemalmostoutofspace + summary: Filesystem has less than 3% space left. + expr: |- + ( + node_filesystem_avail_bytes{job="node-exporter",fstype!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!=""} * 100 < 3 + and + node_filesystem_readonly{job="node-exporter",fstype!=""} == 0 + ) + for: 30m + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemFilesFillingUp | default false) }} + - alert: NodeFilesystemFilesFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left and is filling up. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemfilesfillingup + summary: Filesystem is predicted to run out of inodes within the next 24 hours. + expr: |- + ( + node_filesystem_files_free{job="node-exporter",fstype!=""} / node_filesystem_files{job="node-exporter",fstype!=""} * 100 < 40 + and + predict_linear(node_filesystem_files_free{job="node-exporter",fstype!=""}[6h], 24*60*60) < 0 + and + node_filesystem_readonly{job="node-exporter",fstype!=""} == 0 + ) + for: 1h + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemFilesFillingUp | default false) }} + - alert: NodeFilesystemFilesFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left and is filling up fast. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemfilesfillingup + summary: Filesystem is predicted to run out of inodes within the next 4 hours. + expr: |- + ( + node_filesystem_files_free{job="node-exporter",fstype!=""} / node_filesystem_files{job="node-exporter",fstype!=""} * 100 < 20 + and + predict_linear(node_filesystem_files_free{job="node-exporter",fstype!=""}[6h], 4*60*60) < 0 + and + node_filesystem_readonly{job="node-exporter",fstype!=""} == 0 + ) + for: 1h + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemAlmostOutOfFiles | default false) }} + - alert: NodeFilesystemAlmostOutOfFiles + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemalmostoutoffiles + summary: Filesystem has less than 5% inodes left. + expr: |- + ( + node_filesystem_files_free{job="node-exporter",fstype!=""} / node_filesystem_files{job="node-exporter",fstype!=""} * 100 < 5 + and + node_filesystem_readonly{job="node-exporter",fstype!=""} == 0 + ) + for: 1h + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemAlmostOutOfFiles | default false) }} + - alert: NodeFilesystemAlmostOutOfFiles + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemalmostoutoffiles + summary: Filesystem has less than 3% inodes left. + expr: |- + ( + node_filesystem_files_free{job="node-exporter",fstype!=""} / node_filesystem_files{job="node-exporter",fstype!=""} * 100 < 3 + and + node_filesystem_readonly{job="node-exporter",fstype!=""} == 0 + ) + for: 1h + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeNetworkReceiveErrs | default false) }} + - alert: NodeNetworkReceiveErrs + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: '{{`{{`}} $labels.instance {{`}}`}} interface {{`{{`}} $labels.device {{`}}`}} has encountered {{`{{`}} printf "%.0f" $value {{`}}`}} receive errors in the last two minutes.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodenetworkreceiveerrs + summary: Network interface is reporting many receive errors. + expr: rate(node_network_receive_errs_total[2m]) / rate(node_network_receive_packets_total[2m]) > 0.01 + for: 1h + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeNetworkTransmitErrs | default false) }} + - alert: NodeNetworkTransmitErrs + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: '{{`{{`}} $labels.instance {{`}}`}} interface {{`{{`}} $labels.device {{`}}`}} has encountered {{`{{`}} printf "%.0f" $value {{`}}`}} transmit errors in the last two minutes.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodenetworktransmiterrs + summary: Network interface is reporting many transmit errors. + expr: rate(node_network_transmit_errs_total[2m]) / rate(node_network_transmit_packets_total[2m]) > 0.01 + for: 1h + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeHighNumberConntrackEntriesUsed | default false) }} + - alert: NodeHighNumberConntrackEntriesUsed + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: '{{`{{`}} $value | humanizePercentage {{`}}`}} of conntrack entries are used.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodehighnumberconntrackentriesused + summary: Number of conntrack are getting close to the limit. + expr: (node_nf_conntrack_entries / node_nf_conntrack_entries_limit) > 0.75 + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeTextFileCollectorScrapeError | default false) }} + - alert: NodeTextFileCollectorScrapeError + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Node Exporter text file collector failed to scrape. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodetextfilecollectorscrapeerror + summary: Node Exporter text file collector failed to scrape. + expr: node_textfile_scrape_error{job="node-exporter"} == 1 + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeClockSkewDetected | default false) }} + - alert: NodeClockSkewDetected + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Clock on {{`{{`}} $labels.instance {{`}}`}} is out of sync by more than 300s. Ensure NTP is configured correctly on this host. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodeclockskewdetected + summary: Clock skew detected. + expr: |- + ( + node_timex_offset_seconds{job="node-exporter"} > 0.05 + and + deriv(node_timex_offset_seconds{job="node-exporter"}[5m]) >= 0 + ) + or + ( + node_timex_offset_seconds{job="node-exporter"} < -0.05 + and + deriv(node_timex_offset_seconds{job="node-exporter"}[5m]) <= 0 + ) + for: 10m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeClockNotSynchronising | default false) }} + - alert: NodeClockNotSynchronising + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Clock on {{`{{`}} $labels.instance {{`}}`}} is not synchronising. Ensure NTP is configured on this host. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodeclocknotsynchronising + summary: Clock not synchronising. + expr: |- + min_over_time(node_timex_sync_status{job="node-exporter"}[5m]) == 0 + and + node_timex_maxerror_seconds{job="node-exporter"} >= 16 + for: 10m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeRAIDDegraded | default false) }} + - alert: NodeRAIDDegraded + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: RAID array '{{`{{`}} $labels.device {{`}}`}}' on {{`{{`}} $labels.instance {{`}}`}} is in degraded state due to one or more disks failures. Number of spare drives is insufficient to fix issue automatically. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/noderaiddegraded + summary: RAID Array is degraded + expr: node_md_disks_required{job="node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)"} - ignoring (state) (node_md_disks{state="active",job="node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)"}) > 0 + for: 15m + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeRAIDDiskFailure | default false) }} + - alert: NodeRAIDDiskFailure + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: At least one device in RAID array on {{`{{`}} $labels.instance {{`}}`}} failed. Array '{{`{{`}} $labels.device {{`}}`}}' needs attention and possibly a disk swap. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/noderaiddiskfailure + summary: Failed device in RAID array + expr: node_md_disks{state="failed",job="node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)"} > 0 + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFileDescriptorLimit | default false) }} + - alert: NodeFileDescriptorLimit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: File descriptors limit at {{`{{`}} $labels.instance {{`}}`}} is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}%. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefiledescriptorlimit + summary: Kernel is predicted to exhaust file descriptors limit soon. + expr: |- + ( + node_filefd_allocated{job="node-exporter"} * 100 / node_filefd_maximum{job="node-exporter"} > 70 + ) + for: 15m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFileDescriptorLimit | default false) }} + - alert: NodeFileDescriptorLimit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: File descriptors limit at {{`{{`}} $labels.instance {{`}}`}} is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}%. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefiledescriptorlimit + summary: Kernel is predicted to exhaust file descriptors limit soon. + expr: |- + ( + node_filefd_allocated{job="node-exporter"} * 100 / node_filefd_maximum{job="node-exporter"} > 90 + ) + for: 15m + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/node-network.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/node-network.yaml new file mode 100644 index 0000000000..9320973404 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/node-network.yaml @@ -0,0 +1,44 @@ +{{- /* +Generated from 'node-network' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kubePrometheus-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.network }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "node-network" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: node-network + rules: +{{- if not (.Values.defaultRules.disabled.NodeNetworkInterfaceFlapping | default false) }} + - alert: NodeNetworkInterfaceFlapping + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Network interface "{{`{{`}} $labels.device {{`}}`}}" changing its up status often on node-exporter {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}} + runbook_url: {{ .Values.defaultRules.runbookUrl }}/general/nodenetworkinterfaceflapping + summary: Network interface is often changing its status + expr: changes(node_network_up{job="node-exporter",device!~"veth.+"}[2m]) > 2 + for: 2m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/node.rules.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/node.rules.yaml new file mode 100644 index 0000000000..4f8da294fa --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/node.rules.yaml @@ -0,0 +1,55 @@ +{{- /* +Generated from 'node.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.node }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "node.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: node.rules + rules: + - expr: |- + topk by(cluster, namespace, pod) (1, + max by (cluster, node, namespace, pod) ( + label_replace(kube_pod_info{job="kube-state-metrics",node!=""}, "pod", "$1", "pod", "(.*)") + )) + record: 'node_namespace_pod:kube_pod_info:' + - expr: |- + count by (cluster, node) (sum by (node, cpu) ( + node_cpu_seconds_total{job="node-exporter"} + * on (namespace, pod) group_left(node) + topk by(namespace, pod) (1, node_namespace_pod:kube_pod_info:) + )) + record: node:node_num_cpu:sum + - expr: |- + sum( + node_memory_MemAvailable_bytes{job="node-exporter"} or + ( + node_memory_Buffers_bytes{job="node-exporter"} + + node_memory_Cached_bytes{job="node-exporter"} + + node_memory_MemFree_bytes{job="node-exporter"} + + node_memory_Slab_bytes{job="node-exporter"} + ) + ) by (cluster) + record: :node_memory_MemAvailable_bytes:sum + - expr: |- + sum(rate(node_cpu_seconds_total{job="node-exporter",mode!="idle",mode!="iowait",mode!="steal"}[5m])) / + count(sum(node_cpu_seconds_total{job="node-exporter"}) by (cluster, instance, cpu)) + record: cluster:node_cpu:ratio_rate5m +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/prometheus-operator.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/prometheus-operator.yaml new file mode 100644 index 0000000000..1c6b5c5d56 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/prometheus-operator.yaml @@ -0,0 +1,148 @@ +{{- /* +Generated from 'prometheus-operator' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/prometheusOperator-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.prometheusOperator }} +{{- $operatorJob := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "operator" }} +{{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "prometheus-operator" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: prometheus-operator + rules: +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorListErrors | default false) }} + - alert: PrometheusOperatorListErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Errors while performing List operations in controller {{`{{`}}$labels.controller{{`}}`}} in {{`{{`}}$labels.namespace{{`}}`}} namespace. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatorlisterrors + summary: Errors while performing list operations in controller. + expr: (sum by (controller,namespace) (rate(prometheus_operator_list_operations_failed_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[10m])) / sum by (controller,namespace) (rate(prometheus_operator_list_operations_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[10m]))) > 0.4 + for: 15m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorWatchErrors | default false) }} + - alert: PrometheusOperatorWatchErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Errors while performing watch operations in controller {{`{{`}}$labels.controller{{`}}`}} in {{`{{`}}$labels.namespace{{`}}`}} namespace. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatorwatcherrors + summary: Errors while performing watch operations in controller. + expr: (sum by (controller,namespace) (rate(prometheus_operator_watch_operations_failed_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m])) / sum by (controller,namespace) (rate(prometheus_operator_watch_operations_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]))) > 0.4 + for: 15m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorSyncFailed | default false) }} + - alert: PrometheusOperatorSyncFailed + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Controller {{`{{`}} $labels.controller {{`}}`}} in {{`{{`}} $labels.namespace {{`}}`}} namespace fails to reconcile {{`{{`}} $value {{`}}`}} objects. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatorsyncfailed + summary: Last controller reconciliation failed + expr: min_over_time(prometheus_operator_syncs{status="failed",job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: 10m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorReconcileErrors | default false) }} + - alert: PrometheusOperatorReconcileErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: '{{`{{`}} $value | humanizePercentage {{`}}`}} of reconciling operations failed for {{`{{`}} $labels.controller {{`}}`}} controller in {{`{{`}} $labels.namespace {{`}}`}} namespace.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatorreconcileerrors + summary: Errors while reconciling controller. + expr: (sum by (controller,namespace) (rate(prometheus_operator_reconcile_errors_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]))) / (sum by (controller,namespace) (rate(prometheus_operator_reconcile_operations_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]))) > 0.1 + for: 10m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorNodeLookupErrors | default false) }} + - alert: PrometheusOperatorNodeLookupErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Errors while reconciling Prometheus in {{`{{`}} $labels.namespace {{`}}`}} Namespace. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatornodelookuperrors + summary: Errors while reconciling Prometheus. + expr: rate(prometheus_operator_node_address_lookup_errors_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]) > 0.1 + for: 10m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorNotReady | default false) }} + - alert: PrometheusOperatorNotReady + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Prometheus operator in {{`{{`}} $labels.namespace {{`}}`}} namespace isn't ready to reconcile {{`{{`}} $labels.controller {{`}}`}} resources. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatornotready + summary: Prometheus operator not ready + expr: min by (controller,namespace) (max_over_time(prometheus_operator_ready{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]) == 0) + for: 5m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorRejectedResources | default false) }} + - alert: PrometheusOperatorRejectedResources + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Prometheus operator in {{`{{`}} $labels.namespace {{`}}`}} namespace rejected {{`{{`}} printf "%0.0f" $value {{`}}`}} {{`{{`}} $labels.controller {{`}}`}}/{{`{{`}} $labels.resource {{`}}`}} resources. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatorrejectedresources + summary: Resources rejected by Prometheus operator + expr: min_over_time(prometheus_operator_managed_resources{state="rejected",job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: 5m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/prometheus.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/prometheus.yaml new file mode 100644 index 0000000000..358ca7a4ef --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/rules-1.14/prometheus.yaml @@ -0,0 +1,448 @@ +{{- /* +Generated from 'prometheus' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/prometheus-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.prometheus }} +{{- $prometheusJob := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "prometheus" }} +{{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "prometheus" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: prometheus + rules: +{{- if not (.Values.defaultRules.disabled.PrometheusBadConfig | default false) }} + - alert: PrometheusBadConfig + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has failed to reload its configuration. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusbadconfig + summary: Failed Prometheus configuration reload. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + max_over_time(prometheus_config_last_reload_successful{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) == 0 + for: 10m + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusNotificationQueueRunningFull | default false) }} + - alert: PrometheusNotificationQueueRunningFull + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Alert notification queue of Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} is running full. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusnotificationqueuerunningfull + summary: Prometheus alert notification queue predicted to run full in less than 30m. + expr: |- + # Without min_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + ( + predict_linear(prometheus_notifications_queue_length{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m], 60 * 30) + > + min_over_time(prometheus_notifications_queue_capacity{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + ) + for: 15m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusErrorSendingAlertsToSomeAlertmanagers | default false) }} + - alert: PrometheusErrorSendingAlertsToSomeAlertmanagers + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: '{{`{{`}} printf "%.1f" $value {{`}}`}}% errors while sending alerts from Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} to Alertmanager {{`{{`}}$labels.alertmanager{{`}}`}}.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheuserrorsendingalertstosomealertmanagers + summary: Prometheus has encountered more than 1% errors sending alerts to a specific Alertmanager. + expr: |- + ( + rate(prometheus_notifications_errors_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + / + rate(prometheus_notifications_sent_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + ) + * 100 + > 1 + for: 15m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusNotConnectedToAlertmanagers | default false) }} + - alert: PrometheusNotConnectedToAlertmanagers + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} is not connected to any Alertmanagers. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusnotconnectedtoalertmanagers + summary: Prometheus is not connected to any Alertmanagers. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + max_over_time(prometheus_notifications_alertmanagers_discovered{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) < 1 + for: 10m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusTSDBReloadsFailing | default false) }} + - alert: PrometheusTSDBReloadsFailing + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has detected {{`{{`}}$value | humanize{{`}}`}} reload failures over the last 3h. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheustsdbreloadsfailing + summary: Prometheus has issues reloading blocks from disk. + expr: increase(prometheus_tsdb_reloads_failures_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[3h]) > 0 + for: 4h + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusTSDBCompactionsFailing | default false) }} + - alert: PrometheusTSDBCompactionsFailing + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has detected {{`{{`}}$value | humanize{{`}}`}} compaction failures over the last 3h. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheustsdbcompactionsfailing + summary: Prometheus has issues compacting blocks. + expr: increase(prometheus_tsdb_compactions_failed_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[3h]) > 0 + for: 4h + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusNotIngestingSamples | default false) }} + - alert: PrometheusNotIngestingSamples + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} is not ingesting samples. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusnotingestingsamples + summary: Prometheus is not ingesting samples. + expr: |- + ( + rate(prometheus_tsdb_head_samples_appended_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) <= 0 + and + ( + sum without(scrape_job) (prometheus_target_metadata_cache_entries{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}) > 0 + or + sum without(rule_group) (prometheus_rule_group_rules{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}) > 0 + ) + ) + for: 10m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusDuplicateTimestamps | default false) }} + - alert: PrometheusDuplicateTimestamps + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} is dropping {{`{{`}} printf "%.4g" $value {{`}}`}} samples/s with different values but duplicated timestamp. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusduplicatetimestamps + summary: Prometheus is dropping samples with duplicate timestamps. + expr: rate(prometheus_target_scrapes_sample_duplicate_timestamp_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: 10m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOutOfOrderTimestamps | default false) }} + - alert: PrometheusOutOfOrderTimestamps + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} is dropping {{`{{`}} printf "%.4g" $value {{`}}`}} samples/s with timestamps arriving out of order. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusoutofordertimestamps + summary: Prometheus drops samples with out-of-order timestamps. + expr: rate(prometheus_target_scrapes_sample_out_of_order_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: 10m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusRemoteStorageFailures | default false) }} + - alert: PrometheusRemoteStorageFailures + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} failed to send {{`{{`}} printf "%.1f" $value {{`}}`}}% of the samples to {{`{{`}} $labels.remote_name{{`}}`}}:{{`{{`}} $labels.url {{`}}`}} + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusremotestoragefailures + summary: Prometheus fails to send samples to remote storage. + expr: |- + ( + (rate(prometheus_remote_storage_failed_samples_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m])) + / + ( + (rate(prometheus_remote_storage_failed_samples_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m])) + + + (rate(prometheus_remote_storage_succeeded_samples_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) or rate(prometheus_remote_storage_samples_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m])) + ) + ) + * 100 + > 1 + for: 15m + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusRemoteWriteBehind | default false) }} + - alert: PrometheusRemoteWriteBehind + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} remote write is {{`{{`}} printf "%.1f" $value {{`}}`}}s behind for {{`{{`}} $labels.remote_name{{`}}`}}:{{`{{`}} $labels.url {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusremotewritebehind + summary: Prometheus remote write is behind. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + ( + max_over_time(prometheus_remote_storage_highest_timestamp_in_seconds{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + - ignoring(remote_name, url) group_right + max_over_time(prometheus_remote_storage_queue_highest_sent_timestamp_seconds{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + ) + > 120 + for: 15m + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusRemoteWriteDesiredShards | default false) }} + - alert: PrometheusRemoteWriteDesiredShards + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} remote write desired shards calculation wants to run {{`{{`}} $value {{`}}`}} shards for queue {{`{{`}} $labels.remote_name{{`}}`}}:{{`{{`}} $labels.url {{`}}`}}, which is more than the max of {{`{{`}} printf `prometheus_remote_storage_shards_max{instance="%s",job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}` $labels.instance | query | first | value {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusremotewritedesiredshards + summary: Prometheus remote write desired shards calculation wants to run more than configured max shards. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + ( + max_over_time(prometheus_remote_storage_shards_desired{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + > + max_over_time(prometheus_remote_storage_shards_max{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + ) + for: 15m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusRuleFailures | default false) }} + - alert: PrometheusRuleFailures + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has failed to evaluate {{`{{`}} printf "%.0f" $value {{`}}`}} rules in the last 5m. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusrulefailures + summary: Prometheus is failing rule evaluations. + expr: increase(prometheus_rule_evaluation_failures_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: 15m + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusMissingRuleEvaluations | default false) }} + - alert: PrometheusMissingRuleEvaluations + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has missed {{`{{`}} printf "%.0f" $value {{`}}`}} rule group evaluations in the last 5m. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusmissingruleevaluations + summary: Prometheus is missing rule evaluations due to slow rule group evaluation. + expr: increase(prometheus_rule_group_iterations_missed_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: 15m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusTargetLimitHit | default false) }} + - alert: PrometheusTargetLimitHit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has dropped {{`{{`}} printf "%.0f" $value {{`}}`}} targets because the number of targets exceeded the configured target_limit. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheustargetlimithit + summary: Prometheus has dropped targets because some scrape configs have exceeded the targets limit. + expr: increase(prometheus_target_scrape_pool_exceeded_target_limit_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: 15m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusLabelLimitHit | default false) }} + - alert: PrometheusLabelLimitHit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has dropped {{`{{`}} printf "%.0f" $value {{`}}`}} targets because some samples exceeded the configured label_limit, label_name_length_limit or label_value_length_limit. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheuslabellimithit + summary: Prometheus has dropped targets because some scrape configs have exceeded the labels limit. + expr: increase(prometheus_target_scrape_pool_exceeded_label_limits_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: 15m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusScrapeBodySizeLimitHit | default false) }} + - alert: PrometheusScrapeBodySizeLimitHit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has failed {{`{{`}} printf "%.0f" $value {{`}}`}} scrapes in the last 5m because some targets exceeded the configured body_size_limit. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusscrapebodysizelimithit + summary: Prometheus has dropped some targets that exceeded body size limit. + expr: increase(prometheus_target_scrapes_exceeded_body_size_limit_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: 15m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusScrapeSampleLimitHit | default false) }} + - alert: PrometheusScrapeSampleLimitHit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has failed {{`{{`}} printf "%.0f" $value {{`}}`}} scrapes in the last 5m because some targets exceeded the configured sample_limit. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusscrapesamplelimithit + summary: Prometheus has failed scrapes that have exceeded the configured sample limit. + expr: increase(prometheus_target_scrapes_exceeded_sample_limit_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: 15m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusTargetSyncFailure | default false) }} + - alert: PrometheusTargetSyncFailure + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: '{{`{{`}} printf "%.0f" $value {{`}}`}} targets in Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} have failed to sync because invalid configuration was supplied.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheustargetsyncfailure + summary: Prometheus has failed to sync targets. + expr: increase(prometheus_target_sync_failed_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[30m]) > 0 + for: 5m + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusHighQueryLoad | default false) }} + - alert: PrometheusHighQueryLoad + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} query API has less than 20% available capacity in its query engine for the last 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheushighqueryload + summary: Prometheus is reaching its maximum capacity serving concurrent requests. + expr: avg_over_time(prometheus_engine_queries{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) / max_over_time(prometheus_engine_queries_concurrent_max{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0.8 + for: 15m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusErrorSendingAlertsToAnyAlertmanager | default false) }} + - alert: PrometheusErrorSendingAlertsToAnyAlertmanager + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: '{{`{{`}} printf "%.1f" $value {{`}}`}}% minimum errors while sending alerts from Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} to any Alertmanager.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheuserrorsendingalertstoanyalertmanager + summary: Prometheus encounters more than 3% errors sending alerts to any Alertmanager. + expr: |- + min without (alertmanager) ( + rate(prometheus_notifications_errors_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}",alertmanager!~``}[5m]) + / + rate(prometheus_notifications_sent_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}",alertmanager!~``}[5m]) + ) + * 100 + > 3 + for: 15m + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/service.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/service.yaml new file mode 100644 index 0000000000..1e1f9c7b7f --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/service.yaml @@ -0,0 +1,64 @@ +{{- if .Values.prometheus.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + self-monitor: {{ .Values.prometheus.serviceMonitor.selfMonitor | quote }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.service.labels }} +{{ toYaml .Values.prometheus.service.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheus.service.annotations }} + annotations: +{{ toYaml .Values.prometheus.service.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.prometheus.service.clusterIP }} + clusterIP: {{ .Values.prometheus.service.clusterIP }} +{{- end }} +{{- if .Values.prometheus.service.externalIPs }} + externalIPs: +{{ toYaml .Values.prometheus.service.externalIPs | indent 4 }} +{{- end }} +{{- if .Values.prometheus.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.prometheus.service.loadBalancerIP }} +{{- end }} +{{- if .Values.prometheus.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.prometheus.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if ne .Values.prometheus.service.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.prometheus.service.externalTrafficPolicy }} +{{- end }} + ports: + - name: {{ .Values.prometheus.prometheusSpec.portName }} + {{- if eq .Values.prometheus.service.type "NodePort" }} + nodePort: {{ .Values.prometheus.service.nodePort }} + {{- end }} + port: {{ .Values.prometheus.service.port }} + targetPort: {{ .Values.prometheus.service.targetPort }} + {{- if .Values.prometheus.thanosIngress.enabled }} + - name: grpc + {{- if eq .Values.prometheus.service.type "NodePort" }} + nodePort: {{ .Values.prometheus.thanosIngress.nodePort }} + {{- end }} + port: {{ .Values.prometheus.thanosIngress.servicePort }} + targetPort: {{ .Values.prometheus.thanosIngress.servicePort }} + {{- end }} +{{- if .Values.prometheus.service.additionalPorts }} +{{ toYaml .Values.prometheus.service.additionalPorts | indent 2 }} +{{- end }} + publishNotReadyAddresses: {{ .Values.prometheus.service.publishNotReadyAddresses }} + selector: + app.kubernetes.io/name: prometheus + prometheus: {{ template "kube-prometheus-stack.prometheus.crname" . }} +{{- if .Values.prometheus.service.sessionAffinity }} + sessionAffinity: {{ .Values.prometheus.service.sessionAffinity }} +{{- end }} + type: "{{ .Values.prometheus.service.type }}" +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/serviceThanosSidecar.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/serviceThanosSidecar.yaml new file mode 100644 index 0000000000..2b80e7742f --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/serviceThanosSidecar.yaml @@ -0,0 +1,39 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.thanosService.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-thanos-discovery + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-thanos-discovery +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.thanosService.labels }} +{{ toYaml .Values.prometheus.thanosService.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheus.thanosService.annotations }} + annotations: +{{ toYaml .Values.prometheus.thanosService.annotations | indent 4 }} +{{- end }} +spec: + type: {{ .Values.prometheus.thanosService.type }} + clusterIP: {{ .Values.prometheus.thanosService.clusterIP }} +{{- if ne .Values.prometheus.thanosService.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.prometheus.thanosService.externalTrafficPolicy }} +{{- end }} + ports: + - name: {{ .Values.prometheus.thanosService.portName }} + port: {{ .Values.prometheus.thanosService.port }} + targetPort: {{ .Values.prometheus.thanosService.targetPort }} + {{- if eq .Values.prometheus.thanosService.type "NodePort" }} + nodePort: {{ .Values.prometheus.thanosService.nodePort }} + {{- end }} + - name: {{ .Values.prometheus.thanosService.httpPortName }} + port: {{ .Values.prometheus.thanosService.httpPort }} + targetPort: {{ .Values.prometheus.thanosService.targetHttpPort }} + {{- if eq .Values.prometheus.thanosService.type "NodePort" }} + nodePort: {{ .Values.prometheus.thanosService.httpNodePort }} + {{- end }} + selector: + app.kubernetes.io/name: prometheus + prometheus: {{ template "kube-prometheus-stack.prometheus.crname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/serviceThanosSidecarExternal.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/serviceThanosSidecarExternal.yaml new file mode 100644 index 0000000000..fa45934d74 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/serviceThanosSidecarExternal.yaml @@ -0,0 +1,46 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.thanosServiceExternal.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-thanos-external + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.thanosServiceExternal.labels }} +{{ toYaml .Values.prometheus.thanosServiceExternal.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheus.thanosServiceExternal.annotations }} + annotations: +{{ toYaml .Values.prometheus.thanosServiceExternal.annotations | indent 4 }} +{{- end }} +spec: + type: {{ .Values.prometheus.thanosServiceExternal.type }} +{{- if .Values.prometheus.thanosServiceExternal.loadBalancerIP }} + loadBalancerIP: {{ .Values.prometheus.thanosServiceExternal.loadBalancerIP }} +{{- end }} +{{- if .Values.prometheus.thanosServiceExternal.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.prometheus.thanosServiceExternal.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if ne .Values.prometheus.thanosServiceExternal.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.prometheus.thanosServiceExternal.externalTrafficPolicy }} +{{- end }} + ports: + - name: {{ .Values.prometheus.thanosServiceExternal.portName }} + port: {{ .Values.prometheus.thanosServiceExternal.port }} + targetPort: {{ .Values.prometheus.thanosServiceExternal.targetPort }} + {{- if eq .Values.prometheus.thanosServiceExternal.type "NodePort" }} + nodePort: {{ .Values.prometheus.thanosServiceExternal.nodePort }} + {{- end }} + - name: {{ .Values.prometheus.thanosServiceExternal.httpPortName }} + port: {{ .Values.prometheus.thanosServiceExternal.httpPort }} + targetPort: {{ .Values.prometheus.thanosServiceExternal.targetHttpPort }} + {{- if eq .Values.prometheus.thanosServiceExternal.type "NodePort" }} + nodePort: {{ .Values.prometheus.thanosServiceExternal.httpNodePort }} + {{- end }} + selector: + app.kubernetes.io/name: prometheus + prometheus: {{ template "kube-prometheus-stack.prometheus.crname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/serviceaccount.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/serviceaccount.yaml new file mode 100644 index 0000000000..dde1632d6f --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/serviceaccount.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "kube-prometheus-stack.prometheus.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + app.kubernetes.io/name: {{ template "kube-prometheus-stack.name" . }}-prometheus + app.kubernetes.io/component: prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.serviceAccount.annotations }} + annotations: +{{ toYaml .Values.prometheus.serviceAccount.annotations | indent 4 }} +{{- end }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{ include "kube-prometheus-stack.imagePullSecrets" . | trim | indent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/servicemonitor.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/servicemonitor.yaml new file mode 100644 index 0000000000..550d41b11d --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/servicemonitor.yaml @@ -0,0 +1,52 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.serviceMonitor.selfMonitor }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + release: {{ $.Release.Name | quote }} + self-monitor: "true" + namespaceSelector: + matchNames: + - {{ printf "%s" (include "kube-prometheus-stack.namespace" .) | quote }} + endpoints: + - port: {{ .Values.prometheus.prometheusSpec.portName }} + {{- if .Values.prometheus.serviceMonitor.interval }} + interval: {{ .Values.prometheus.serviceMonitor.interval }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.scheme }} + scheme: {{ .Values.prometheus.serviceMonitor.scheme }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.tlsConfig }} + tlsConfig: {{ toYaml .Values.prometheus.serviceMonitor.tlsConfig | nindent 6 }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.bearerTokenFile }} + bearerTokenFile: {{ .Values.prometheus.serviceMonitor.bearerTokenFile }} + {{- end }} + path: "{{ trimSuffix "/" .Values.prometheus.prometheusSpec.routePrefix }}/metrics" + metricRelabelings: + {{- if .Values.prometheus.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.prometheus.serviceMonitor.metricRelabelings | indent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.prometheus.serviceMonitor.relabelings }} + relabelings: +{{ toYaml .Values.prometheus.serviceMonitor.relabelings | indent 6 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/servicemonitorThanosSidecar.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/servicemonitorThanosSidecar.yaml new file mode 100644 index 0000000000..9d410fd38d --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/servicemonitorThanosSidecar.yaml @@ -0,0 +1,51 @@ +{{- if and .Values.prometheus.thanosService.enabled .Values.prometheus.thanosServiceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-thanos-sidecar + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-thanos-sidecar +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-thanos-discovery + release: {{ $.Release.Name | quote }} + namespaceSelector: + matchNames: + - {{ printf "%s" (include "kube-prometheus-stack.namespace" .) | quote }} + endpoints: + - port: {{ .Values.prometheus.thanosService.httpPortName }} + {{- if .Values.prometheus.thanosServiceMonitor.interval }} + interval: {{ .Values.prometheus.thanosServiceMonitor.interval }} + {{- end }} + {{- if .Values.prometheus.thanosServiceMonitor.scheme }} + scheme: {{ .Values.prometheus.thanosServiceMonitor.scheme }} + {{- end }} + {{- if .Values.prometheus.thanosServiceMonitor.tlsConfig }} + tlsConfig: {{ toYaml .Values.prometheus.thanosServiceMonitor.tlsConfig | nindent 6 }} + {{- end }} + {{- if .Values.prometheus.thanosServiceMonitor.bearerTokenFile }} + bearerTokenFile: {{ .Values.prometheus.thanosServiceMonitor.bearerTokenFile }} + {{- end }} + path: "/metrics" + metricRelabelings: + {{- if .Values.prometheus.thanosServiceMonitor.metricRelabelings}} + {{ tpl (toYaml .Values.prometheus.thanosServiceMonitor.metricRelabelings | indent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.prometheus.thanosServiceMonitor.relabelings }} + relabelings: +{{ toYaml .Values.prometheus.thanosServiceMonitor.relabelings | indent 6 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/servicemonitors.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/servicemonitors.yaml new file mode 100644 index 0000000000..a78d1cd009 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/servicemonitors.yaml @@ -0,0 +1,38 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.additionalServiceMonitors }} +apiVersion: v1 +kind: List +items: +{{- range .Values.prometheus.additionalServiceMonitors }} + - apiVersion: monitoring.coreos.com/v1 + kind: ServiceMonitor + metadata: + name: {{ .name }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-prometheus +{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if .additionalLabels }} +{{ toYaml .additionalLabels | indent 8 }} + {{- end }} + spec: + endpoints: +{{ toYaml .endpoints | indent 8 }} + {{- if .jobLabel }} + jobLabel: {{ .jobLabel }} + {{- end }} + {{- if .namespaceSelector }} + namespaceSelector: +{{ toYaml .namespaceSelector | indent 8 }} + {{- end }} + selector: +{{ toYaml .selector | indent 8 }} + {{- if .targetLabels }} + targetLabels: +{{ toYaml .targetLabels | indent 8 }} + {{- end }} + {{- if .podTargetLabels }} + podTargetLabels: +{{ toYaml .podTargetLabels | indent 8 }} + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/serviceperreplica.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/serviceperreplica.yaml new file mode 100644 index 0000000000..8d2fdc33d7 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/prometheus/serviceperreplica.yaml @@ -0,0 +1,49 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.servicePerReplica.enabled }} +{{- $count := .Values.prometheus.prometheusSpec.replicas | int -}} +{{- $serviceValues := .Values.prometheus.servicePerReplica -}} +apiVersion: v1 +kind: List +metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-serviceperreplica + namespace: {{ template "kube-prometheus-stack.namespace" . }} +items: +{{- range $i, $e := until $count }} + - apiVersion: v1 + kind: Service + metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-{{ $i }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ include "kube-prometheus-stack.name" $ }}-prometheus +{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if $serviceValues.annotations }} + annotations: +{{ toYaml $serviceValues.annotations | indent 8 }} + {{- end }} + spec: + {{- if $serviceValues.clusterIP }} + clusterIP: {{ $serviceValues.clusterIP }} + {{- end }} + {{- if $serviceValues.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := $serviceValues.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} + {{- end }} + {{- if ne $serviceValues.type "ClusterIP" }} + externalTrafficPolicy: {{ $serviceValues.externalTrafficPolicy }} + {{- end }} + ports: + - name: {{ $.Values.prometheus.prometheusSpec.portName }} + {{- if eq $serviceValues.type "NodePort" }} + nodePort: {{ $serviceValues.nodePort }} + {{- end }} + port: {{ $serviceValues.port }} + targetPort: {{ $serviceValues.targetPort }} + selector: + app.kubernetes.io/name: prometheus + prometheus: {{ include "kube-prometheus-stack.prometheus.crname" $ }} + statefulset.kubernetes.io/pod-name: prometheus-{{ include "kube-prometheus-stack.prometheus.crname" $ }}-{{ $i }} + type: "{{ $serviceValues.type }}" +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/clusterrole.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/clusterrole.yaml new file mode 100644 index 0000000000..2fe9124c5c --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/clusterrole.yaml @@ -0,0 +1,134 @@ +{{- if and .Values.global.rbac.create .Values.global.rbac.userRoles.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: monitoring-admin + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} + {{- if .Values.global.rbac.userRoles.aggregateToDefaultRoles }} + rbac.authorization.k8s.io/aggregate-to-admin: "true" + {{- end }} +rules: +- apiGroups: + - monitoring.coreos.com + resources: + - alertmanagers + - prometheuses + - prometheuses/finalizers + - alertmanagers/finalizers + verbs: + - 'get' + - 'list' + - 'watch' +- apiGroups: + - monitoring.coreos.com + resources: + - thanosrulers + - thanosrulers/finalizers + - servicemonitors + - podmonitors + - prometheusrules + - podmonitors + - probes + - probes/finalizers + - alertmanagerconfigs + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: monitoring-edit + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} + {{- if .Values.global.rbac.userRoles.aggregateToDefaultRoles }} + rbac.authorization.k8s.io/aggregate-to-edit: "true" + {{- end }} +rules: +- apiGroups: + - monitoring.coreos.com + resources: + - alertmanagers + - prometheuses + - prometheuses/finalizers + - alertmanagers/finalizers + verbs: + - 'get' + - 'list' + - 'watch' +- apiGroups: + - monitoring.coreos.com + resources: + - thanosrulers + - thanosrulers/finalizers + - servicemonitors + - podmonitors + - prometheusrules + - podmonitors + - probes + - alertmanagerconfigs + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: monitoring-view + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} + {{- if .Values.global.rbac.userRoles.aggregateToDefaultRoles }} + rbac.authorization.k8s.io/aggregate-to-view: "true" + {{- end }} +rules: +- apiGroups: + - monitoring.coreos.com + resources: + - alertmanagers + - prometheuses + - prometheuses/finalizers + - alertmanagers/finalizers + - thanosrulers + - thanosrulers/finalizers + - servicemonitors + - podmonitors + - prometheusrules + - podmonitors + - probes + - probes/finalizers + - alertmanagerconfigs + verbs: + - 'get' + - 'list' + - 'watch' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: monitoring-ui-view + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - services/proxy + resourceNames: + - "http:{{ template "kube-prometheus-stack.fullname" . }}-prometheus:{{ .Values.prometheus.service.port }}" + - "https:{{ template "kube-prometheus-stack.fullname" . }}-prometheus:{{ .Values.prometheus.service.port }}" + - "http:{{ template "kube-prometheus-stack.fullname" . }}-alertmanager:{{ .Values.alertmanager.service.port }}" + - "https:{{ template "kube-prometheus-stack.fullname" . }}-alertmanager:{{ .Values.alertmanager.service.port }}" +{{- if .Values.grafana.enabled }} + - "http:{{ include "call-nested" (list . "grafana" "grafana.fullname") }}:{{ .Values.grafana.service.port }}" + - "https:{{ include "call-nested" (list . "grafana" "grafana.fullname") }}:{{ .Values.grafana.service.port }}" +{{- end }} + verbs: + - 'get' +- apiGroups: + - "" + resourceNames: + - {{ template "kube-prometheus-stack.fullname" . }}-prometheus + - {{ template "kube-prometheus-stack.fullname" . }}-alertmanager +{{- if .Values.grafana.enabled }} + - {{ include "call-nested" (list . "grafana" "grafana.fullname") }} +{{- end }} + resources: + - endpoints + verbs: + - get +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/config-role.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/config-role.yaml new file mode 100644 index 0000000000..f48ffc827e --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/config-role.yaml @@ -0,0 +1,48 @@ +{{- if and .Values.global.rbac.create .Values.global.rbac.userRoles.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: monitoring-config-admin + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + - secrets + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: monitoring-config-edit + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + - secrets + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: monitoring-config-view + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + - secrets + verbs: + - 'get' + - 'list' + - 'watch' +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/dashboard-role.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/dashboard-role.yaml new file mode 100644 index 0000000000..d2f81976a2 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/dashboard-role.yaml @@ -0,0 +1,47 @@ +{{- if and .Values.global.rbac.create .Values.global.rbac.userRoles.create .Values.grafana.enabled }} +{{- if .Values.grafana.defaultDashboardsEnabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: monitoring-dashboard-admin + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: monitoring-dashboard-edit + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: monitoring-dashboard-view + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - 'get' + - 'list' + - 'watch' +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/dashboards/addons/ingress-nginx-dashboard.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/dashboards/addons/ingress-nginx-dashboard.yaml new file mode 100644 index 0000000000..7b51a0bf7a --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/dashboards/addons/ingress-nginx-dashboard.yaml @@ -0,0 +1,18 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled .Values.ingressNginx.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "ingress-nginx" | trunc 63 | trimSuffix "-" }} + {{- if .Values.grafana.sidecar.dashboards.annotations }} + annotations: {{ toYaml .Values.grafana.sidecar.dashboards.annotations | nindent 4 }} + {{- end }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/ingress-nginx/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/dashboards/rancher/cluster-dashboards.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/dashboards/rancher/cluster-dashboards.yaml new file mode 100644 index 0000000000..d73b257451 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/dashboards/rancher/cluster-dashboards.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-cluster + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/cluster/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/dashboards/rancher/default-dashboard.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/dashboards/rancher/default-dashboard.yaml new file mode 100644 index 0000000000..8865efa932 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/dashboards/rancher/default-dashboard.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-home + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/home/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/dashboards/rancher/k8s-dashboards.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/dashboards/rancher/k8s-dashboards.yaml new file mode 100644 index 0000000000..2afae10ef7 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/dashboards/rancher/k8s-dashboards.yaml @@ -0,0 +1,31 @@ +{{- $files := (.Files.Glob "files/rancher/k8s/*").AsConfig }} +{{- $filesDict := (fromYaml $files) }} +{{- if not (include "exporter.kubeEtcd.enabled" .) }} +{{- $filesDict = (unset $filesDict "rancher-etcd-nodes.json") -}} +{{- $filesDict = (unset $filesDict "rancher-etcd.json") -}} +{{- end }} +{{- if not (include "exporter.kubeControllerManager.enabled" .) }} +{{- $filesDict = (unset $filesDict "rancher-k8s-components-nodes.json") -}} +{{- $filesDict = (unset $filesDict "rancher-k8s-components.json") -}} +{{- else }} +{{- $_ := (set $filesDict "rancher-k8s-components-nodes.json" (get $filesDict "rancher-k8s-components-nodes.json" | replace "kube-controller-manager" (include "exporter.kubeControllerManager.jobName" .))) -}} +{{- $_ := (set $filesDict "rancher-k8s-components.json" (get $filesDict "rancher-k8s-components.json" | replace "kube-controller-manager" (include "exporter.kubeControllerManager.jobName" .))) -}} +{{- end }} +{{ $files = (toYaml $filesDict) }} +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-k8s + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ $files | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/dashboards/rancher/nodes-dashboards.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/dashboards/rancher/nodes-dashboards.yaml new file mode 100644 index 0000000000..172c36e9d1 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/dashboards/rancher/nodes-dashboards.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-nodes + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/nodes/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/dashboards/rancher/performance-dashboards.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/dashboards/rancher/performance-dashboards.yaml new file mode 100644 index 0000000000..19836ec4e4 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/dashboards/rancher/performance-dashboards.yaml @@ -0,0 +1,18 @@ +{{- $selector := (include "rancher.serviceMonitor.selector" .) -}} +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled .Values.rancherMonitoring.enabled $selector }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-performance-debugging + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/performance/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/dashboards/rancher/pods-dashboards.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/dashboards/rancher/pods-dashboards.yaml new file mode 100644 index 0000000000..940f18869b --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/dashboards/rancher/pods-dashboards.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-pods + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/pods/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/dashboards/rancher/workload-dashboards.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/dashboards/rancher/workload-dashboards.yaml new file mode 100644 index 0000000000..d146dacdd0 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/dashboards/rancher/workload-dashboards.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-workloads + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/workloads/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/exporters/ingress-nginx/service.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/exporters/ingress-nginx/service.yaml new file mode 100644 index 0000000000..53a9ad6897 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/exporters/ingress-nginx/service.yaml @@ -0,0 +1,27 @@ +{{- if and (not .Values.ingressNginx.enabled) (.Values.rkeIngressNginx.enabled) }} +{{- fail "Cannot set .Values.rkeIngressNginx.enabled=true when .Values.ingressNginx.enabled=false" }} +{{- end }} +{{- if and .Values.ingressNginx.enabled (not .Values.rkeIngressNginx.enabled) }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-ingress-nginx + labels: + app: {{ template "kube-prometheus-stack.name" . }}-ingress-nginx + jobLabel: ingress-nginx +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: {{ .Values.ingressNginx.namespace }} +spec: + clusterIP: None + ports: + - name: http-metrics + port: {{ .Values.ingressNginx.service.port }} + protocol: TCP + targetPort: {{ .Values.ingressNginx.service.targetPort }} + selector: + {{- if .Values.ingressNginx.service.selector }} +{{ toYaml .Values.ingressNginx.service.selector | indent 4 }} + {{- else }} + app: ingress-nginx + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/exporters/ingress-nginx/servicemonitor.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/exporters/ingress-nginx/servicemonitor.yaml new file mode 100644 index 0000000000..b0f92e63b5 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/exporters/ingress-nginx/servicemonitor.yaml @@ -0,0 +1,49 @@ +{{- if and (not .Values.ingressNginx.enabled) (.Values.rkeIngressNginx.enabled) }} +{{- fail "Cannot set .Values.rkeIngressNginx.enabled=true when .Values.ingressNginx.enabled=false" }} +{{- end }} +{{- if and .Values.ingressNginx.enabled (not .Values.rkeIngressNginx.enabled) }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-ingress-nginx + namespace: {{ .Values.ingressNginx.namespace }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-ingress-nginx +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: jobLabel + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-ingress-nginx + release: {{ $.Release.Name | quote }} + namespaceSelector: + matchNames: + - {{ .Values.ingressNginx.namespace }} + endpoints: + - port: http-metrics + {{- if .Values.ingressNginx.serviceMonitor.interval}} + interval: {{ .Values.ingressNginx.serviceMonitor.interval }} + {{- end }} + {{- if .Values.ingressNginx.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.ingressNginx.serviceMonitor.proxyUrl}} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + metricRelabelings: + {{- if .Values.ingressNginx.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.ingressNginx.serviceMonitor.metricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.ingressNginx.serviceMonitor.relabelings }} + relabelings: +{{ toYaml .Values.ingressNginx.serviceMonitor.relabelings | indent 4 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/exporters/rancher/servicemonitor.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/exporters/rancher/servicemonitor.yaml new file mode 100644 index 0000000000..1fba8f23f7 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/exporters/rancher/servicemonitor.yaml @@ -0,0 +1,58 @@ +{{- $selector := (include "rancher.serviceMonitor.selector" .) -}} +{{- if and .Values.rancherMonitoring.enabled $selector }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} + name: rancher + namespace: cattle-system +spec: + endpoints: + - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + port: http + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecureSkipVerify: true + serverName: rancher + metricRelabelings: + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + jobLabel: rancher +{{- if .Values.rancherMonitoring.namespaceSelector }} + namespaceSelector: {{ .Values.rancherMonitoring.namespaceSelector | toYaml | nindent 4 }} +{{- end }} + selector: {{ include "rancher.serviceMonitor.selector" . | nindent 4 }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-rancher-metrics +rules: +- apiGroups: + - management.cattle.io + resources: + - ranchermetrics + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-rancher-metrics +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.fullname" . }}-rancher-metrics +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/hardened.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/hardened.yaml new file mode 100644 index 0000000000..4a90c16953 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/hardened.yaml @@ -0,0 +1,128 @@ +{{- $namespaces := dict "_0" .Release.Namespace -}} +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled (not .Values.grafana.defaultDashboards.useExistingNamespace) -}} +{{- $_ := set $namespaces "_1" .Values.grafana.defaultDashboards.namespace -}} +{{- end -}} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ .Chart.Name }}-patch-sa + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Chart.Name }}-patch-sa + annotations: + "helm.sh/hook": post-install, post-upgrade + "helm.sh/hook-delete-policy": hook-succeeded, before-hook-creation +spec: + template: + metadata: + name: {{ .Chart.Name }}-patch-sa + labels: + app: {{ .Chart.Name }}-patch-sa + spec: + serviceAccountName: {{ .Chart.Name }}-patch-sa + securityContext: + runAsNonRoot: true + runAsUser: 1000 + restartPolicy: Never + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} + containers: + {{- range $_, $ns := $namespaces }} + - name: patch-sa-{{ $ns }} + image: {{ template "system_default_registry" $ }}{{ $.Values.global.kubectl.repository }}:{{ $.Values.global.kubectl.tag }} + imagePullPolicy: {{ $.Values.global.kubectl.pullPolicy }} + command: ["kubectl", "patch", "serviceaccount", "default", "-p", "{\"automountServiceAccountToken\": false}"] + args: ["-n", "{{ $ns }}"] + {{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ .Chart.Name }}-patch-sa + labels: + app: {{ .Chart.Name }}-patch-sa +rules: +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: ['get', 'patch'] +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ .Chart.Name }}-patch-sa +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ .Chart.Name }}-patch-sa + labels: + app: {{ .Chart.Name }}-patch-sa +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ .Chart.Name }}-patch-sa +subjects: +- kind: ServiceAccount + name: {{ .Chart.Name }}-patch-sa + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Chart.Name }}-patch-sa + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Chart.Name }}-patch-sa +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ .Chart.Name }}-patch-sa + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Chart.Name }}-patch-sa +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- range $_, $ns := $namespaces }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: default-allow-all + namespace: {{ $ns }} +spec: + podSelector: {} + ingress: + - {} + egress: + - {} + policyTypes: + - Ingress + - Egress +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/upgrade/configmap.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/upgrade/configmap.yaml new file mode 100644 index 0000000000..53cb898214 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/upgrade/configmap.yaml @@ -0,0 +1,13 @@ +{{- if .Values.upgrade.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "0" +data: +{{ (.Files.Glob "files/upgrade/scripts/*").AsConfig | indent 2 }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/upgrade/job.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/upgrade/job.yaml new file mode 100644 index 0000000000..8f2771740c --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/upgrade/job.yaml @@ -0,0 +1,46 @@ +{{- if .Values.upgrade.enabled }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "2" +spec: + template: + metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + spec: + serviceAccountName: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + securityContext: + runAsNonRoot: false + runAsUser: 0 + restartPolicy: Never + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} + containers: + - name: run-scripts + image: {{ template "system_default_registry" . }}{{ .Values.upgrade.image.repository }}:{{ .Values.upgrade.image.tag }} + imagePullPolicy: {{ $.Values.global.kubectl.pullPolicy }} + command: + - /bin/sh + - -c + - > + for s in $(find /etc/scripts -type f); do + echo "Running $s..."; + cat $s | bash + done; + volumeMounts: + - name: upgrade + mountPath: /etc/scripts + volumes: + - name: upgrade + configMap: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/upgrade/rbac.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/upgrade/rbac.yaml new file mode 100644 index 0000000000..e929a19925 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/rancher-monitoring/upgrade/rbac.yaml @@ -0,0 +1,131 @@ +{{- if .Values.upgrade.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded + "helm.sh/hook-weight": "1" +rules: +- apiGroups: + - apps + resources: + - deployments + - daemonsets + - statefulsets + verbs: + - 'list' + - 'delete' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "1" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade +subjects: +- kind: ServiceAccount + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "1" +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "kube-prometheus-stack.fullname" . }}-upgrade +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "1" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade +subjects: +- kind: ServiceAccount + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "1" +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "1" +spec: + privileged: false + allowPrivilegeEscalation: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'configMap' + - 'secret' +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/thanos-ruler/extrasecret.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/thanos-ruler/extrasecret.yaml new file mode 100644 index 0000000000..fe2ea5be6d --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/thanos-ruler/extrasecret.yaml @@ -0,0 +1,20 @@ +{{- if .Values.thanosRuler.extraSecret.data -}} +{{- $secretName := printf "thanos-ruler-%s-extra" (include "kube-prometheus-stack.fullname" . ) -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ default $secretName .Values.thanosRuler.extraSecret.name }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.thanosRuler.extraSecret.annotations }} + annotations: +{{ toYaml .Values.thanosRuler.extraSecret.annotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-thanos-ruler + app.kubernetes.io/component: thanos-ruler +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: +{{- range $key, $val := .Values.thanosRuler.extraSecret.data }} + {{ $key }}: {{ $val | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/thanos-ruler/ingress.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/thanos-ruler/ingress.yaml new file mode 100644 index 0000000000..2760805c53 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/thanos-ruler/ingress.yaml @@ -0,0 +1,77 @@ +{{- if and .Values.thanosRuler.enabled .Values.thanosRuler.ingress.enabled }} +{{- $pathType := .Values.thanosRuler.ingress.pathType | default "ImplementationSpecific" }} +{{- $serviceName := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "thanos-ruler" }} +{{- $servicePort := .Values.thanosRuler.service.port -}} +{{- $routePrefix := list .Values.thanosRuler.thanosRulerSpec.routePrefix }} +{{- $paths := .Values.thanosRuler.ingress.paths | default $routePrefix -}} +{{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "kube-prometheus-stack.ingress.supportsPathType" .) "true" -}} +apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" . }} +kind: Ingress +metadata: + name: {{ $serviceName }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.thanosRuler.ingress.annotations }} + annotations: +{{ toYaml .Values.thanosRuler.ingress.annotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-thanos-ruler +{{- if .Values.thanosRuler.ingress.labels }} +{{ toYaml .Values.thanosRuler.ingress.labels | indent 4 }} +{{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- if $apiIsStable }} + {{- if .Values.thanosRuler.ingress.ingressClassName }} + ingressClassName: {{ .Values.thanosRuler.ingress.ingressClassName }} + {{- end }} + {{- end }} + rules: + {{- if .Values.thanosRuler.ingress.hosts }} + {{- range $host := .Values.thanosRuler.ingress.hosts }} + - host: {{ tpl $host $ }} + http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $serviceName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $serviceName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- else }} + - http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $serviceName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $serviceName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- if .Values.thanosRuler.ingress.tls }} + tls: +{{ tpl (toYaml .Values.thanosRuler.ingress.tls | indent 4) . }} + {{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/thanos-ruler/podDisruptionBudget.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/thanos-ruler/podDisruptionBudget.yaml new file mode 100644 index 0000000000..d3d378d693 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/thanos-ruler/podDisruptionBudget.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.thanosRuler.enabled .Values.thanosRuler.podDisruptionBudget.enabled }} +apiVersion: {{ include "kube-prometheus-stack.pdb.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-thanos-ruler + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-thanos-ruler +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- if .Values.thanosRuler.podDisruptionBudget.minAvailable }} + minAvailable: {{ .Values.thanosRuler.podDisruptionBudget.minAvailable }} + {{- end }} + {{- if .Values.thanosRuler.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ .Values.thanosRuler.podDisruptionBudget.maxUnavailable }} + {{- end }} + selector: + matchLabels: + app.kubernetes.io/name: thanos-ruler + thanos-ruler: {{ template "kube-prometheus-stack.fullname" . }}-thanos-ruler +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/thanos-ruler/ruler.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/thanos-ruler/ruler.yaml new file mode 100644 index 0000000000..c914e755d3 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/thanos-ruler/ruler.yaml @@ -0,0 +1,168 @@ +{{- if .Values.thanosRuler.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ThanosRuler +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-thanos-ruler + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-thanos-ruler +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.thanosRuler.annotations }} + annotations: +{{ toYaml .Values.thanosRuler.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.thanosRuler.thanosRulerSpec.image }} + {{- if and .Values.thanosRuler.thanosRulerSpec.image.tag .Values.thanosRuler.thanosRulerSpec.image.sha }} + image: "{{ template "system_default_registry" . }}{{ .Values.thanosRuler.thanosRulerSpec.image.repository }}:{{ .Values.thanosRuler.thanosRulerSpec.image.tag }}@sha256:{{ .Values.thanosRuler.thanosRulerSpec.image.sha }}" + {{- else if .Values.thanosRuler.thanosRulerSpec.image.sha }} + image: "{{ template "system_default_registry" . }}{{ .Values.thanosRuler.thanosRulerSpec.image.repository }}@sha256:{{ .Values.thanosRuler.thanosRulerSpec.image.sha }}" + {{- else if .Values.thanosRuler.thanosRulerSpec.image.tag }} + image: "{{ template "system_default_registry" . }}{{ .Values.thanosRuler.thanosRulerSpec.image.repository }}:{{ .Values.thanosRuler.thanosRulerSpec.image.tag }}" + {{- else }} + image: "{{ template "system_default_registry" . }}{{ .Values.thanosRuler.thanosRulerSpec.image.repository }}" + {{- end }} + {{- if .Values.thanosRuler.thanosRulerSpec.image.sha }} + sha: {{ .Values.thanosRuler.thanosRulerSpec.image.sha }} + {{- end }} +{{- end }} + replicas: {{ .Values.thanosRuler.thanosRulerSpec.replicas }} + listenLocal: {{ .Values.thanosRuler.thanosRulerSpec.listenLocal }} + serviceAccountName: {{ template "kube-prometheus-stack.thanosRuler.serviceAccountName" . }} +{{- if .Values.thanosRuler.thanosRulerSpec.externalPrefix }} + externalPrefix: "{{ tpl .Values.thanosRuler.thanosRulerSpec.externalPrefix . }}" +{{- else if and .Values.thanosRuler.ingress.enabled .Values.thanosRuler.ingress.hosts }} + externalPrefix: "http://{{ tpl (index .Values.thanosRuler.ingress.hosts 0) . }}{{ .Values.thanosRuler.thanosRulerSpec.routePrefix }}" +{{- else }} + externalPrefix: http://{{ template "kube-prometheus-stack.fullname" . }}-thanosRuler.{{ template "kube-prometheus-stack.namespace" . }}:{{ .Values.thanosRuler.service.port }} +{{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 4 }} +{{- if .Values.thanosRuler.thanosRulerSpec.nodeSelector }} +{{ toYaml .Values.thanosRuler.thanosRulerSpec.nodeSelector | indent 4 }} +{{- end }} + paused: {{ .Values.thanosRuler.thanosRulerSpec.paused }} + logFormat: {{ .Values.thanosRuler.thanosRulerSpec.logFormat | quote }} + logLevel: {{ .Values.thanosRuler.thanosRulerSpec.logLevel | quote }} + retention: {{ .Values.thanosRuler.thanosRulerSpec.retention | quote }} +{{- if .Values.thanosRuler.thanosRulerSpec.evaluationInterval }} + evaluationInterval: {{ .Values.thanosRuler.thanosRulerSpec.evaluationInterval }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.ruleNamespaceSelector }} + ruleNamespaceSelector: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.ruleNamespaceSelector | indent 4 }} +{{ else }} + ruleNamespaceSelector: {} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.ruleSelector }} + ruleSelector: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.ruleSelector | indent 4}} +{{- else if .Values.thanosRuler.thanosRulerSpec.ruleSelectorNilUsesHelmValues }} + ruleSelector: + matchLabels: + release: {{ $.Release.Name | quote }} +{{ else }} + ruleSelector: {} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.alertQueryUrl }} + alertQueryUrl: "{{ .Values.thanosRuler.thanosRulerSpec.alertQueryUrl }}" +{{- end}} +{{- if .Values.thanosRuler.thanosRulerSpec.alertmanagersUrl }} + alertmanagersUrl: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.alertmanagersUrl | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.alertmanagersConfig }} + alertmanagersConfig: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.alertmanagersConfig | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.queryEndpoints }} + queryEndpoints: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.queryEndpoints | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.resources }} + resources: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.resources | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.routePrefix }} + routePrefix: "{{ .Values.thanosRuler.thanosRulerSpec.routePrefix }}" +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.securityContext }} + securityContext: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.securityContext | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.storage }} + storage: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.storage | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.objectStorageConfig }} + objectStorageConfig: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.objectStorageConfig | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.labels }} + labels: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.labels | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.objectStorageConfigFile }} + objectStorageConfigFile: {{ .Values.thanosRuler.thanosRulerSpec.objectStorageConfigFile }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.podMetadata }} + podMetadata: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.podMetadata | indent 4 }} +{{- end }} +{{- if or .Values.thanosRuler.thanosRulerSpec.podAntiAffinity .Values.thanosRuler.thanosRulerSpec.affinity }} + affinity: +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.affinity }} +{{ toYaml .Values.thanosRuler.thanosRulerSpec.affinity | indent 4 }} +{{- end }} +{{- if eq .Values.thanosRuler.thanosRulerSpec.podAntiAffinity "hard" }} + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: {{ .Values.thanosRuler.thanosRulerSpec.podAntiAffinityTopologyKey }} + labelSelector: + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [thanos-ruler]} + - {key: thanos-ruler, operator: In, values: [{{ template "kube-prometheus-stack.fullname" . }}-thanos-ruler]} +{{- else if eq .Values.thanosRuler.thanosRulerSpec.podAntiAffinity "soft" }} + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: {{ .Values.thanosRuler.thanosRulerSpec.podAntiAffinityTopologyKey }} + labelSelector: + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [thanos-ruler]} + - {key: thanos-ruler, operator: In, values: [{{ template "kube-prometheus-stack.fullname" . }}-thanos-ruler]} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 4 }} +{{- if .Values.thanosRuler.thanosRulerSpec.tolerations }} +{{ toYaml .Values.thanosRuler.thanosRulerSpec.tolerations | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.topologySpreadConstraints }} + topologySpreadConstraints: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.topologySpreadConstraints | indent 4 }} +{{- end }} +{{- if .Values.global.imagePullSecrets }} + imagePullSecrets: +{{ toYaml .Values.global.imagePullSecrets | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.containers }} + containers: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.containers | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.initContainers }} + initContainers: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.initContainers | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.priorityClassName }} + priorityClassName: {{.Values.thanosRuler.thanosRulerSpec.priorityClassName }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.volumes }} + volumes: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.volumes | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.volumeMounts }} + volumeMounts: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.volumeMounts | indent 4 }} +{{- end }} + portName: {{ .Values.thanosRuler.thanosRulerSpec.portName }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/thanos-ruler/service.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/thanos-ruler/service.yaml new file mode 100644 index 0000000000..093dbf7cd4 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/thanos-ruler/service.yaml @@ -0,0 +1,53 @@ +{{- if .Values.thanosRuler.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-thanos-ruler + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-thanos-ruler + self-monitor: {{ .Values.thanosRuler.serviceMonitor.selfMonitor | quote }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.thanosRuler.service.labels }} +{{ toYaml .Values.thanosRuler.service.labels | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.service.annotations }} + annotations: +{{ toYaml .Values.thanosRuler.service.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.thanosRuler.service.clusterIP }} + clusterIP: {{ .Values.thanosRuler.service.clusterIP }} +{{- end }} +{{- if .Values.thanosRuler.service.externalIPs }} + externalIPs: +{{ toYaml .Values.thanosRuler.service.externalIPs | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.thanosRuler.service.loadBalancerIP }} +{{- end }} +{{- if .Values.thanosRuler.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.thanosRuler.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if ne .Values.thanosRuler.service.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.thanosRuler.service.externalTrafficPolicy }} +{{- end }} + ports: + - name: {{ .Values.thanosRuler.thanosRulerSpec.portName }} + {{- if eq .Values.thanosRuler.service.type "NodePort" }} + nodePort: {{ .Values.thanosRuler.service.nodePort }} + {{- end }} + port: {{ .Values.thanosRuler.service.port }} + targetPort: {{ .Values.thanosRuler.service.targetPort }} + protocol: TCP +{{- if .Values.thanosRuler.service.additionalPorts }} +{{ toYaml .Values.thanosRuler.service.additionalPorts | indent 2 }} +{{- end }} + selector: + app.kubernetes.io/name: thanos-ruler + thanos-ruler: {{ template "kube-prometheus-stack.fullname" . }}-thanos-ruler + type: "{{ .Values.thanosRuler.service.type }}" +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/thanos-ruler/serviceaccount.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/thanos-ruler/serviceaccount.yaml new file mode 100644 index 0000000000..0138c357f5 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/thanos-ruler/serviceaccount.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.thanosRuler.enabled .Values.thanosRuler.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "kube-prometheus-stack.thanosRuler.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-thanos-ruler + app.kubernetes.io/name: {{ template "kube-prometheus-stack.name" . }}-thanos-ruler + app.kubernetes.io/component: thanos-ruler +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.thanosRuler.serviceAccount.annotations }} + annotations: +{{ toYaml .Values.thanosRuler.serviceAccount.annotations | indent 4 }} +{{- end }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{ toYaml .Values.global.imagePullSecrets | indent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/thanos-ruler/servicemonitor.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/thanos-ruler/servicemonitor.yaml new file mode 100644 index 0000000000..1f2bd417fa --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/thanos-ruler/servicemonitor.yaml @@ -0,0 +1,56 @@ +{{- if and .Values.thanosRuler.enabled .Values.thanosRuler.serviceMonitor.selfMonitor }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-thanos-ruler + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-thanos-ruler +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-thanos-ruler + release: {{ $.Release.Name | quote }} + self-monitor: {{ .Values.thanosRuler.serviceMonitor.selfMonitor | quote }} + namespaceSelector: + matchNames: + - {{ printf "%s" (include "kube-prometheus-stack.namespace" .) | quote }} + endpoints: + - port: {{ .Values.thanosRuler.thanosRulerSpec.portName }} + {{- if .Values.thanosRuler.serviceMonitor.interval }} + interval: {{ .Values.thanosRuler.serviceMonitor.interval }} + {{- end }} + {{- if .Values.thanosRuler.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.thanosRuler.serviceMonitor.proxyUrl}} + {{- end }} + {{- if .Values.thanosRuler.serviceMonitor.scheme }} + scheme: {{ .Values.thanosRuler.serviceMonitor.scheme }} + {{- end }} + {{- if .Values.thanosRuler.serviceMonitor.bearerTokenFile }} + bearerTokenFile: {{ .Values.thanosRuler.serviceMonitor.bearerTokenFile }} + {{- end }} + {{- if .Values.thanosRuler.serviceMonitor.tlsConfig }} + tlsConfig: {{ toYaml .Values.thanosRuler.serviceMonitor.tlsConfig | nindent 6 }} + {{- end }} + path: "{{ trimSuffix "/" .Values.thanosRuler.thanosRulerSpec.routePrefix }}/metrics" + + metricRelabelings: + {{- if .Values.thanosRuler.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.thanosRuler.serviceMonitor.metricRelabelings | indent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.thanosRuler.serviceMonitor.relabelings }} + relabelings: +{{ toYaml .Values.thanosRuler.serviceMonitor.relabelings | indent 6 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/validate-install-crd.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..ac7921f586 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/validate-install-crd.yaml @@ -0,0 +1,21 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1alpha1/AlertmanagerConfig" false -}} +# {{- set $found "monitoring.coreos.com/v1/Alertmanager" false -}} +# {{- set $found "monitoring.coreos.com/v1/PodMonitor" false -}} +# {{- set $found "monitoring.coreos.com/v1/Probe" false -}} +# {{- set $found "monitoring.coreos.com/v1/Prometheus" false -}} +# {{- set $found "monitoring.coreos.com/v1/PrometheusRule" false -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- set $found "monitoring.coreos.com/v1/ThanosRuler" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install the corresponding CRD chart before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} \ No newline at end of file diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/templates/validate-psp-install.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/102.0.5+up40.1.2/values.yaml b/charts/rancher-monitoring/102.0.5+up40.1.2/values.yaml new file mode 100644 index 0000000000..e546566659 --- /dev/null +++ b/charts/rancher-monitoring/102.0.5+up40.1.2/values.yaml @@ -0,0 +1,4190 @@ +# Default values for kube-prometheus-stack. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Rancher Monitoring Configuration + +## Configuration for prometheus-adapter +## ref: https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-adapter +## +prometheus-adapter: + enabled: true + prometheus: + # Change this if you change the namespaceOverride or nameOverride of prometheus-operator + url: http://rancher-monitoring-prometheus.cattle-monitoring-system.svc + port: 9090 + +## RKE PushProx Monitoring +## ref: https://github.com/rancher/charts/tree/dev-v2.5-source/packages/rancher-pushprox +## +rkeControllerManager: + enabled: false + metricsPort: 10257 # default to secure port as of k8s >= 1.22 + component: kube-controller-manager + clients: + https: + enabled: true + insecureSkipVerify: true + useServiceAccountCredentials: true + port: 10011 + useLocalhost: true + nodeSelector: + node-role.kubernetes.io/controlplane: "true" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + kubeVersionOverrides: + - constraint: "< 1.22" + values: + metricsPort: 10252 # default to insecure port in k8s < 1.22 + clients: + https: + enabled: false + insecureSkipVerify: false + useServiceAccountCredentials: false + +rkeScheduler: + enabled: false + metricsPort: 10259 + component: kube-scheduler + clients: + https: + enabled: true + insecureSkipVerify: true + useServiceAccountCredentials: true + port: 10012 + useLocalhost: true + nodeSelector: + node-role.kubernetes.io/controlplane: "true" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + kubeVersionOverrides: + - constraint: "< 1.23" + values: + metricsPort: 10251 # default to insecure port in k8s < 1.23 + clients: + https: + enabled: false + insecureSkipVerify: false + useServiceAccountCredentials: false + +rkeProxy: + enabled: false + metricsPort: 10249 + component: kube-proxy + clients: + port: 10013 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +rkeEtcd: + enabled: false + metricsPort: 2379 + component: kube-etcd + clients: + port: 10014 + https: + enabled: true + certDir: /etc/kubernetes/ssl + certFile: kube-etcd-*.pem + keyFile: kube-etcd-*-key.pem + caCertFile: kube-ca.pem + seLinuxOptions: + # Gives rkeEtcd permissions to read files in /etc/kubernetes/* + # Type is defined in https://github.com/rancher/rancher-selinux + type: rke_kubereader_t + nodeSelector: + node-role.kubernetes.io/etcd: "true" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +rkeIngressNginx: + enabled: false + metricsPort: 10254 + component: ingress-nginx + clients: + port: 10015 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + nodeSelector: + node-role.kubernetes.io/worker: "true" + +## k3s PushProx Monitoring +## ref: https://github.com/rancher/charts/tree/dev-v2.5-source/packages/rancher-pushprox +## +k3sServer: + enabled: false + metricsPort: 10250 + component: k3s-server + clients: + port: 10013 + useLocalhost: true + https: + enabled: true + useServiceAccountCredentials: true + insecureSkipVerify: true + rbac: + additionalRules: + - nonResourceURLs: ["/metrics/cadvisor"] + verbs: ["get"] + - apiGroups: [""] + resources: ["nodes/metrics"] + verbs: ["get"] + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + serviceMonitor: + endpoints: + - port: metrics + honorLabels: true + relabelings: + - sourceLabels: [__metrics_path__] + targetLabel: metrics_path + - port: metrics + path: /metrics/cadvisor + honorLabels: true + relabelings: + - sourceLabels: [__metrics_path__] + targetLabel: metrics_path + - port: metrics + path: /metrics/probes + honorLabels: true + relabelings: + - sourceLabels: [__metrics_path__] + targetLabel: metrics_path + +## KubeADM PushProx Monitoring +## ref: https://github.com/rancher/charts/tree/dev-v2.5-source/packages/rancher-pushprox +## +kubeAdmControllerManager: + enabled: false + metricsPort: 10257 + component: kube-controller-manager + clients: + port: 10011 + useLocalhost: true + https: + enabled: true + useServiceAccountCredentials: true + insecureSkipVerify: true + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +kubeAdmScheduler: + enabled: false + metricsPort: 10259 + component: kube-scheduler + clients: + port: 10012 + useLocalhost: true + https: + enabled: true + useServiceAccountCredentials: true + insecureSkipVerify: true + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +kubeAdmProxy: + enabled: false + metricsPort: 10249 + component: kube-proxy + clients: + port: 10013 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +kubeAdmEtcd: + enabled: false + metricsPort: 2381 + component: kube-etcd + clients: + port: 10014 + useLocalhost: true + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +## rke2 PushProx Monitoring +## ref: https://github.com/rancher/charts/tree/dev-v2.5-source/packages/rancher-pushprox +## +rke2ControllerManager: + enabled: false + metricsPort: 10257 # default to secure port as of k8s >= 1.22 + component: kube-controller-manager + clients: + https: + enabled: true + insecureSkipVerify: true + useServiceAccountCredentials: true + port: 10011 + useLocalhost: true + nodeSelector: + node-role.kubernetes.io/master: "true" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + kubeVersionOverrides: + - constraint: "< 1.22" + values: + metricsPort: 10252 # default to insecure port in k8s < 1.22 + clients: + https: + enabled: false + insecureSkipVerify: false + useServiceAccountCredentials: false + +rke2Scheduler: + enabled: false + metricsPort: 10259 # default to secure port as of k8s >= 1.22 + component: kube-scheduler + clients: + https: + enabled: true + insecureSkipVerify: true + useServiceAccountCredentials: true + port: 10012 + useLocalhost: true + nodeSelector: + node-role.kubernetes.io/master: "true" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + kubeVersionOverrides: + - constraint: "< 1.22" + values: + metricsPort: 10251 # default to insecure port in k8s < 1.22 + clients: + https: + enabled: false + insecureSkipVerify: false + useServiceAccountCredentials: false + +rke2Proxy: + enabled: false + metricsPort: 10249 + component: kube-proxy + clients: + port: 10013 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +rke2Etcd: + enabled: false + metricsPort: 2381 + component: kube-etcd + clients: + port: 10014 + useLocalhost: true + nodeSelector: + node-role.kubernetes.io/etcd: "true" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +rke2IngressNginx: + enabled: false + metricsPort: 10254 + component: ingress-nginx + # in the RKE2 cluster, the ingress-nginx-controller is deployed + # as a non-hostNetwork workload starting at the following versions + # - >= v1.22.12+rke2r1 < 1.23.0-0 + # - >= v1.23.9+rke2r1 < 1.24.0-0 + # - >= v1.24.3+rke2r1 < 1.25.0-0 + # - >= v1.25.0+rke2r1 + # As a result we do not need clients and proxies as we can directly create + # a service that targets the workload with the given app name + namespaceOverride: kube-system + clients: + enabled: false + proxy: + enabled: false + service: + selector: + app.kubernetes.io/name: rke2-ingress-nginx + kubeVersionOverrides: + - constraint: "< 1.21.0-0" + values: + namespaceOverride: "" + clients: + enabled: true + port: 10015 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "app.kubernetes.io/component" + operator: "In" + values: + - "controller" + topologyKey: "kubernetes.io/hostname" + namespaces: + - "kube-system" + # in the RKE2 cluster, the ingress-nginx-controller is deployed as + # a DaemonSet with 1 pod when RKE2 version is < 1.21.0-0 + deployment: + enabled: false + proxy: + enabled: true + service: + selector: false + - constraint: ">= 1.21.0-0 < 1.22.12-0" + values: + namespaceOverride: "" + clients: + enabled: true + port: 10015 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "app.kubernetes.io/component" + operator: "In" + values: + - "controller" + topologyKey: "kubernetes.io/hostname" + namespaces: + - "kube-system" + # in the RKE2 cluster, the ingress-nginx-controller is deployed as + # a hostNetwork Deployment with 1 pod when RKE2 version is >= 1.21.0-0 + deployment: + enabled: true + replicas: 1 + proxy: + enabled: true + service: + selector: false + - constraint: ">= 1.23.0-0 < v1.23.9-0" + values: + namespaceOverride: "" + clients: + enabled: true + port: 10015 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "app.kubernetes.io/component" + operator: "In" + values: + - "controller" + topologyKey: "kubernetes.io/hostname" + namespaces: + - "kube-system" + # in the RKE2 cluster, the ingress-nginx-controller is deployed as + # a hostNetwork Deployment with 1 pod when RKE2 version is >= 1.20.0-0 + deployment: + enabled: true + replicas: 1 + proxy: + enabled: true + service: + selector: false + - constraint: ">= 1.24.0-0 < v1.24.3-0" + values: + namespaceOverride: "" + clients: + enabled: true + port: 10015 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "app.kubernetes.io/component" + operator: "In" + values: + - "controller" + topologyKey: "kubernetes.io/hostname" + namespaces: + - "kube-system" + # in the RKE2 cluster, the ingress-nginx-controller is deployed as + # a hostNetwork Deployment with 1 pod when RKE2 version is >= 1.20.0-0 + deployment: + enabled: true + replicas: 1 + proxy: + enabled: true + service: + selector: false + + + +## Additional PushProx Monitoring +## ref: https://github.com/rancher/charts/tree/dev-v2.5-source/packages/rancher-pushprox +## + +# hardenedKubelet can only be deployed if kubelet.enabled=true +# If enabled, it replaces the ServiceMonitor deployed by the default kubelet option with a +# PushProx-based exporter that does not require a host port to be open to scrape metrics. +hardenedKubelet: + enabled: false + metricsPort: 10250 + component: kubelet + clients: + port: 10015 + useLocalhost: true + https: + enabled: true + useServiceAccountCredentials: true + insecureSkipVerify: true + rbac: + additionalRules: + - nonResourceURLs: ["/metrics/cadvisor"] + verbs: ["get"] + - apiGroups: [""] + resources: ["nodes/metrics"] + verbs: ["get"] + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + serviceMonitor: + endpoints: + - port: metrics + honorLabels: true + relabelings: + - sourceLabels: [__metrics_path__] + targetLabel: metrics_path + - port: metrics + path: /metrics/cadvisor + honorLabels: true + relabelings: + - sourceLabels: [__metrics_path__] + targetLabel: metrics_path + - port: metrics + path: /metrics/probes + honorLabels: true + relabelings: + - sourceLabels: [__metrics_path__] + targetLabel: metrics_path + +# hardenedNodeExporter can only be deployed if nodeExporter.enabled=true +# If enabled, it replaces the ServiceMonitor deployed by the default nodeExporter with a +# PushProx-based exporter that does not require a host port to be open to scrape metrics. +hardenedNodeExporter: + enabled: false + metricsPort: 9796 + component: node-exporter + clients: + port: 10016 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +## Upgrades +upgrade: + ## Run upgrade scripts before an upgrade or rollback via a Job hook + enabled: true + ## Image to use to run the scripts + image: + repository: rancher/shell + tag: v0.1.25 + +## Rancher Monitoring +## + +rancherMonitoring: + enabled: true + + ## A namespaceSelector to identify the namespace to find the Rancher deployment + ## + namespaceSelector: + matchNames: + - cattle-system + + ## A selector to identify the Rancher deployment + ## If not set, the chart will try to search for the Rancher deployment in the cattle-system namespace and infer the selector values from it + ## If the Rancher deployment does not exist, no resources will be deployed. + ## + selector: {} + +## Component scraping nginx-ingress-controller +## +ingressNginx: + enabled: false + + ## The namespace to search for your nginx-ingress-controller + ## + namespace: ingress-nginx + + service: + port: 9913 + targetPort: 10254 + # selector: + # app: ingress-nginx + serviceMonitor: + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "30s" + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## metric relabel configs to apply to samples before ingestion. + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + # relabel configs to apply to samples before ingestion. + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + +# Prometheus Operator Configuration + +## Provide a name in place of kube-prometheus-stack for `app:` labels +## NOTE: If you change this value, you must update the prometheus-adapter.prometheus.url +## +nameOverride: "rancher-monitoring" + +## Override the deployment namespace +## NOTE: If you change this value, you must update the prometheus-adapter.prometheus.url +## +namespaceOverride: "cattle-monitoring-system" + +## Provide a k8s version to auto dashboard import script example: kubeTargetVersionOverride: 1.16.6 +## +kubeTargetVersionOverride: "" + +## Allow kubeVersion to be overridden while creating the ingress +## +kubeVersionOverride: "" + +## Provide a name to substitute for the full names of resources +## +fullnameOverride: "" + +## Labels to apply to all resources +## +commonLabels: {} +# scmhash: abc123 +# myLabel: aakkmd + +## Create default rules for monitoring the cluster +## +defaultRules: + create: true + rules: + alertmanager: true + etcd: true + configReloaders: true + general: true + k8s: true + kubeApiserverAvailability: true + kubeApiserverBurnrate: true + kubeApiserverHistogram: true + kubeApiserverSlos: true + kubeControllerManager: true + kubelet: true + kubeProxy: true + kubePrometheusGeneral: true + kubePrometheusNodeRecording: true + kubernetesApps: true + kubernetesResources: true + kubernetesStorage: true + kubernetesSystem: true + kubeScheduler: true + kubeStateMetrics: true + network: true + node: true + nodeExporterAlerting: true + nodeExporterRecording: true + prometheus: true + prometheusOperator: true + + ## Reduce app namespace alert scope + appNamespacesTarget: ".*" + + ## Labels for default rules + labels: {} + ## Annotations for default rules + annotations: {} + + ## Additional labels for PrometheusRule alerts + additionalRuleLabels: {} + + ## Additional annotations for PrometheusRule alerts + additionalRuleAnnotations: {} + + ## Prefix for runbook URLs. Use this to override the first part of the runbookURLs that is common to all rules. + runbookUrl: "https://runbooks.prometheus-operator.dev/runbooks" + + ## Disabled PrometheusRule alerts + disabled: {} + # KubeAPIDown: true + # NodeRAIDDegraded: true + +## Deprecated way to provide custom recording or alerting rules to be deployed into the cluster. +## +# additionalPrometheusRules: [] +# - name: my-rule-file +# groups: +# - name: my_group +# rules: +# - record: my_record +# expr: 100 * my_record + +## Provide custom recording or alerting rules to be deployed into the cluster. +## +additionalPrometheusRulesMap: {} +# rule-name: +# groups: +# - name: my_group +# rules: +# - record: my_record +# expr: 100 * my_record + +## +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + ## Windows Monitoring + ## ref: https://github.com/rancher/charts/tree/dev-v2.5-source/packages/rancher-windows-exporter + ## + ## Deploys a DaemonSet of Prometheus exporters based on https://github.com/prometheus-community/windows_exporter. + ## Every Windows host must have a wins version of 0.1.0+ to use this chart (default as of Rancher 2.5.8). + ## To upgrade wins versions on Windows hosts, see https://github.com/rancher/wins/tree/master/charts/rancher-wins-upgrader. + ## + windows: + enabled: false + seLinux: + enabled: false + kubectl: + repository: rancher/kubectl + tag: v1.20.2 + pullPolicy: IfNotPresent + rbac: + ## Create RBAC resources for ServiceAccounts and users + ## + create: true + + userRoles: + ## Create default user ClusterRoles to allow users to interact with Prometheus CRs, ConfigMaps, and Secrets + create: true + ## Aggregate default user ClusterRoles into default k8s ClusterRoles + aggregateToDefaultRoles: true + + pspAnnotations: {} + ## Specify pod annotations + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#apparmor + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#seccomp + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#sysctl + ## + # seccomp.security.alpha.kubernetes.io/allowedProfileNames: '*' + # seccomp.security.alpha.kubernetes.io/defaultProfileName: 'docker/default' + # apparmor.security.beta.kubernetes.io/defaultProfileName: 'runtime/default' + + ## Reference to one or more secrets to be used when pulling images + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + imagePullSecrets: [] + # - name: "image-pull-secret" + # or + # - "image-pull-secret" + +## Configuration for alertmanager +## ref: https://prometheus.io/docs/alerting/alertmanager/ +## +alertmanager: + + ## Deploy alertmanager + ## + enabled: true + + ## Annotations for Alertmanager + ## + annotations: {} + + ## Api that prometheus will use to communicate with alertmanager. Possible values are v1, v2 + ## + apiVersion: v2 + + ## Service account for Alertmanager to use. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + ## + serviceAccount: + create: true + name: "" + annotations: {} + + ## Configure pod disruption budgets for Alertmanager + ## ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/#specifying-a-poddisruptionbudget + ## This configuration is immutable once created and will require the PDB to be deleted to be changed + ## https://github.com/kubernetes/kubernetes/issues/45398 + ## + podDisruptionBudget: + enabled: false + minAvailable: 1 + maxUnavailable: "" + + ## Alertmanager configuration directives + ## ref: https://prometheus.io/docs/alerting/configuration/#configuration-file + ## https://prometheus.io/webtools/alerting/routing-tree-editor/ + ## + config: + global: + resolve_timeout: 5m + inhibit_rules: + - source_matchers: + - 'severity = critical' + target_matchers: + - 'severity =~ warning|info' + equal: + - 'namespace' + - 'alertname' + - source_matchers: + - 'severity = warning' + target_matchers: + - 'severity = info' + equal: + - 'namespace' + - 'alertname' + - source_matchers: + - 'alertname = InfoInhibitor' + target_matchers: + - 'severity = info' + equal: + - 'namespace' + route: + group_by: ['namespace'] + group_wait: 30s + group_interval: 5m + repeat_interval: 12h + receiver: 'null' + routes: + - receiver: 'null' + matchers: + - alertname =~ "InfoInhibitor|Watchdog" + receivers: + - name: 'null' + templates: + - '/etc/alertmanager/config/*.tmpl' + + ## Pass the Alertmanager configuration directives through Helm's templating + ## engine. If the Alertmanager configuration contains Alertmanager templates, + ## they'll need to be properly escaped so that they are not interpreted by + ## Helm + ## ref: https://helm.sh/docs/developing_charts/#using-the-tpl-function + ## https://prometheus.io/docs/alerting/configuration/#tmpl_string + ## https://prometheus.io/docs/alerting/notifications/ + ## https://prometheus.io/docs/alerting/notification_examples/ + tplConfig: false + + ## Alertmanager template files to format alerts + ## By default, templateFiles are placed in /etc/alertmanager/config/ and if + ## they have a .tmpl file suffix will be loaded. See config.templates above + ## to change, add other suffixes. If adding other suffixes, be sure to update + ## config.templates above to include those suffixes. + ## ref: https://prometheus.io/docs/alerting/notifications/ + ## https://prometheus.io/docs/alerting/notification_examples/ + ## + + templateFiles: + rancher_defaults.tmpl: |- + {{- define "slack.rancher.text" -}} + {{ template "rancher.text_multiple" . }} + {{- end -}} + + {{- define "rancher.text_multiple" -}} + *[GROUP - Details]* + One or more alarms in this group have triggered a notification. + + {{- if gt (len .GroupLabels.Values) 0 }} + *Group Labels:* + {{- range .GroupLabels.SortedPairs }} + • *{{ .Name }}:* `{{ .Value }}` + {{- end }} + {{- end }} + {{- if .ExternalURL }} + *Link to AlertManager:* {{ .ExternalURL }} + {{- end }} + + {{- range .Alerts }} + {{ template "rancher.text_single" . }} + {{- end }} + {{- end -}} + + {{- define "rancher.text_single" -}} + {{- if .Labels.alertname }} + *[ALERT - {{ .Labels.alertname }}]* + {{- else }} + *[ALERT]* + {{- end }} + {{- if .Labels.severity }} + *Severity:* `{{ .Labels.severity }}` + {{- end }} + {{- if .Labels.cluster }} + *Cluster:* {{ .Labels.cluster }} + {{- end }} + {{- if .Annotations.summary }} + *Summary:* {{ .Annotations.summary }} + {{- end }} + {{- if .Annotations.message }} + *Message:* {{ .Annotations.message }} + {{- end }} + {{- if .Annotations.description }} + *Description:* {{ .Annotations.description }} + {{- end }} + {{- if .Annotations.runbook_url }} + *Runbook URL:* <{{ .Annotations.runbook_url }}|:spiral_note_pad:> + {{- end }} + {{- with .Labels }} + {{- with .Remove (stringSlice "alertname" "severity" "cluster") }} + {{- if gt (len .) 0 }} + *Additional Labels:* + {{- range .SortedPairs }} + • *{{ .Name }}:* `{{ .Value }}` + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- with .Annotations }} + {{- with .Remove (stringSlice "summary" "message" "description" "runbook_url") }} + {{- if gt (len .) 0 }} + *Additional Annotations:* + {{- range .SortedPairs }} + • *{{ .Name }}:* `{{ .Value }}` + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end -}} + + ingress: + enabled: false + + # For Kubernetes >= 1.18 you should specify the ingress-controller via the field ingressClassName + # See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#specifying-the-class-of-an-ingress + # ingressClassName: nginx + + annotations: {} + + labels: {} + + ## Redirect ingress to an additional defined port on the service + # servicePort: 8081 + + ## Hosts must be provided if Ingress is enabled. + ## + hosts: [] + # - alertmanager.domain.com + + ## Paths to use for ingress rules - one path should match the alertmanagerSpec.routePrefix + ## + paths: [] + # - / + + ## For Kubernetes >= 1.18 you should specify the pathType (determines how Ingress paths should be matched) + ## See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#better-path-matching-with-path-types + # pathType: ImplementationSpecific + + ## TLS configuration for Alertmanager Ingress + ## Secret must be manually created in the namespace + ## + tls: [] + # - secretName: alertmanager-general-tls + # hosts: + # - alertmanager.example.com + + ## Configuration for Alertmanager secret + ## + secret: + annotations: {} + + ## Configuration for creating an Ingress that will map to each Alertmanager replica service + ## alertmanager.servicePerReplica must be enabled + ## + ingressPerReplica: + enabled: false + + # For Kubernetes >= 1.18 you should specify the ingress-controller via the field ingressClassName + # See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#specifying-the-class-of-an-ingress + # ingressClassName: nginx + + annotations: {} + labels: {} + + ## Final form of the hostname for each per replica ingress is + ## {{ ingressPerReplica.hostPrefix }}-{{ $replicaNumber }}.{{ ingressPerReplica.hostDomain }} + ## + ## Prefix for the per replica ingress that will have `-$replicaNumber` + ## appended to the end + hostPrefix: "" + ## Domain that will be used for the per replica ingress + hostDomain: "" + + ## Paths to use for ingress rules + ## + paths: [] + # - / + + ## For Kubernetes >= 1.18 you should specify the pathType (determines how Ingress paths should be matched) + ## See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#better-path-matching-with-path-types + # pathType: ImplementationSpecific + + ## Secret name containing the TLS certificate for alertmanager per replica ingress + ## Secret must be manually created in the namespace + tlsSecretName: "" + + ## Separated secret for each per replica Ingress. Can be used together with cert-manager + ## + tlsSecretPerReplica: + enabled: false + ## Final form of the secret for each per replica ingress is + ## {{ tlsSecretPerReplica.prefix }}-{{ $replicaNumber }} + ## + prefix: "alertmanager" + + ## Configuration for Alertmanager service + ## + service: + annotations: {} + labels: {} + clusterIP: "" + + ## Port for Alertmanager Service to listen on + ## + port: 9093 + ## To be used with a proxy extraContainer port + ## + targetPort: 9093 + ## Port to expose on each node + ## Only used if service.type is 'NodePort' + ## + nodePort: 30903 + ## List of IP addresses at which the Prometheus server service is available + ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips + ## + + ## Additional ports to open for Alertmanager service + additionalPorts: [] + # additionalPorts: + # - name: authenticated + # port: 8081 + # targetPort: 8081 + + externalIPs: [] + loadBalancerIP: "" + loadBalancerSourceRanges: [] + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## + type: ClusterIP + + ## Configuration for creating a separate Service for each statefulset Alertmanager replica + ## + servicePerReplica: + enabled: false + annotations: {} + + ## Port for Alertmanager Service per replica to listen on + ## + port: 9093 + + ## To be used with a proxy extraContainer port + targetPort: 9093 + + ## Port to expose on each node + ## Only used if servicePerReplica.type is 'NodePort' + ## + nodePort: 30904 + + ## Loadbalancer source IP ranges + ## Only used if servicePerReplica.type is "LoadBalancer" + loadBalancerSourceRanges: [] + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## + type: ClusterIP + + ## If true, create a serviceMonitor for alertmanager + ## + serviceMonitor: + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + selfMonitor: true + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## scheme: HTTP scheme to use for scraping. Can be used with `tlsConfig` for example if using istio mTLS. + scheme: "" + + ## tlsConfig: TLS configuration to use when scraping the endpoint. For example if using istio mTLS. + ## Of type: https://github.com/coreos/prometheus-operator/blob/main/Documentation/api.md#tlsconfig + tlsConfig: {} + + bearerTokenFile: + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Settings affecting alertmanagerSpec + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#alertmanagerspec + ## + alertmanagerSpec: + ## Standard object's metadata. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#metadata + ## Metadata Labels and Annotations gets propagated to the Alertmanager pods. + ## + podMetadata: {} + + ## Image of Alertmanager + ## + image: + repository: rancher/mirrored-prometheus-alertmanager + tag: v0.24.0 + sha: "" + + ## If true then the user will be responsible to provide a secret with alertmanager configuration + ## So when true the config part will be ignored (including templateFiles) and the one in the secret will be used + ## + useExistingSecret: false + + ## Secrets is a list of Secrets in the same namespace as the Alertmanager object, which shall be mounted into the + ## Alertmanager Pods. The Secrets are mounted into /etc/alertmanager/secrets/. + ## + secrets: [] + + ## ConfigMaps is a list of ConfigMaps in the same namespace as the Alertmanager object, which shall be mounted into the Alertmanager Pods. + ## The ConfigMaps are mounted into /etc/alertmanager/configmaps/. + ## + configMaps: [] + + ## ConfigSecret is the name of a Kubernetes Secret in the same namespace as the Alertmanager object, which contains configuration for + ## this Alertmanager instance. Defaults to 'alertmanager-' The secret is mounted into /etc/alertmanager/config. + ## + # configSecret: + + ## WebTLSConfig defines the TLS parameters for HTTPS + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#alertmanagerwebspec + web: {} + + ## AlertmanagerConfigs to be selected to merge and configure Alertmanager with. + ## + alertmanagerConfigSelector: {} + ## Example which selects all alertmanagerConfig resources + ## with label "alertconfig" with values any of "example-config" or "example-config-2" + # alertmanagerConfigSelector: + # matchExpressions: + # - key: alertconfig + # operator: In + # values: + # - example-config + # - example-config-2 + # + ## Example which selects all alertmanagerConfig resources with label "role" set to "example-config" + # alertmanagerConfigSelector: + # matchLabels: + # role: example-config + + ## Namespaces to be selected for AlertmanagerConfig discovery. If nil, only check own namespace. + ## + alertmanagerConfigNamespaceSelector: {} + ## Example which selects all namespaces + ## with label "alertmanagerconfig" with values any of "example-namespace" or "example-namespace-2" + # alertmanagerConfigNamespaceSelector: + # matchExpressions: + # - key: alertmanagerconfig + # operator: In + # values: + # - example-namespace + # - example-namespace-2 + + ## Example which selects all namespaces with label "alertmanagerconfig" set to "enabled" + # alertmanagerConfigNamespaceSelector: + # matchLabels: + # alertmanagerconfig: enabled + + ## AlermanagerConfig to be used as top level configuration + ## + alertmanagerConfiguration: {} + ## Example with select a global alertmanagerconfig + # alertmanagerConfiguration: + # name: global-alertmanager-Configuration + + ## Define Log Format + # Use logfmt (default) or json logging + logFormat: logfmt + + ## Log level for Alertmanager to be configured with. + ## + logLevel: info + + ## Size is the expected size of the alertmanager cluster. The controller will eventually make the size of the + ## running cluster equal to the expected size. + replicas: 1 + + ## Time duration Alertmanager shall retain data for. Default is '120h', and must match the regular expression + ## [0-9]+(ms|s|m|h) (milliseconds seconds minutes hours). + ## + retention: 120h + + ## Storage is the definition of how storage will be used by the Alertmanager instances. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/storage.md + ## + storage: {} + # volumeClaimTemplate: + # spec: + # storageClassName: gluster + # accessModes: ["ReadWriteOnce"] + # resources: + # requests: + # storage: 50Gi + # selector: {} + + + ## The external URL the Alertmanager instances will be available under. This is necessary to generate correct URLs. This is necessary if Alertmanager is not served from root of a DNS name. string false + ## + externalUrl: + + ## The route prefix Alertmanager registers HTTP handlers for. This is useful, if using ExternalURL and a proxy is rewriting HTTP routes of a request, and the actual ExternalURL is still true, + ## but the server serves requests under a different route prefix. For example for use with kubectl proxy. + ## + routePrefix: / + + ## If set to true all actions on the underlying managed objects are not going to be performed, except for delete actions. + ## + paused: false + + ## Define which Nodes the Pods are scheduled on. + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + ## Define resources requests and limits for single Pods. + ## ref: https://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: + limits: + memory: 500Mi + cpu: 1000m + requests: + memory: 100Mi + cpu: 100m + + ## Pod anti-affinity can prevent the scheduler from placing Prometheus replicas on the same node. + ## The default value "soft" means that the scheduler should *prefer* to not schedule two replica pods onto the same node but no guarantee is provided. + ## The value "hard" means that the scheduler is *required* to not schedule two replica pods onto the same node. + ## The value "" will disable pod anti-affinity so that no anti-affinity rules will be configured. + ## + podAntiAffinity: "" + + ## If anti-affinity is enabled sets the topologyKey to use for anti-affinity. + ## This can be changed to, for example, failure-domain.beta.kubernetes.io/zone + ## + podAntiAffinityTopologyKey: kubernetes.io/hostname + + ## Assign custom affinity rules to the alertmanager instance + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + ## + affinity: {} + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: kubernetes.io/e2e-az-name + # operator: In + # values: + # - e2e-az1 + # - e2e-az2 + + ## If specified, the pod's tolerations. + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + + ## If specified, the pod's topology spread constraints. + ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ + ## + topologySpreadConstraints: [] + # - maxSkew: 1 + # topologyKey: topology.kubernetes.io/zone + # whenUnsatisfiable: DoNotSchedule + # labelSelector: + # matchLabels: + # app: alertmanager + + ## SecurityContext holds pod-level security attributes and common container settings. + ## This defaults to non root user with uid 1000 and gid 2000. *v1.PodSecurityContext false + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + ## + securityContext: + runAsGroup: 2000 + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 2000 + + ## ListenLocal makes the Alertmanager server listen on loopback, so that it does not bind against the Pod IP. + ## Note this is only for the Alertmanager UI, not the gossip communication. + ## + listenLocal: false + + ## Containers allows injecting additional containers. This is meant to allow adding an authentication proxy to an Alertmanager pod. + ## + containers: [] + # containers: + # - name: oauth-proxy + # image: quay.io/oauth2-proxy/oauth2-proxy:v7.3.0 + # args: + # - --upstream=http://127.0.0.1:9093 + # - --http-address=0.0.0.0:8081 + # - ... + # ports: + # - containerPort: 8081 + # name: oauth-proxy + # protocol: TCP + # resources: {} + + # Additional volumes on the output StatefulSet definition. + volumes: [] + + # Additional VolumeMounts on the output StatefulSet definition. + volumeMounts: [] + + ## InitContainers allows injecting additional initContainers. This is meant to allow doing some changes + ## (permissions, dir tree) on mounted volumes before starting prometheus + initContainers: [] + + ## Priority class assigned to the Pods + ## + priorityClassName: "" + + ## AdditionalPeers allows injecting a set of additional Alertmanagers to peer with to form a highly available cluster. + ## + additionalPeers: [] + + ## PortName to use for Alert Manager. + ## + portName: "http-web" + + ## ClusterAdvertiseAddress is the explicit address to advertise in cluster. Needs to be provided for non RFC1918 [1] (public) addresses. [1] RFC1918: https://tools.ietf.org/html/rfc1918 + ## + clusterAdvertiseAddress: false + + ## ForceEnableClusterMode ensures Alertmanager does not deactivate the cluster mode when running with a single replica. + ## Use case is e.g. spanning an Alertmanager cluster across Kubernetes clusters with a single replica in each. + forceEnableClusterMode: false + + ## Minimum number of seconds for which a newly created pod should be ready without any of its container crashing for it to + ## be considered available. Defaults to 0 (pod will be considered available as soon as it is ready). + minReadySeconds: 0 + + ## ExtraSecret can be used to store various data in an extra secret + ## (use it for example to store hashed basic auth credentials) + extraSecret: + ## if not set, name will be auto generated + # name: "" + annotations: {} + data: {} + # auth: | + # foo:$apr1$OFG3Xybp$ckL0FHDAkoXYIlH9.cysT0 + # someoneelse:$apr1$DMZX2Z4q$6SbQIfyuLQd.xmo/P0m2c. + +## Using default values from https://github.com/grafana/helm-charts/blob/main/charts/grafana/values.yaml +## +grafana: + enabled: true + namespaceOverride: "" + + ## Grafana's primary configuration + ## NOTE: values in map will be converted to ini format + ## ref: http://docs.grafana.org/installation/configuration/ + ## + grafana.ini: + users: + auto_assign_org_role: Viewer + auth: + disable_login_form: false + auth.anonymous: + enabled: true + org_role: Viewer + auth.basic: + enabled: false + dashboards: + # Modify this value to change the default dashboard shown on the main Grafana page + default_home_dashboard_path: /tmp/dashboards/rancher-default-home.json + security: + # Required to embed dashboards in Rancher Cluster Overview Dashboard on Cluster Explorer + allow_embedding: true + + deploymentStrategy: + type: Recreate + + ## ForceDeployDatasources Create datasource configmap even if grafana deployment has been disabled + ## + forceDeployDatasources: false + + ## ForceDeployDashboard Create dashboard configmap even if grafana deployment has been disabled + ## + forceDeployDashboards: false + + ## Deploy default dashboards + ## + defaultDashboardsEnabled: true + + # Additional options for defaultDashboards + defaultDashboards: + # The default namespace to place defaultDashboards within + namespace: cattle-dashboards + # Whether to create the default namespace as a Helm managed namespace or use an existing namespace + # If false, the defaultDashboards.namespace will be created as a Helm managed namespace + useExistingNamespace: false + # Whether the Helm managed namespace created by this chart should be left behind on a Helm uninstall + # If you place other dashboards in this namespace, then they will be deleted on a helm uninstall + # Ignore if useExistingNamespace is true + cleanupOnUninstall: false + + ## Timezone for the default dashboards + ## Other options are: browser or a specific timezone, i.e. Europe/Luxembourg + ## + defaultDashboardsTimezone: utc + + adminPassword: prom-operator + + ingress: + ## If true, Grafana Ingress will be created + ## + enabled: false + + ## IngressClassName for Grafana Ingress. + ## Should be provided if Ingress is enable. + ## + # ingressClassName: nginx + + ## Annotations for Grafana Ingress + ## + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + + ## Labels to be added to the Ingress + ## + labels: {} + + ## Hostnames. + ## Must be provided if Ingress is enable. + ## + # hosts: + # - grafana.domain.com + hosts: [] + + ## Path for grafana ingress + path: / + + ## TLS configuration for grafana Ingress + ## Secret must be manually created in the namespace + ## + tls: [] + # - secretName: grafana-general-tls + # hosts: + # - grafana.example.com + + sidecar: + dashboards: + enabled: true + label: grafana_dashboard + searchNamespace: cattle-dashboards + labelValue: "1" + + ## Annotations for Grafana dashboard configmaps + ## + annotations: {} + multicluster: + global: + enabled: false + etcd: + enabled: false + provider: + allowUiUpdates: false + datasources: + enabled: true + defaultDatasourceEnabled: true + + uid: prometheus + + ## URL of prometheus datasource + ## + # url: http://prometheus-stack-prometheus:9090/ + + # If not defined, will use prometheus.prometheusSpec.scrapeInterval or its default + # defaultDatasourceScrapeInterval: 15s + + ## Annotations for Grafana datasource configmaps + ## + annotations: {} + + ## Create datasource for each Pod of Prometheus StatefulSet; + ## this uses headless service `prometheus-operated` which is + ## created by Prometheus Operator + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/0fee93e12dc7c2ea1218f19ae25ec6b893460590/pkg/prometheus/statefulset.go#L255-L286 + createPrometheusReplicasDatasources: false + label: grafana_datasource + labelValue: "1" + + ## Field with internal link pointing to existing data source in Grafana. + ## Can be provisioned via additionalDataSources + exemplarTraceIdDestinations: {} + # datasourceUid: Jaeger + # traceIdLabelName: trace_id + + extraConfigmapMounts: [] + # - name: certs-configmap + # mountPath: /etc/grafana/ssl/ + # configMap: certs-configmap + # readOnly: true + + deleteDatasources: [] + # - name: example-datasource + # orgId: 1 + + ## Configure additional grafana datasources (passed through tpl) + ## ref: http://docs.grafana.org/administration/provisioning/#datasources + additionalDataSources: [] + # - name: prometheus-sample + # access: proxy + # basicAuth: true + # basicAuthPassword: pass + # basicAuthUser: daco + # editable: false + # jsonData: + # tlsSkipVerify: true + # orgId: 1 + # type: prometheus + # url: https://{{ printf "%s-prometheus.svc" .Release.Name }}:9090 + # version: 1 + + ## Passed to grafana subchart and used by servicemonitor below + ## + service: + portName: nginx-http + ## Port for Grafana Service to listen on + ## + port: 80 + ## To be used with a proxy extraContainer port + ## + targetPort: 8080 + ## Port to expose on each node + ## Only used if service.type is 'NodePort' + ## + nodePort: 30950 + ## Service type + ## + type: ClusterIP + + proxy: + image: + repository: rancher/mirrored-library-nginx + tag: 1.24.0-alpine + + ## Enable an Specify container in extraContainers. This is meant to allow adding an authentication proxy to a grafana pod + extraContainers: | + - name: grafana-proxy + args: + - nginx + - -g + - daemon off; + - -c + - /nginx/nginx.conf + image: "{{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }}" + ports: + - containerPort: 8080 + name: nginx-http + protocol: TCP + volumeMounts: + - mountPath: /nginx + name: grafana-nginx + - mountPath: /var/cache/nginx + name: nginx-home + securityContext: + runAsUser: 101 + runAsGroup: 101 + + ## Volumes that can be used in containers + extraContainerVolumes: + - name: nginx-home + emptyDir: {} + - name: grafana-nginx + configMap: + name: grafana-nginx-proxy-config + items: + - key: nginx.conf + mode: 438 + path: nginx.conf + + ## If true, create a serviceMonitor for grafana + ## + serviceMonitor: + # If true, a ServiceMonitor CRD is created for a prometheus operator + # https://github.com/coreos/prometheus-operator + # + enabled: true + + # Path to use for scraping metrics. Might be different if server.root_url is set + # in grafana.ini + path: "/metrics" + + # namespace: monitoring (defaults to use the namespace this chart is deployed to) + + # labels for the ServiceMonitor + labels: {} + + # Scrape interval. If not set, the Prometheus default scrape interval is used. + # + interval: "" + scheme: http + tlsConfig: {} + scrapeTimeout: 30s + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + resources: + limits: + memory: 200Mi + cpu: 200m + requests: + memory: 100Mi + cpu: 100m + + testFramework: + enabled: false + +## Component scraping the kube api server +## +kubeApiServer: + enabled: true + tlsConfig: + serverName: kubernetes + insecureSkipVerify: false + serviceMonitor: + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + jobLabel: component + selector: + matchLabels: + component: apiserver + provider: kubernetes + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: + # Drop excessively noisy apiserver buckets. + - action: drop + regex: apiserver_request_duration_seconds_bucket;(0.15|0.2|0.3|0.35|0.4|0.45|0.6|0.7|0.8|0.9|1.25|1.5|1.75|2|3|3.5|4|4.5|6|7|8|9|15|25|40|50) + sourceLabels: + - __name__ + - le + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: + # - __meta_kubernetes_namespace + # - __meta_kubernetes_service_name + # - __meta_kubernetes_endpoint_port_name + # action: keep + # regex: default;kubernetes;https + # - targetLabel: __address__ + # replacement: kubernetes.default.svc:443 + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping the kubelet and kubelet-hosted cAdvisor +## +kubelet: + enabled: true + namespace: kube-system + + serviceMonitor: + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## Enable scraping the kubelet over https. For requirements to enable this see + ## https://github.com/prometheus-operator/prometheus-operator/issues/926 + ## + https: true + + ## Enable scraping /metrics/cadvisor from kubelet's service + ## + cAdvisor: true + + ## Enable scraping /metrics/probes from kubelet's service + ## + probes: true + + ## Enable scraping /metrics/resource from kubelet's service + ## This is disabled by default because container metrics are already exposed by cAdvisor + ## + resource: false + # From kubernetes 1.18, /metrics/resource/v1alpha1 renamed to /metrics/resource + resourcePath: "/metrics/resource/v1alpha1" + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + cAdvisorMetricRelabelings: + # Drop less useful container CPU metrics. + - sourceLabels: [__name__] + action: drop + regex: 'container_cpu_(cfs_throttled_seconds_total|load_average_10s|system_seconds_total|user_seconds_total)' + # Drop less useful container / always zero filesystem metrics. + - sourceLabels: [__name__] + action: drop + regex: 'container_fs_(io_current|io_time_seconds_total|io_time_weighted_seconds_total|reads_merged_total|sector_reads_total|sector_writes_total|writes_merged_total)' + # Drop less useful / always zero container memory metrics. + - sourceLabels: [__name__] + action: drop + regex: 'container_memory_(mapped_file|swap)' + # Drop less useful container process metrics. + - sourceLabels: [__name__] + action: drop + regex: 'container_(file_descriptors|tasks_state|threads_max)' + # Drop container spec metrics that overlap with kube-state-metrics. + - sourceLabels: [__name__] + action: drop + regex: 'container_spec.*' + # Drop cgroup metrics with no pod. + - sourceLabels: [id, pod] + action: drop + regex: '.+;' + # - sourceLabels: [__name__, image] + # separator: ; + # regex: container_([a-z_]+); + # replacement: $1 + # action: drop + # - sourceLabels: [__name__] + # separator: ; + # regex: container_(network_tcp_usage_total|network_udp_usage_total|tasks_state|cpu_load_average_10s) + # replacement: $1 + # action: drop + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + probesMetricRelabelings: [] + # - sourceLabels: [__name__, image] + # separator: ; + # regex: container_([a-z_]+); + # replacement: $1 + # action: drop + # - sourceLabels: [__name__] + # separator: ; + # regex: container_(network_tcp_usage_total|network_udp_usage_total|tasks_state|cpu_load_average_10s) + # replacement: $1 + # action: drop + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + ## metrics_path is required to match upstream rules and charts + cAdvisorRelabelings: + - sourceLabels: [__metrics_path__] + targetLabel: metrics_path + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + probesRelabelings: + - sourceLabels: [__metrics_path__] + targetLabel: metrics_path + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + resourceRelabelings: + - sourceLabels: [__metrics_path__] + targetLabel: metrics_path + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - sourceLabels: [__name__, image] + # separator: ; + # regex: container_([a-z_]+); + # replacement: $1 + # action: drop + # - sourceLabels: [__name__] + # separator: ; + # regex: container_(network_tcp_usage_total|network_udp_usage_total|tasks_state|cpu_load_average_10s) + # replacement: $1 + # action: drop + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + ## metrics_path is required to match upstream rules and charts + relabelings: + - sourceLabels: [__metrics_path__] + targetLabel: metrics_path + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping the kube controller manager +## +kubeControllerManager: + enabled: false + + ## If your kube controller manager is not deployed as a pod, specify IPs it can be found on + ## + endpoints: [] + # - 10.141.4.22 + # - 10.141.4.23 + # - 10.141.4.24 + + ## If using kubeControllerManager.endpoints only the port and targetPort are used + ## + service: + enabled: true + ## If null or unset, the value is determined dynamically based on target Kubernetes version due to change + ## of default port in Kubernetes 1.22. + ## + port: null + targetPort: null + # selector: + # component: kube-controller-manager + + serviceMonitor: + enabled: true + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## Enable scraping kube-controller-manager over https. + ## Requires proper certs (not self-signed) and delegated authentication/authorization checks. + ## If null or unset, the value is determined dynamically based on target Kubernetes version. + ## + https: null + + # Skip TLS certificate validation when scraping + insecureSkipVerify: null + + # Name of the server to use when validating TLS certificate + serverName: null + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping coreDns. Use either this or kubeDns +## +coreDns: + enabled: true + service: + port: 9153 + targetPort: 9153 + # selector: + # k8s-app: kube-dns + serviceMonitor: + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping kubeDns. Use either this or coreDns +## +kubeDns: + enabled: false + service: + dnsmasq: + port: 10054 + targetPort: 10054 + skydns: + port: 10055 + targetPort: 10055 + # selector: + # k8s-app: kube-dns + serviceMonitor: + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + dnsmasqMetricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + dnsmasqRelabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping etcd +## +kubeEtcd: + enabled: false + + ## If your etcd is not deployed as a pod, specify IPs it can be found on + ## + endpoints: [] + # - 10.141.4.22 + # - 10.141.4.23 + # - 10.141.4.24 + + ## Etcd service. If using kubeEtcd.endpoints only the port and targetPort are used + ## + service: + enabled: true + port: 2381 + targetPort: 2381 + # selector: + # component: etcd + + ## Configure secure access to the etcd cluster by loading a secret into prometheus and + ## specifying security configuration below. For example, with a secret named etcd-client-cert + ## + ## serviceMonitor: + ## scheme: https + ## insecureSkipVerify: false + ## serverName: localhost + ## caFile: /etc/prometheus/secrets/etcd-client-cert/etcd-ca + ## certFile: /etc/prometheus/secrets/etcd-client-cert/etcd-client + ## keyFile: /etc/prometheus/secrets/etcd-client-cert/etcd-client-key + ## + serviceMonitor: + enabled: true + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + scheme: http + insecureSkipVerify: false + serverName: "" + caFile: "" + certFile: "" + keyFile: "" + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping kube scheduler +## +kubeScheduler: + enabled: false + + ## If your kube scheduler is not deployed as a pod, specify IPs it can be found on + ## + endpoints: [] + # - 10.141.4.22 + # - 10.141.4.23 + # - 10.141.4.24 + + ## If using kubeScheduler.endpoints only the port and targetPort are used + ## + service: + enabled: true + ## If null or unset, the value is determined dynamically based on target Kubernetes version due to change + ## of default port in Kubernetes 1.23. + ## + port: null + targetPort: null + # selector: + # component: kube-scheduler + + serviceMonitor: + enabled: true + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + ## Enable scraping kube-scheduler over https. + ## Requires proper certs (not self-signed) and delegated authentication/authorization checks. + ## If null or unset, the value is determined dynamically based on target Kubernetes version. + ## + https: null + + ## Skip TLS certificate validation when scraping + insecureSkipVerify: null + + ## Name of the server to use when validating TLS certificate + serverName: null + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping kube proxy +## +kubeProxy: + enabled: false + + ## If your kube proxy is not deployed as a pod, specify IPs it can be found on + ## + endpoints: [] + # - 10.141.4.22 + # - 10.141.4.23 + # - 10.141.4.24 + + service: + enabled: true + port: 10249 + targetPort: 10249 + # selector: + # k8s-app: kube-proxy + + serviceMonitor: + enabled: true + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## Enable scraping kube-proxy over https. + ## Requires proper certs (not self-signed) and delegated authentication/authorization checks + ## + https: false + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping kube state metrics +## +kubeStateMetrics: + enabled: true + +## Configuration for kube-state-metrics subchart +## +kube-state-metrics: + namespaceOverride: "" + rbac: + create: true + releaseLabel: true + prometheus: + monitor: + enabled: true + + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## Scrape Timeout. If not set, the Prometheus default scrape timeout is used. + ## + scrapeTimeout: "" + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + # Keep labels from scraped data, overriding server-side labels + ## + honorLabels: true + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + selfMonitor: + enabled: false + +## Deploy node exporter as a daemonset to all nodes +## +nodeExporter: + enabled: true + +## Configuration for prometheus-node-exporter subchart +## +prometheus-node-exporter: + namespaceOverride: "" + podLabels: + ## Add the 'node-exporter' label to be used by serviceMonitor to match standard common usage in rules and grafana dashboards + ## + jobLabel: node-exporter + releaseLabel: true + extraArgs: + - --collector.filesystem.mount-points-exclude=^/(dev|proc|sys|var/lib/docker/.+|var/lib/kubelet/.+)($|/) + - --collector.filesystem.fs-types-exclude=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ + service: + portName: http-metrics + prometheus: + monitor: + enabled: true + + jobLabel: jobLabel + + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## How long until a scrape request times out. If not set, the Prometheus default scape timeout is used. + ## + scrapeTimeout: "" + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - sourceLabels: [__name__] + # separator: ; + # regex: ^node_mountstats_nfs_(event|operations|transport)_.+ + # replacement: $1 + # action: drop + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + +## Manages Prometheus and Alertmanager components +## +prometheusOperator: + enabled: true + + ## Prometheus-Operator v0.39.0 and later support TLS natively. + ## + tls: + enabled: true + # Value must match version names from https://golang.org/pkg/crypto/tls/#pkg-constants + tlsMinVersion: VersionTLS13 + # Users who are deploying this chart in GKE private clusters will need to add firewall rules to expose this port for admissions webhooks + internalPort: 8443 + + ## Admission webhook support for PrometheusRules resources added in Prometheus Operator 0.30 can be enabled to prevent incorrectly formatted + ## rules from making their way into prometheus and potentially preventing the container from starting + admissionWebhooks: + failurePolicy: Fail + ## The default timeoutSeconds is 10 and the maximum value is 30. + timeoutSeconds: 10 + enabled: true + ## A PEM encoded CA bundle which will be used to validate the webhook's server certificate. + ## If unspecified, system trust roots on the apiserver are used. + caBundle: "" + ## If enabled, generate a self-signed certificate, then patch the webhook configurations with the generated data. + ## On chart upgrades (or if the secret exists) the cert will not be re-generated. You can use this to provide your own + ## certs ahead of time if you wish. + ## + patch: + enabled: true + image: + repository: rancher/mirrored-ingress-nginx-kube-webhook-certgen + tag: v1.3.0 + sha: "" + pullPolicy: IfNotPresent + resources: {} + ## Provide a priority class name to the webhook patching job + ## + priorityClassName: "" + podAnnotations: {} + nodeSelector: {} + affinity: {} + tolerations: [] + + ## SecurityContext holds pod-level security attributes and common container settings. + ## This defaults to non root user with uid 2000 and gid 2000. *v1.PodSecurityContext false + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + ## + securityContext: + runAsGroup: 2000 + runAsNonRoot: true + runAsUser: 2000 + + # Security context for create job container + createSecretJob: + securityContext: {} + + # Security context for patch job container + patchWebhookJob: + securityContext: {} + + # Use certmanager to generate webhook certs + certManager: + enabled: false + # self-signed root certificate + rootCert: + duration: "" # default to be 5y + admissionCert: + duration: "" # default to be 1y + # issuerRef: + # name: "issuer" + # kind: "ClusterIssuer" + + ## Namespaces to scope the interaction of the Prometheus Operator and the apiserver (allow list). + ## This is mutually exclusive with denyNamespaces. Setting this to an empty object will disable the configuration + ## + namespaces: {} + # releaseNamespace: true + # additional: + # - kube-system + + ## Namespaces not to scope the interaction of the Prometheus Operator (deny list). + ## + denyNamespaces: [] + + ## Filter namespaces to look for prometheus-operator custom resources + ## + alertmanagerInstanceNamespaces: [] + alertmanagerConfigNamespaces: [] + prometheusInstanceNamespaces: [] + thanosRulerInstanceNamespaces: [] + + ## The clusterDomain value will be added to the cluster.peer option of the alertmanager. + ## Without this specified option cluster.peer will have value alertmanager-monitoring-alertmanager-0.alertmanager-operated:9094 (default value) + ## With this specified option cluster.peer will have value alertmanager-monitoring-alertmanager-0.alertmanager-operated.namespace.svc.cluster-domain:9094 + ## + # clusterDomain: "cluster.local" + + ## Service account for Alertmanager to use. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + ## + serviceAccount: + create: true + name: "" + + ## Configuration for Prometheus operator service + ## + service: + annotations: {} + labels: {} + clusterIP: "" + + ## Port to expose on each node + ## Only used if service.type is 'NodePort' + ## + nodePort: 30080 + + nodePortTls: 30443 + + ## Additional ports to open for Prometheus service + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#multi-port-services + ## + additionalPorts: [] + + ## Loadbalancer IP + ## Only use if service.type is "LoadBalancer" + ## + loadBalancerIP: "" + loadBalancerSourceRanges: [] + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## NodePort, ClusterIP, LoadBalancer + ## + type: ClusterIP + + ## List of IP addresses at which the Prometheus server service is available + ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips + ## + externalIPs: [] + + ## Annotations to add to the operator deployment + ## + annotations: {} + + ## Labels to add to the operator pod + ## + podLabels: {} + + ## Annotations to add to the operator pod + ## + podAnnotations: {} + + ## Assign a PriorityClassName to pods if set + # priorityClassName: "" + + ## Define Log Format + # Use logfmt (default) or json logging + # logFormat: logfmt + + ## Decrease log verbosity to errors only + # logLevel: error + + ## If true, the operator will create and maintain a service for scraping kubelets + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/helm/prometheus-operator/README.md + ## + kubeletService: + enabled: true + namespace: kube-system + ## Use '{{ template "kube-prometheus-stack.fullname" . }}-kubelet' by default + name: "" + + ## Create a servicemonitor for the operator + ## + serviceMonitor: + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + ## Scrape timeout. If not set, the Prometheus default scrape timeout is used. + scrapeTimeout: "" + selfMonitor: true + + ## Metric relabel configs to apply to samples before ingestion. + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + # relabel configs to apply to samples before ingestion. + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Resource limits & requests + ## + resources: + limits: + cpu: 200m + memory: 500Mi + requests: + cpu: 100m + memory: 100Mi + + # Required for use in managed kubernetes clusters (such as AWS EKS) with custom CNI (such as calico), + # because control-plane managed by AWS cannot communicate with pods' IP CIDR and admission webhooks are not working + ## + hostNetwork: false + + ## Define which Nodes the Pods are scheduled on. + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + ## Tolerations for use with node taints + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + + ## Assign custom affinity rules to the prometheus operator + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + ## + affinity: {} + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: kubernetes.io/e2e-az-name + # operator: In + # values: + # - e2e-az1 + # - e2e-az2 + dnsConfig: {} + # nameservers: + # - 1.2.3.4 + # searches: + # - ns1.svc.cluster-domain.example + # - my.dns.search.suffix + # options: + # - name: ndots + # value: "2" + # - name: edns0 + securityContext: + fsGroup: 65534 + runAsGroup: 65534 + runAsNonRoot: true + runAsUser: 65534 + + ## Container-specific security context configuration + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + ## + containerSecurityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + + ## Prometheus-operator image + ## + image: + repository: rancher/mirrored-prometheus-operator-prometheus-operator + tag: v0.59.1 + sha: "" + pullPolicy: IfNotPresent + + ## Prometheus image to use for prometheuses managed by the operator + ## + # prometheusDefaultBaseImage: quay.io/prometheus/prometheus + + ## Alertmanager image to use for alertmanagers managed by the operator + ## + # alertmanagerDefaultBaseImage: quay.io/prometheus/alertmanager + + ## Prometheus-config-reloader + ## + prometheusConfigReloader: + image: + repository: rancher/mirrored-prometheus-operator-prometheus-config-reloader + tag: v0.59.1 + sha: "" + + # resource config for prometheusConfigReloader + resources: + requests: + cpu: 200m + memory: 50Mi + limits: + cpu: 200m + memory: 50Mi + + ## Thanos side-car image when configured + ## + thanosImage: + repository: rancher/mirrored-thanos-thanos + tag: v0.28.0 + sha: "" + + ## Set a Field Selector to filter watched secrets + ## + secretFieldSelector: "" + +## Deploy a Prometheus instance +## +prometheus: + + enabled: true + + ## Annotations for Prometheus + ## + annotations: {} + + ## Service account for Prometheuses to use. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + ## + serviceAccount: + create: true + name: "" + annotations: {} + + # Service for thanos service discovery on sidecar + # Enable this can make Thanos Query can use + # `--store=dnssrv+_grpc._tcp.${kube-prometheus-stack.fullname}-thanos-discovery.${namespace}.svc.cluster.local` to discovery + # Thanos sidecar on prometheus nodes + # (Please remember to change ${kube-prometheus-stack.fullname} and ${namespace}. Not just copy and paste!) + thanosService: + enabled: false + annotations: {} + labels: {} + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## + type: ClusterIP + + ## gRPC port config + portName: grpc + port: 10901 + targetPort: "grpc" + + ## HTTP port config (for metrics) + httpPortName: http + httpPort: 10902 + targetHttpPort: "http" + + ## ClusterIP to assign + # Default is to make this a headless service ("None") + clusterIP: "None" + + ## Port to expose on each node, if service type is NodePort + ## + nodePort: 30901 + httpNodePort: 30902 + + # ServiceMonitor to scrape Sidecar metrics + # Needs thanosService to be enabled as well + thanosServiceMonitor: + enabled: false + interval: "" + + ## scheme: HTTP scheme to use for scraping. Can be used with `tlsConfig` for example if using istio mTLS. + scheme: "" + + ## tlsConfig: TLS configuration to use when scraping the endpoint. For example if using istio mTLS. + ## Of type: https://github.com/coreos/prometheus-operator/blob/main/Documentation/api.md#tlsconfig + tlsConfig: {} + + bearerTokenFile: + + ## Metric relabel configs to apply to samples before ingestion. + metricRelabelings: [] + + ## relabel configs to apply to samples before ingestion. + relabelings: [] + + # Service for external access to sidecar + # Enabling this creates a service to expose thanos-sidecar outside the cluster. + thanosServiceExternal: + enabled: false + annotations: {} + labels: {} + loadBalancerIP: "" + loadBalancerSourceRanges: [] + + ## gRPC port config + portName: grpc + port: 10901 + targetPort: "grpc" + + ## HTTP port config (for metrics) + httpPortName: http + httpPort: 10902 + targetHttpPort: "http" + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## + type: LoadBalancer + + ## Port to expose on each node + ## + nodePort: 30901 + httpNodePort: 30902 + + ## Configuration for Prometheus service + ## + service: + annotations: {} + labels: {} + clusterIP: "" + + ## Port for Prometheus Service to listen on + ## + port: 9090 + + ## To be used with a proxy extraContainer port + targetPort: 8081 + + ## List of IP addresses at which the Prometheus server service is available + ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips + ## + externalIPs: [] + + ## Port to expose on each node + ## Only used if service.type is 'NodePort' + ## + nodePort: 30090 + + ## Loadbalancer IP + ## Only use if service.type is "LoadBalancer" + loadBalancerIP: "" + loadBalancerSourceRanges: [] + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## + type: ClusterIP + + ## Additional port to define in the Service + additionalPorts: [] + # additionalPorts: + # - name: authenticated + # port: 8081 + # targetPort: 8081 + + ## Consider that all endpoints are considered "ready" even if the Pods themselves are not + ## Ref: https://kubernetes.io/docs/reference/kubernetes-api/service-resources/service-v1/#ServiceSpec + publishNotReadyAddresses: false + + sessionAffinity: "" + + ## Configuration for creating a separate Service for each statefulset Prometheus replica + ## + servicePerReplica: + enabled: false + annotations: {} + + ## Port for Prometheus Service per replica to listen on + ## + port: 9090 + + ## To be used with a proxy extraContainer port + targetPort: 9090 + + ## Port to expose on each node + ## Only used if servicePerReplica.type is 'NodePort' + ## + nodePort: 30091 + + ## Loadbalancer source IP ranges + ## Only used if servicePerReplica.type is "LoadBalancer" + loadBalancerSourceRanges: [] + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## + type: ClusterIP + + ## Configure pod disruption budgets for Prometheus + ## ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/#specifying-a-poddisruptionbudget + ## This configuration is immutable once created and will require the PDB to be deleted to be changed + ## https://github.com/kubernetes/kubernetes/issues/45398 + ## + podDisruptionBudget: + enabled: false + minAvailable: 1 + maxUnavailable: "" + + # Ingress exposes thanos sidecar outside the cluster + thanosIngress: + enabled: false + + # For Kubernetes >= 1.18 you should specify the ingress-controller via the field ingressClassName + # See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#specifying-the-class-of-an-ingress + # ingressClassName: nginx + + annotations: {} + labels: {} + servicePort: 10901 + + ## Port to expose on each node + ## Only used if service.type is 'NodePort' + ## + nodePort: 30901 + + ## Hosts must be provided if Ingress is enabled. + ## + hosts: [] + # - thanos-gateway.domain.com + + ## Paths to use for ingress rules + ## + paths: [] + # - / + + ## For Kubernetes >= 1.18 you should specify the pathType (determines how Ingress paths should be matched) + ## See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#better-path-matching-with-path-types + # pathType: ImplementationSpecific + + ## TLS configuration for Thanos Ingress + ## Secret must be manually created in the namespace + ## + tls: [] + # - secretName: thanos-gateway-tls + # hosts: + # - thanos-gateway.domain.com + # + + ## ExtraSecret can be used to store various data in an extra secret + ## (use it for example to store hashed basic auth credentials) + extraSecret: + ## if not set, name will be auto generated + # name: "" + annotations: {} + data: {} + # auth: | + # foo:$apr1$OFG3Xybp$ckL0FHDAkoXYIlH9.cysT0 + # someoneelse:$apr1$DMZX2Z4q$6SbQIfyuLQd.xmo/P0m2c. + + ingress: + enabled: false + + # For Kubernetes >= 1.18 you should specify the ingress-controller via the field ingressClassName + # See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#specifying-the-class-of-an-ingress + # ingressClassName: nginx + + annotations: {} + labels: {} + + ## Redirect ingress to an additional defined port on the service + # servicePort: 8081 + + ## Hostnames. + ## Must be provided if Ingress is enabled. + ## + # hosts: + # - prometheus.domain.com + hosts: [] + + ## Paths to use for ingress rules - one path should match the prometheusSpec.routePrefix + ## + paths: [] + # - / + + ## For Kubernetes >= 1.18 you should specify the pathType (determines how Ingress paths should be matched) + ## See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#better-path-matching-with-path-types + # pathType: ImplementationSpecific + + ## TLS configuration for Prometheus Ingress + ## Secret must be manually created in the namespace + ## + tls: [] + # - secretName: prometheus-general-tls + # hosts: + # - prometheus.example.com + + ## Configuration for creating an Ingress that will map to each Prometheus replica service + ## prometheus.servicePerReplica must be enabled + ## + ingressPerReplica: + enabled: false + + # For Kubernetes >= 1.18 you should specify the ingress-controller via the field ingressClassName + # See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#specifying-the-class-of-an-ingress + # ingressClassName: nginx + + annotations: {} + labels: {} + + ## Final form of the hostname for each per replica ingress is + ## {{ ingressPerReplica.hostPrefix }}-{{ $replicaNumber }}.{{ ingressPerReplica.hostDomain }} + ## + ## Prefix for the per replica ingress that will have `-$replicaNumber` + ## appended to the end + hostPrefix: "" + ## Domain that will be used for the per replica ingress + hostDomain: "" + + ## Paths to use for ingress rules + ## + paths: [] + # - / + + ## For Kubernetes >= 1.18 you should specify the pathType (determines how Ingress paths should be matched) + ## See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#better-path-matching-with-path-types + # pathType: ImplementationSpecific + + ## Secret name containing the TLS certificate for Prometheus per replica ingress + ## Secret must be manually created in the namespace + tlsSecretName: "" + + ## Separated secret for each per replica Ingress. Can be used together with cert-manager + ## + tlsSecretPerReplica: + enabled: false + ## Final form of the secret for each per replica ingress is + ## {{ tlsSecretPerReplica.prefix }}-{{ $replicaNumber }} + ## + prefix: "prometheus" + + ## Configure additional options for default pod security policy for Prometheus + ## ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/ + podSecurityPolicy: + allowedCapabilities: [] + allowedHostPaths: [] + volumes: [] + + serviceMonitor: + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + selfMonitor: true + + ## scheme: HTTP scheme to use for scraping. Can be used with `tlsConfig` for example if using istio mTLS. + scheme: "" + + ## tlsConfig: TLS configuration to use when scraping the endpoint. For example if using istio mTLS. + ## Of type: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#tlsconfig + tlsConfig: {} + + bearerTokenFile: + + ## Metric relabel configs to apply to samples before ingestion. + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + # relabel configs to apply to samples before ingestion. + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Settings affecting prometheusSpec + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#prometheusspec + ## + prometheusSpec: + ## If true, pass --storage.tsdb.max-block-duration=2h to prometheus. This is already done if using Thanos + ## + disableCompaction: false + ## APIServerConfig + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#apiserverconfig + ## + apiserverConfig: {} + + ## Interval between consecutive scrapes. + ## Defaults to 30s. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/release-0.44/pkg/prometheus/promcfg.go#L180-L183 + ## + scrapeInterval: "" + + ## Number of seconds to wait for target to respond before erroring + ## + scrapeTimeout: "" + + ## Interval between consecutive evaluations. + ## + evaluationInterval: "" + + ## ListenLocal makes the Prometheus server listen on loopback, so that it does not bind against the Pod IP. + ## + listenLocal: false + + ## EnableAdminAPI enables Prometheus the administrative HTTP API which includes functionality such as deleting time series. + ## This is disabled by default. + ## ref: https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-admin-apis + ## + enableAdminAPI: false + + ## WebTLSConfig defines the TLS parameters for HTTPS + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#webtlsconfig + web: {} + + ## Exemplars related settings that are runtime reloadable. + ## It requires to enable the exemplar storage feature to be effective. + exemplars: "" + ## Maximum number of exemplars stored in memory for all series. + ## If not set, Prometheus uses its default value. + ## A value of zero or less than zero disables the storage. + # maxSize: 100000 + + # EnableFeatures API enables access to Prometheus disabled features. + # ref: https://prometheus.io/docs/prometheus/latest/disabled_features/ + enableFeatures: [] + # - exemplar-storage + + ## Image of Prometheus. + ## + image: + repository: rancher/mirrored-prometheus-prometheus + tag: v2.38.0 + sha: "" + + ## Tolerations for use with node taints + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + + ## If specified, the pod's topology spread constraints. + ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ + ## + topologySpreadConstraints: [] + # - maxSkew: 1 + # topologyKey: topology.kubernetes.io/zone + # whenUnsatisfiable: DoNotSchedule + # labelSelector: + # matchLabels: + # app: prometheus + + ## Alertmanagers to which alerts will be sent + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#alertmanagerendpoints + ## + ## Default configuration will connect to the alertmanager deployed as part of this release + ## + alertingEndpoints: [] + # - name: "" + # namespace: "" + # port: http + # scheme: http + # pathPrefix: "" + # tlsConfig: {} + # bearerTokenFile: "" + # apiVersion: v2 + + ## External labels to add to any time series or alerts when communicating with external systems + ## + externalLabels: {} + + ## enable --web.enable-remote-write-receiver flag on prometheus-server + ## + enableRemoteWriteReceiver: false + + ## Name of the external label used to denote replica name + ## + replicaExternalLabelName: "" + + ## If true, the Operator won't add the external label used to denote replica name + ## + replicaExternalLabelNameClear: false + + ## Name of the external label used to denote Prometheus instance name + ## + prometheusExternalLabelName: "" + + ## If true, the Operator won't add the external label used to denote Prometheus instance name + ## + prometheusExternalLabelNameClear: false + + ## External URL at which Prometheus will be reachable. + ## + externalUrl: "" + + ## Define which Nodes the Pods are scheduled on. + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + ## Secrets is a list of Secrets in the same namespace as the Prometheus object, which shall be mounted into the Prometheus Pods. + ## The Secrets are mounted into /etc/prometheus/secrets/. Secrets changes after initial creation of a Prometheus object are not + ## reflected in the running Pods. To change the secrets mounted into the Prometheus Pods, the object must be deleted and recreated + ## with the new list of secrets. + ## + secrets: [] + + ## ConfigMaps is a list of ConfigMaps in the same namespace as the Prometheus object, which shall be mounted into the Prometheus Pods. + ## The ConfigMaps are mounted into /etc/prometheus/configmaps/. + ## + configMaps: [] + + ## QuerySpec defines the query command line flags when starting Prometheus. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#queryspec + ## + query: {} + + ## Namespaces to be selected for PrometheusRules discovery. + ## If nil, select own namespace. Namespaces to be selected for ServiceMonitor discovery. + ## See https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#namespaceselector for usage + ## + ruleNamespaceSelector: {} + + ## If true, a nil or {} value for prometheus.prometheusSpec.ruleSelector will cause the + ## prometheus resource to be created with selectors based on values in the helm deployment, + ## which will also match the PrometheusRule resources created + ## + ruleSelectorNilUsesHelmValues: false + + ## PrometheusRules to be selected for target discovery. + ## If {}, select all PrometheusRules + ## + ruleSelector: {} + ## Example which select all PrometheusRules resources + ## with label "prometheus" with values any of "example-rules" or "example-rules-2" + # ruleSelector: + # matchExpressions: + # - key: prometheus + # operator: In + # values: + # - example-rules + # - example-rules-2 + # + ## Example which select all PrometheusRules resources with label "role" set to "example-rules" + # ruleSelector: + # matchLabels: + # role: example-rules + + ## If true, a nil or {} value for prometheus.prometheusSpec.serviceMonitorSelector will cause the + ## prometheus resource to be created with selectors based on values in the helm deployment, + ## which will also match the servicemonitors created + ## + serviceMonitorSelectorNilUsesHelmValues: false + + ## ServiceMonitors to be selected for target discovery. + ## If {}, select all ServiceMonitors + ## + serviceMonitorSelector: {} + ## Example which selects ServiceMonitors with label "prometheus" set to "somelabel" + # serviceMonitorSelector: + # matchLabels: + # prometheus: somelabel + + ## Namespaces to be selected for ServiceMonitor discovery. + ## + serviceMonitorNamespaceSelector: {} + ## Example which selects ServiceMonitors in namespaces with label "prometheus" set to "somelabel" + # serviceMonitorNamespaceSelector: + # matchLabels: + # prometheus: somelabel + + ## If true, a nil or {} value for prometheus.prometheusSpec.podMonitorSelector will cause the + ## prometheus resource to be created with selectors based on values in the helm deployment, + ## which will also match the podmonitors created + ## + podMonitorSelectorNilUsesHelmValues: false + + ## PodMonitors to be selected for target discovery. + ## If {}, select all PodMonitors + ## + podMonitorSelector: {} + ## Example which selects PodMonitors with label "prometheus" set to "somelabel" + # podMonitorSelector: + # matchLabels: + # prometheus: somelabel + + ## Namespaces to be selected for PodMonitor discovery. + ## See https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#namespaceselector for usage + ## + podMonitorNamespaceSelector: {} + + ## If true, a nil or {} value for prometheus.prometheusSpec.probeSelector will cause the + ## prometheus resource to be created with selectors based on values in the helm deployment, + ## which will also match the probes created + ## + probeSelectorNilUsesHelmValues: true + + ## Probes to be selected for target discovery. + ## If {}, select all Probes + ## + probeSelector: {} + ## Example which selects Probes with label "prometheus" set to "somelabel" + # probeSelector: + # matchLabels: + # prometheus: somelabel + + ## Namespaces to be selected for Probe discovery. + ## See https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#namespaceselector for usage + ## + probeNamespaceSelector: {} + + ## How long to retain metrics + ## + retention: 10d + + ## Maximum size of metrics + ## + retentionSize: "" + + ## Enable compression of the write-ahead log using Snappy. + ## + walCompression: true + + ## If true, the Operator won't process any Prometheus configuration changes + ## + paused: false + + ## Number of replicas of each shard to deploy for a Prometheus deployment. + ## Number of replicas multiplied by shards is the total number of Pods created. + ## + replicas: 1 + + ## EXPERIMENTAL: Number of shards to distribute targets onto. + ## Number of replicas multiplied by shards is the total number of Pods created. + ## Note that scaling down shards will not reshard data onto remaining instances, it must be manually moved. + ## Increasing shards will not reshard data either but it will continue to be available from the same instances. + ## To query globally use Thanos sidecar and Thanos querier or remote write data to a central location. + ## Sharding is done on the content of the `__address__` target meta-label. + ## + shards: 1 + + ## Log level for Prometheus be configured in + ## + logLevel: info + + ## Log format for Prometheus be configured in + ## + logFormat: logfmt + + ## Prefix used to register routes, overriding externalUrl route. + ## Useful for proxies that rewrite URLs. + ## + routePrefix: / + + ## Standard object's metadata. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#metadata + ## Metadata Labels and Annotations gets propagated to the prometheus pods. + ## + podMetadata: {} + # labels: + # app: prometheus + # k8s-app: prometheus + + ## Pod anti-affinity can prevent the scheduler from placing Prometheus replicas on the same node. + ## The default value "soft" means that the scheduler should *prefer* to not schedule two replica pods onto the same node but no guarantee is provided. + ## The value "hard" means that the scheduler is *required* to not schedule two replica pods onto the same node. + ## The value "" will disable pod anti-affinity so that no anti-affinity rules will be configured. + podAntiAffinity: "" + + ## If anti-affinity is enabled sets the topologyKey to use for anti-affinity. + ## This can be changed to, for example, failure-domain.beta.kubernetes.io/zone + ## + podAntiAffinityTopologyKey: kubernetes.io/hostname + + ## Assign custom affinity rules to the prometheus instance + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + ## + affinity: {} + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: kubernetes.io/e2e-az-name + # operator: In + # values: + # - e2e-az1 + # - e2e-az2 + + ## The remote_read spec configuration for Prometheus. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#remotereadspec + remoteRead: [] + # - url: http://remote1/read + ## additionalRemoteRead is appended to remoteRead + additionalRemoteRead: [] + + ## The remote_write spec configuration for Prometheus. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#remotewritespec + remoteWrite: [] + # - url: http://remote1/push + ## additionalRemoteWrite is appended to remoteWrite + additionalRemoteWrite: [] + + ## Enable/Disable Grafana dashboards provisioning for prometheus remote write feature + remoteWriteDashboards: false + + ## Resource limits & requests + ## + resources: + limits: + memory: 3000Mi + cpu: 1000m + requests: + memory: 750Mi + cpu: 750m + + ## Prometheus StorageSpec for persistent data + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/storage.md + ## + storageSpec: {} + ## Using PersistentVolumeClaim + ## + # volumeClaimTemplate: + # spec: + # storageClassName: gluster + # accessModes: ["ReadWriteOnce"] + # resources: + # requests: + # storage: 50Gi + # selector: {} + + ## Using tmpfs volume + ## + # emptyDir: + # medium: Memory + + # Additional volumes on the output StatefulSet definition. + volumes: + - name: nginx-home + emptyDir: {} + - name: prometheus-nginx + configMap: + name: prometheus-nginx-proxy-config + defaultMode: 438 + + # Additional VolumeMounts on the output StatefulSet definition. + volumeMounts: [] + + ## AdditionalScrapeConfigs allows specifying additional Prometheus scrape configurations. Scrape configurations + ## are appended to the configurations generated by the Prometheus Operator. Job configurations must have the form + ## as specified in the official Prometheus documentation: + ## https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config. As scrape configs are + ## appended, the user is responsible to make sure it is valid. Note that using this feature may expose the possibility + ## to break upgrades of Prometheus. It is advised to review Prometheus release notes to ensure that no incompatible + ## scrape configs are going to break Prometheus after the upgrade. + ## AdditionalScrapeConfigs can be defined as a list or as a templated string. + ## + ## The scrape configuration example below will find master nodes, provided they have the name .*mst.*, relabel the + ## port to 2379 and allow etcd scraping provided it is running on all Kubernetes master nodes + ## + additionalScrapeConfigs: [] + # - job_name: kube-etcd + # kubernetes_sd_configs: + # - role: node + # scheme: https + # tls_config: + # ca_file: /etc/prometheus/secrets/etcd-client-cert/etcd-ca + # cert_file: /etc/prometheus/secrets/etcd-client-cert/etcd-client + # key_file: /etc/prometheus/secrets/etcd-client-cert/etcd-client-key + # relabel_configs: + # - action: labelmap + # regex: __meta_kubernetes_node_label_(.+) + # - source_labels: [__address__] + # action: replace + # targetLabel: __address__ + # regex: ([^:;]+):(\d+) + # replacement: ${1}:2379 + # - source_labels: [__meta_kubernetes_node_name] + # action: keep + # regex: .*mst.* + # - source_labels: [__meta_kubernetes_node_name] + # action: replace + # targetLabel: node + # regex: (.*) + # replacement: ${1} + # metric_relabel_configs: + # - regex: (kubernetes_io_hostname|failure_domain_beta_kubernetes_io_region|beta_kubernetes_io_os|beta_kubernetes_io_arch|beta_kubernetes_io_instance_type|failure_domain_beta_kubernetes_io_zone) + # action: labeldrop + # + ## If scrape config contains a repetitive section, you may want to use a template. + ## In the following example, you can see how to define `gce_sd_configs` for multiple zones + # additionalScrapeConfigs: | + # - job_name: "node-exporter" + # gce_sd_configs: + # {{range $zone := .Values.gcp_zones}} + # - project: "project1" + # zone: "{{$zone}}" + # port: 9100 + # {{end}} + # relabel_configs: + # ... + + + ## If additional scrape configurations are already deployed in a single secret file you can use this section. + ## Expected values are the secret name and key + ## Cannot be used with additionalScrapeConfigs + additionalScrapeConfigsSecret: {} + # enabled: false + # name: + # key: + + ## additionalPrometheusSecretsAnnotations allows to add annotations to the kubernetes secret. This can be useful + ## when deploying via spinnaker to disable versioning on the secret, strategy.spinnaker.io/versioned: 'false' + additionalPrometheusSecretsAnnotations: {} + + ## AdditionalAlertManagerConfigs allows for manual configuration of alertmanager jobs in the form as specified + ## in the official Prometheus documentation https://prometheus.io/docs/prometheus/latest/configuration/configuration/#. + ## AlertManager configurations specified are appended to the configurations generated by the Prometheus Operator. + ## As AlertManager configs are appended, the user is responsible to make sure it is valid. Note that using this + ## feature may expose the possibility to break upgrades of Prometheus. It is advised to review Prometheus release + ## notes to ensure that no incompatible AlertManager configs are going to break Prometheus after the upgrade. + ## + additionalAlertManagerConfigs: [] + # - consul_sd_configs: + # - server: consul.dev.test:8500 + # scheme: http + # datacenter: dev + # tag_separator: ',' + # services: + # - metrics-prometheus-alertmanager + + ## If additional alertmanager configurations are already deployed in a single secret, or you want to manage + ## them separately from the helm deployment, you can use this section. + ## Expected values are the secret name and key + ## Cannot be used with additionalAlertManagerConfigs + additionalAlertManagerConfigsSecret: {} + # name: + # key: + # optional: false + + ## AdditionalAlertRelabelConfigs allows specifying Prometheus alert relabel configurations. Alert relabel configurations specified are appended + ## to the configurations generated by the Prometheus Operator. Alert relabel configurations specified must have the form as specified in the + ## official Prometheus documentation: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#alert_relabel_configs. + ## As alert relabel configs are appended, the user is responsible to make sure it is valid. Note that using this feature may expose the + ## possibility to break upgrades of Prometheus. It is advised to review Prometheus release notes to ensure that no incompatible alert relabel + ## configs are going to break Prometheus after the upgrade. + ## + additionalAlertRelabelConfigs: [] + # - separator: ; + # regex: prometheus_replica + # replacement: $1 + # action: labeldrop + + ## If additional alert relabel configurations are already deployed in a single secret, or you want to manage + ## them separately from the helm deployment, you can use this section. + ## Expected values are the secret name and key + ## Cannot be used with additionalAlertRelabelConfigs + additionalAlertRelabelConfigsSecret: {} + # name: + # key: + + ## SecurityContext holds pod-level security attributes and common container settings. + ## This defaults to non root user with uid 1000 and gid 2000. + ## https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md + ## + securityContext: + runAsGroup: 2000 + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 2000 + + ## Priority class assigned to the Pods + ## + priorityClassName: "" + + ## Thanos configuration allows configuring various aspects of a Prometheus server in a Thanos environment. + ## This section is experimental, it may change significantly without deprecation notice in any release. + ## This is experimental and may change significantly without backward compatibility in any release. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#thanosspec + ## + thanos: {} + # secretProviderClass: + # provider: gcp + # parameters: + # secrets: | + # - resourceName: "projects/$PROJECT_ID/secrets/testsecret/versions/latest" + # fileName: "objstore.yaml" + # objectStorageConfigFile: /var/secrets/object-store.yaml + + proxy: + image: + repository: rancher/mirrored-library-nginx + tag: 1.24.0-alpine + + ## Containers allows injecting additional containers. This is meant to allow adding an authentication proxy to a Prometheus pod. + ## if using proxy extraContainer update targetPort with proxy container port + containers: | + - name: prometheus-proxy + args: + - nginx + - -g + - daemon off; + - -c + - /nginx/nginx.conf + image: "{{ template "system_default_registry" . }}{{ .Values.prometheus.prometheusSpec.proxy.image.repository }}:{{ .Values.prometheus.prometheusSpec.proxy.image.tag }}" + ports: + - containerPort: 8081 + name: nginx-http + protocol: TCP + volumeMounts: + - mountPath: /nginx + name: prometheus-nginx + - mountPath: /var/cache/nginx + name: nginx-home + securityContext: + runAsUser: 101 + runAsGroup: 101 + + ## InitContainers allows injecting additional initContainers. This is meant to allow doing some changes + ## (permissions, dir tree) on mounted volumes before starting prometheus + initContainers: [] + + ## PortName to use for Prometheus. + ## + portName: "http-web" + + ## ArbitraryFSAccessThroughSMs configures whether configuration based on a service monitor can access arbitrary files + ## on the file system of the Prometheus container e.g. bearer token files. + arbitraryFSAccessThroughSMs: false + + ## OverrideHonorLabels if set to true overrides all user configured honor_labels. If HonorLabels is set in ServiceMonitor + ## or PodMonitor to true, this overrides honor_labels to false. + overrideHonorLabels: false + + ## OverrideHonorTimestamps allows to globally enforce honoring timestamps in all scrape configs. + overrideHonorTimestamps: false + + ## IgnoreNamespaceSelectors if set to true will ignore NamespaceSelector settings from the podmonitor and servicemonitor + ## configs, and they will only discover endpoints within their current namespace. Defaults to false. + ignoreNamespaceSelectors: false + + ## EnforcedNamespaceLabel enforces adding a namespace label of origin for each alert and metric that is user created. + ## The label value will always be the namespace of the object that is being created. + ## Disabled by default + enforcedNamespaceLabel: "" + + ## PrometheusRulesExcludedFromEnforce - list of prometheus rules to be excluded from enforcing of adding namespace labels. + ## Works only if enforcedNamespaceLabel set to true. Make sure both ruleNamespace and ruleName are set for each pair + ## Deprecated, use `excludedFromEnforcement` instead + prometheusRulesExcludedFromEnforce: [] + + ## ExcludedFromEnforcement - list of object references to PodMonitor, ServiceMonitor, Probe and PrometheusRule objects + ## to be excluded from enforcing a namespace label of origin. + ## Works only if enforcedNamespaceLabel set to true. + ## See https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#objectreference + excludedFromEnforcement: [] + + ## QueryLogFile specifies the file to which PromQL queries are logged. Note that this location must be writable, + ## and can be persisted using an attached volume. Alternatively, the location can be set to a stdout location such + ## as /dev/stdout to log querie information to the default Prometheus log stream. This is only available in versions + ## of Prometheus >= 2.16.0. For more details, see the Prometheus docs (https://prometheus.io/docs/guides/query-log/) + queryLogFile: false + + ## EnforcedSampleLimit defines global limit on number of scraped samples that will be accepted. This overrides any SampleLimit + ## set per ServiceMonitor or/and PodMonitor. It is meant to be used by admins to enforce the SampleLimit to keep overall + ## number of samples/series under the desired limit. Note that if SampleLimit is lower that value will be taken instead. + enforcedSampleLimit: false + + ## EnforcedTargetLimit defines a global limit on the number of scraped targets. This overrides any TargetLimit set + ## per ServiceMonitor or/and PodMonitor. It is meant to be used by admins to enforce the TargetLimit to keep the overall + ## number of targets under the desired limit. Note that if TargetLimit is lower, that value will be taken instead, except + ## if either value is zero, in which case the non-zero value will be used. If both values are zero, no limit is enforced. + enforcedTargetLimit: false + + + ## Per-scrape limit on number of labels that will be accepted for a sample. If more than this number of labels are present + ## post metric-relabeling, the entire scrape will be treated as failed. 0 means no limit. Only valid in Prometheus versions + ## 2.27.0 and newer. + enforcedLabelLimit: false + + ## Per-scrape limit on length of labels name that will be accepted for a sample. If a label name is longer than this number + ## post metric-relabeling, the entire scrape will be treated as failed. 0 means no limit. Only valid in Prometheus versions + ## 2.27.0 and newer. + enforcedLabelNameLengthLimit: false + + ## Per-scrape limit on length of labels value that will be accepted for a sample. If a label value is longer than this + ## number post metric-relabeling, the entire scrape will be treated as failed. 0 means no limit. Only valid in Prometheus + ## versions 2.27.0 and newer. + enforcedLabelValueLengthLimit: false + + ## AllowOverlappingBlocks enables vertical compaction and vertical query merge in Prometheus. This is still experimental + ## in Prometheus so it may change in any upcoming release. + allowOverlappingBlocks: false + + ## Minimum number of seconds for which a newly created pod should be ready without any of its container crashing for it to + ## be considered available. Defaults to 0 (pod will be considered available as soon as it is ready). + minReadySeconds: 0 + + additionalRulesForClusterRole: [] + # - apiGroups: [ "" ] + # resources: + # - nodes/proxy + # verbs: [ "get", "list", "watch" ] + + additionalServiceMonitors: [] + ## Name of the ServiceMonitor to create + ## + # - name: "" + + ## Additional labels to set used for the ServiceMonitorSelector. Together with standard labels from + ## the chart + ## + # additionalLabels: {} + + ## Service label for use in assembling a job name of the form

    !w_)Dl=AEq%aCCE00{Wkz ze^Y%dr2qYcgM;2-*8gMwU>pBoBhNEXNIYAucj};~wrZBb0!3kq95ISK5HhuQcICBs z5E$w@woJRk5gv~b<7^jv(lUor&U3Os&sB9=ZmR96bfxSrWH7aZk+i{=dueKO?SwUnEi&co&XIt2G=BJrXh!pqAvULNt*F3u{(9GMmLR zBVHb@(vMxewwOr&s*ZMSOj}jPIHJc^>q?-?r(yDOn4*Yy*JV?+@1&=e_~%NL+S^f5 z1DVaF^_ab>^vq~Vq(*zDr&^%O2kWx*1)|)7rb!K zCCc2)u2Wm-qdF1J$PHq^_y*cMFFmh7#)yZddMCLDQ3$RKN&{IzS&OS6q|PiPa8yo0 zQnG3eWLzGjl6U+vrAkUEgv8JsjkuHn63yZahOxA_OI7nH&LkSGnHnJQsPE)-S5${} z3nM92RbHZ6M114exI`sNMrgV6dKa!%OPw@pS9D`lFHuoN{*zH~%W6sN7Mny~Nl1I7 zlKr9YG}q)w{}5Y3?!%k?T~GrvjfHGAX6lmN))@(zvO{7@(fmnv_{I@&i9FG>l3gZY zxEdkGuObRPjy=Tolt1*9F>b!{kxW~QNyv6Gn^;X&^%hp;s)?ArK+>=mGpXLtag3F% zc!+Mn4E{-|Q9!j?r}~Afm(qqUFXiRiO}{(n?t{)>!OfuC>-Kg44t=SgW{Hm-l~yn? z1UM9w4sv;oGor{=MM=L!@l&k`fbu-Vx(fcED4Bu%AG&t})`MR6P7_rFlb#A@iJF~6 z)Z_%*^ty*Xbo;xuC@NkLy8ExXymm-dk-uJT$@K#H|F_W~I3sHTE!_%Ig{yl9{UR0*zQGc(H%3Kw3f;pT8PZ9p7 z1-Z=+pA_7RB|kBcQ>ox-0^V_xA-mI+(~2Fr z*sseb%N1cexL8fRv@okJnPP_2JrlVNP0*ZAuJP(Rip#xGKt}3(?uf8^Fv8uL|IFP2 zJAhE|DfO(r&z@7}va<^!C5a++y~z_T9J@o72-Hj<0~aK z3h7Zxjk;||<{}wsF)x-Gf$_N8d)kuNt8VW}OJdEy|Fj@~SV?RRxF1RqTMOiel*F0< z{GlYV)nR{lNo;l4OC+(B108D)^h$7`3v`31GH*s%e?+4hurq5&-Oj0hgBhXPzYK?9 z|EPPn3w-60g1MzlF}9#rq4J))`ilyLqN8>aX?@QGD`p&fEU~EeiPV@W)78|0NyQ-0 zR>|s`kQmpZu6z?N`)dn z_7L!(kPcvg`MhL^f_WQf{#v6vjj*z@%xA|@rqCuR-4fGrsHfv-DdFU}R5&^6_Mf(J za{R=FlV;$5T97}iaIyy64<($e1@c1*CrtqUP{PUTus^(TvO4UQ!pVW=K!;0lprcX_ zbokVd$45_`12qHx(}MhAInWw#KNJUA3*?97KurMtP#kD=*dLw)tqyx72in&h=pf61 zjQMaG6FMwqLi^prCw^Ui_{6zTGw?qx$RCyqtpWE#aiO(9en>9V1mF+Fg;t0C;knT2 zuvc=Sf#yQ{1zbouVHe+|pPAqZH~ZaQx0fVIqeuqF(+0e0G!2;7=HTPvVfDQxZXyE6PzG_BF_ZWf|we0>;_zf_K{bF@f7sQ5OYqG6|t|upbrs zhSL~5aUOi|#CdQt@INicAC?EN0rx}k;I%-0NFLk-;19)vSBL%KdGPA6SMuPVW@!DK zfw%FVk-@cM9Pkbffo4P5%rAnonQ%1}1sEkE$Fm85)a@&aM;#sBxstO8@icp3U3v#K5W^mpz7mvmMG60Tw5*;{+{e^BVKcHkD2ij16zsOKtCS@r# zN8i_yf!NbY7djPBK@9b-?)_IdLcwbp`d<5dLf=;nv~`iTj_1ZnQRm8TYuALQkO3QH zTwJw8rr2m+VtjW?=rtg55>aVH?f{%X9PR=frfz`xs|*WQ+>nM=de0^o$1tD>`g0YA zxL`~&PEx__D49-94jWp*{EZo-!fG9MP0c#MJPG`h&b_m`+MsRlLvD42Aj#6nOQHpr z#8i}hy0H;~KM&yydpb(I^Z>TRS0_-g?t!&bh$|JHI`L$~q3z;!W75{&^t*$BvPkH6 z2m4milVILWj3RY_FA^p>%d{1$8L4dM-*-z#mgFo_kw4^ZKKSM1$Mef1dUU@xI6Pk1 zC5^){+v&dPG@Y`Vef{7pqP7Vcyvi-2c9~?kY|PjVq&PwzRB=qwgkTgU!M`1!=|SJg zWkQ`A`|X#PEyYFaf5Lj9oGW&5blM(eb5X>Lk)A*d9?f-E=}{cOAcV=fIT& zjeDDx&Bl{6pHx6-J9UTs(VS8oi;RIbP#GNO7KTLu=B)fAsLpgrtOV_>&ykA3CZp`g z#A*?g@egz)cby|a2AMG2L@%+-W7JhLis))6=XoIM!VRQYp-EZkw;HZf9feTOMn&lh z1O70{>DFzvPK`619r!idj*R1@Q*bH#f;t}F6#5_}0FB4U<0pwVjS^GdQc9BijH!Rt z2jZgybp2H!>gkA~kc)R5&*bWToFbr-rO!yS~xRVk?899L>6DnPf zIYkL=q%%0mp@hQM3JpPW+;?R+$0<>U_Tz$PanGZvya#I^VAj`@^5_SBJgod_T}Py#syInU`9BB8e@H+^giq5#OzH4v2!`6lJqNv3w5z9oE;}nj^ z*wbhC84CHHX|9$&5z!#}mQ%LYdAClzXtuLJ~o%C`3Zhgr_fwzCi z=-O#ksV^#Fl=!b0Vn_WDEo9VxZk;66Nxoj``V}^05*LD(Q%HS}_{iT$8cjZC8qNj_ zuN;nr9QuNBATJo-#7WL%b8OI*v?#fv2{9+hFAPLcad=LXU+l^9<2Xn&mdnHfiS9F^ zSlw90CV0uhm_b%rjV_ELroDQ*t}n6-dK7x-bpdDF%@-G#|GGe3+39$YdC*6%35}*O zRE#bSsV%c~#+M>q)_MoR9OaeIyDc5lE`WuywRpey-CL4h!VfO^xk$2iMC4rJF686i8 z*yN?RU#mC#n2x|F( z_VGYl=Lq|FpshNz_VsmJUtYIW9!v_d1uEULWPsh4whj{lLebfzpt`0E&g`!L&D#B-z9(RJvsi?48)He)=l%bH6Yy_e_IR4&GNS{&>m34^NGkGz9TuhjqieWF1I1#!J=%a-+PY5l}bA zOV)t)0eHz8&{pyiZ7I_~a(Ky6%1cTEElbxKHy)5l)~y{!Ispz5fOHb8+{sEUsPu}u zEt9tlGw^8?kO_zg<7HtfMS;CRZ_wZC9qje@3lcSSZaRZbPeq~3#(Fm9Fa^0KRa#AD zCWHJ;@{$9={}7}BM$|ddX(OHfp)+-d_1N%H!iJB(OE!G}QG*^Q#E%`;O|#*9i3Rki zfZQw_zTcQRj|bXH#-=S-`UehmKBz~X4-@Kq^j%Wt`;VyeI3a%Qux^?<-%FgFM+M|& zsq_8DlX*PQR#Ine#MIw+sPle3>U@w;=fllX=f^>u??2GYWFRt^pk8(uS~)Vo@YF&yWr)Rwa-pOuCh>>*UQuMfY2qj` z#adU}v)%IGLScpQc(FlOx1iamrJ-yrbu*UJRKGLC2of=pnVt@)eS4kSx?`tG|I%9?suMi89l+QJA49W#S=_DGhvg~pk2eK8_yHwOFH;4lC zh77>ecT+j@GNQ^uz|kxUR171?brwoi2ReJ(X)~TzmU*?n=cMg%9I7;kV1&e~$hZYq z#j;2oij7+ORrjuYXVzC+Uqi>mvnUU@7?<@PGY}#VF$U>eG)}0mQV?1MX4VBiVaoWf z-L|yXm1h14Kx8ZiD6OWNP6rA)9n;Hk_psYb_^&hGSNGrhC)tP2^ZOo$cm@L}HI_BE zrjA16Bv+}4IDr`$4!S~|vv;x9>>pbw0?kIULZBEt-395q2y{>x-lZcDQ>68;uYv5v zq5oP#byU(zYFn=Zbrff`+;yA-l5U4?{;bZJP*2*mrPxJLM{Oa)26cDRBZ}+&EBJwQ z5r82;loB0zfp}g_P1;cv8AatCwZZmaD5{STm=Pb3G4fp^kHZ@nVE;r?q^`*}bpqb0 z_sX9A1l;ty(ud|!7x?@ha&M{ve4NZ+^oemnp#M<+`P?ox%Wt2w^OM*9x(j4$Z|@?C zn2eTj^68Ikj~8lI#h_k>IN14mqB3GhHaeTd9F78%Bp8!Zr{-!Z=$VdVA}#V!=#e<& zh$4R%K)p&+%s8R*VUkYPgQ1wUSo^B^h$A|~A<|)94DyQBZwb8?ZAq6jo&25z!3cWS zLqj5Td8begGfbR38UG|iGR5$jO+#BPnJxEBhs-nKa8xM19~Cp#sq)2Y6erS^LdTvA zad9P+>OS+*_Bwoul)qm`2?rgVw71g*ABnta6v9M8+(!Y@(OQ@ieSE|r4)g^k3T68F zEG=SFa+`Pr)+Rt|eg8{_4xH=z$kUmx`hArS%0!pDv*P+i`y>$0)J?Qrp@uf4idkr-dzASob(b;)XELG6qY^Xe zd^Lhh=CM*@FJtjc?Mb@*Qlixt@qY3oF%n~7x<_Ogr+=*vrG#*lX zqcPgG%CmwB%CApq$9&8@8A0ou_)bw~?}wEB+2OAg$(0n~3mA@Jz~oNJFec56L>;}` zCFAq)c#No+cOMJ+Z4@h3bb}%IWbh)2*)*c$u5j1uLVG^C$?S8nk@h0!UBd}td)dZz zy5JHi5}*A9h8J&7-@Jd@o%t#%!b~g>4pl%MdCCgGt&ATR1ZHGfW;2qB9-mybm`#;h zNe4V^kh9Wjaq8EB*xqYzYKxp|CIb^73{_R*P=iZEi?YCO#%c_Q;Nov@(^zfxdMjWq zO@OvFfk*?r)$d!*@tXZca-Hh;`7WG@bHj%T4)0d)8mm@1p=f4_-rsW1U%gkvJ%5SL z2KPVKL23Wj`%I?+Tz!4_StpaZ@Fn!*af~er*q9AHxSK|Bbg*Sd4B~p9m5(7Fp;Ldh z+5M_Jtpe&^_iarW?q&bh?7P{Ww6IfkuzMc|97jQNIA@AsGI7FDf>=fgAkLaIQ>j3Z zV@5}*n$V&v9w#AbY$toa{U89kQ|@qz>hUeN!MmqO38Vb zQ7g)&6YK@KVJme^5hu8xS~__m2mc`Sx^qfN9P07?EhM)gc*j^Q)=-|=bp%rqf{5Z9 zfk!`Eq_x%hhd9ChOvog#NJYOKA%2UH zK3G@=(YmSs@BjIK#ca!bwR$C!Fqs-SwU{zx!Jo+EUGUCP9}0m@97csCq-zNIP6a0w zxF+zb=y;=amSarOtvta?t-2*SB~9xlILQq{;FrE-u%bfQ>P;P{72KlHl#uHb$JFT= z%0RzKcP^V{f(UV+dX9Y9x8&D$RcaHc;DFc!YNf^5PhCW*Jk&->rxgW?Q4xneJNdHSsY(cYsiDD#(o*v{Wvt2+UG%ty<+MQb$*% zy)*m?`)C#sE;^9Wa}rt*)6l5L%@UJPWvd>KC905^Cy}n@(OGJd1(4UQ+iLZ@;8dW5 zQGf$QHYmBlDxx=ZXCc&BA$fm#OjiSeCK9X|GUReOB)TT;scTLt$*6$n)L9;VA%vVh z&9#QSz1Fu-5=wN3IqS9tYVOiq4v4+4utV@Q;y`Iup=7uur~0l&IJw~hB4g%>!KAQs zTl-z`cNe+eFGOy4HC=VzZnBp#1!IT)2qzr1HL{e+20)T~|r#}&wS4VpSSWSqaz}{o&?Z+L0Y-w@1U@vPy(XGGK^uuO95Ie;K|{xl zPh>}js*0Uw?{9}V+~EJ8y*F)c8%Os?`?Wp=TIqAz_Gq`|txl${<0Rdw=_Edh({pFy z{wpvENf=WE2LQda{m#OjQp z6Z8LvN+B2xoB;I(iRfacVEsS?L{$)-j@gwcoA~?&;lVJP>6geweJl^W7{Y69^^~(j zY)j(WK&mZe1&s*F6}owiu8~9JQ4%psqL7Z5X+iGe>oE6G(~GMmCVM*_ZK8{!2>eUn zTnH@U-KiDe>y$B6liV-ke!H#opq$p)aaRNE&b5#YwiGfKGfYW;TMjg79hIQ=>kwL3 ze{6~WZ-YK@Y($frk}#+ZK=&?WWfg!l*jM&3WgL7JpqdZSD8N&M;sJJL9EnGm*x27q z-O7}^xOwT2OXX*hP&#?YT+r>gU@1lK#qFB|ti@J?GJ`_6wna&#qZlxl*v_DeVPp^2l@Bj)y4(p7=;RPq(?CqFs*(TZ8z3~xgexOCUCHt3IRfwnXFqt_ z!QO{7P#-xGZpg8bzv&dqUCe#dOu3HYV?wzYlVi$FtA(*6;9KhFVsIbJp_Y2>BEu{; zEnmmq))aWRKJc-PSaYD1TAgu7h4LKPFLY{rJ(a$rMs3tX>#SNVr%YBWus{VyuGLpg z?`1+RUAbmeDz8H2R!g{B^~h%vJ(SvbBu03+t*Pm=nmH>1PDVG&06IAt zPh5Z?y8|~+t_U_!>wwr32U|t~UMLyiaES;#`^EK&)|nazHY_Q05pSs2AdD+5Um`&T z!^1??8C`v>kK$=m&gB-Lk;ptG?X<-{b1$q!;1U0Toi7ZrUy(EVU58ppqit~+NuIzl z<)qEL?`0l|XowUVlW|pfvqQly`I~-s18wcM8+LC|3h;iNs9>=7K71$Mzv49$%#*z>=PcoS3~JUNKJuvjh>l zalcKyPLa=0UAyq>b*5wC!@!lC*Qq$bot+Paqn({&ZRm=JA8R*?+$|`$DZAoZy>8RH zy0}%Z+vw6C=)v|0=~x~4SlHKFVc+SnQUrE(aw@d56OXRQ8&9Ihot@ks+1UvOR93V~ zj>`HaJzl|pJFZmzD)-e&-X2BFA0cgn*QYGAcx(vx7LWXi3{(N>bc{z_HX<;MNti2# zBX;$kA-3p*=O`%fBWF(hJsR4F^$35eENo|D+pi)t5M`KgH@qa*;R(^T;EYD9 z+#fpQBJd;LNI4^_xAd_CxBG~KuQ8kUr&lMZMeIKk!ponCu5K#tNFDlo=*s{9*`$}y zYHn95od)>g##4uiMt-2{#YocB?une5bl0AWK2t9IYeaGX4xIfCF2&?U;FBuLZsdwh zEkj|Y@}#KNH5ufKmk}oLRkDTPSK&#Ll(+Srk8DH zAl6xgbZY={tFaxh|0`WZ8kNd!1AJ1&T0WJrW#43ift)U%P(M7@g9qofs72uSpP4)s zz}H@vU^s~9POc$#giapcPm`x~o!@x~{EHD6)V-HKf4#2~uX+rssw1j3Q1u`miYrY~ z>1I&r>cbyD7Y&X_nBknPV3ME6({nr2t8@Y)<=f&p|rO`>;^vM5wQ`|C+6r)MlCv3$PP==F9{BU#5ymc zVEt4`OC~brwE4IKWO>JOtm!wLI*_psaDLzf;W7N7q8t zKjb4}l)fq}jH1Exe6XmX`;&;VL@i81&y182W0-I5IJPy;#KmPWA&a=Uu+&tR7-fH`qd z=x>ujG*FuE0F1@Tp8QilEgi-Ddo7;iqkJoUh~=TrVn8PKA#qU+48_b;7)Omv0T9oA zZv>X)P3ljagT40;xzEjA|Ox5P-% z7WukOgmPunew6XQV^?3-D5SPEnUmouRWgKY`R3oFty zJa+hZFiKRmUmbD|t)Utl!Ku7bBWopc_PZLKV&fd7E$3}DRhsd*t){3)7zvg+dd(uE zOPO{qy;-4OL<3b7BqN)}^30aC*u`2RqN^6BJwgUSUtO{*#O<3r=5! zvABivz)9m0z_2u$NDwEFUyE&Sg+2^pnZjHk7Q8;N@yp2aTaH|YuaQ2Eo`?G}sTra~ zA_N&uq}XclTF_HR+RF3m`{AV2?pvym3JkzNpbNyP(7PyQk_#qC?0nor{GOK=zbBLF z1954sTvylzJ5Jmr6&?R9dm@n|C6Co6((-(R*)YlkiEY5u!bW_xx#!rrLYyU1WCbYt zDg?BUFZoz|L-9gvL0fEV1wE0v?UOw|e zpkri%cxqjo8$>2z$|rWmr{gs$l{)x9Y*Y`7r!^U|Q+Mxi{U zHnJeqivSDb^khthG*AYKbaEa>4n(~!35_;_9!n*Cq+?DS;B8*YL~_i1O&a`zcs5z_ zsXaK6VLIu-X@=<=+j?;PO>a*0ej@2q4t6ejccaEiWQM_RTy`;tNr2@bnd~2~6X~J? ztQgx7#myhFxGx!pE|!fj?j;N;<~M48CR-BiD}dTqu<-Es8>x;Qd?gFMLP*t1N;X&` zG>(aE2|&Rp6%3po%Ahq{2qE+@0^)?T-%quV`F9ps=7R{5$eU#mSF}{R7Srpd8Z!~B zO@(nnH+uWYpQ7_^%&t^;vD%jhI@yAbgAyj;dZUmX&Uu;zZXdQ>o6#(iW0awKXqRp={yHaS` zf)WCY9~^@Z#8rDx(Ip>dczf`hFQ48WgC_GY!z?7$?04Ipu6@|4clJA-`kvKo)er4X zx4zpyxP-gt2wHoW&7l8&p~UxClS$#l-WJ@i55xma?USUe`5s%8Fw*CZQ!?^=j+&w# ztT}zX`|SMAL&u<6RoibV7ZqA!>+J@#Kf2D{F_2dM3XrPZ!u&3=mpKOQUvLG0v9`5i z@LHNbDtTL=sA?6UXNHo?TkIfh1Cao9rXeMFHHmShpth=fKEXk%E+F$xOXt2oIDc52h1TYHi+d5l5T7f~_ zt{q!}bvA}Wd>e%GRe%%hKq|%^B~XR{#*$MSF&!&Yc=QfMYcIJ4g|ZH&Moj@XmR0HQ{v@>+sDo$`^0Bn>v^*W!nsJndTH zsD}9Go08(C#0g>cegZ8tqwZ26Ao=&|s?=`@J(|y`m9)?rx|3L#y3ww|9R4t!2D?(#K~U(jQt2zboCi4f*6! z+z2%(DKZ5 zrEX+19$5`Y+fgi2H+zyyP4ghiL zAYY?Jrb5_aF*Y{pV^vN5x1!hp3UQ7#Yf(-{>x5yN4BDWb$?+@{^! zjxc2{7?BYu)vhokNdPm$AZXxrKj`OBU zEhht@mP!Fo82}Pp>%b!zwi&s0os&9p?a&4t6K0B3bBR&cqQybY^}L44SuOCk+9@^F z^o?026S^R*ub$qz`_x7!YQ3zsOKcPdt1f?)OBBBintEg;xCfW1n-T@f_szte6EE9F z*Y)d8;~>kN6N6~!(sc>jYE@vNk0w}=uOY?6XZbS+K#LN_V*btS(hc~Xhm`0ZRZFQ= zPJhh~Oufz2!NDOTXJDy_u80YUG>vaKK=rr{rruPGUr5QIE$^|eIyrj>G)YRndJ+|r zGe~9yBWDB@%#IRfH9ygd7vPJGx76Hav6o!88=dZ7DwU9M8oQa>H5nKWJ|oMgnBM_u zMw7mlF%cscaV_L=)@1r;O^?|1;7eTv)HSzbp3dMLx-fY$*%ysU<%CJ=3X}2YJ3Arc z^q8jY?0_vol`l@Y?Sp_X>4VSoL@+?NYZ`|rKcuG?9!`LVs1N{vF2HVGMJk4r+WiFc znhX-v{CtY_tl=>)N|n?3rva4+?guGe5Ri9XX-f0jZ5ei*!q@?Km1lpH9?2NHAGK^ zeMN(Xcz`%y+_$cj>jkSwwhF&G))Kqz& zgvz0kUdKYBiA5H&FfF#Zo#H$VL+%QN5pvNfmfM7M!hoYl?4f)@)05mL$?=qk4bN|- z0>qMlKk>k44qaR3C8jh*U!A2PZEb(?z4m@%ztK`h9JCs3b?vCpYV0bv{s+Rbg`^i+ z2XP!%0-txK4!&uBbt9Gt$q=O0Fr-QplOTGELp@Y7kEtNA;7PTULp0~PFjo&am~nO7 zL)djLxoK1?f6L&lN?nCf+<9_E|E6)Lq<6&8iK?=9USo;pby&=Am~nw=vTQ}k#FZ=y z=Qp_~>p?34?zb!u%=wv)s8FOuDm+8<^+(HsEDV9;F+%aa9A|R0NJ?v_q81_|k|0+; z=5nRJgCw~KYUG}u@GY975L`u73EaYgQKuH>2Z$jHCFzo!*aUEj2Lly68fjBkQ+sec zLJjbBx3kyYAB%f~e|4OS{J{(PMYBiO4?Bm4hwX#T{`PY=(ys86v$^PF2eD>?C$K61 z-}o0J?i}wJYyWMx4)zb)N&9c7-D>R_`|o3X?kfOPdp+!WK4;Zq@U>C_;E$TR8qoG6 zuSbd0;>Nx7z*NP0Egw_P;QJt~7*sEP?C^JP^%%5k(bY&@RF6Tm9vL6%^=j-fg;%P_ zicmMYKE!s;RcsOWlsJUSi2lJP+-lW8r`@iBPPbbF?d^Ec>QDkuJ&BqF{#%*7S^PPn zSHyFn8m#Jj54CJ#lg;`m_H~Lzp<7SZocW_It&z9n#OvwF*o(L(dkS#$9LRW&B8Nb` z$9(CF?fXtorqgmb@_Gze#I@OduyOtI%eet4N3pJU1 zApCYwefYZDVt;H)qKK{Z4-@^ZMW#^aIGIUV9qp_Ck}`Af2#aCXZnbhGsT_silkAOz z4B3#}$RQa28RJN%e(biA@`i)Ip*u0OtLgTn9+UIpkN7pr9rRXi_cQfDcHW?pr3B+i z{`*HokE|-PF;5MY>cB<(on1W!uJ1VN3WeVAoDhd&FKA#D2Mivfue!T?yWOMx?%v^U zN8WX?dll8a>OmJdES$W-Xs%k&8lrPNLd55>H>!I=Vy~ceHIS$vEmw74CimvwVzIRX z1ajZ&)Hzu!6s~&m(g@xL4NVTq5q5(ct<+e|nFoAG5gQW6RtKw$d*VH$S4eiD z7#Hc)s5+q7J|iqy%VG5x?8XWcRoIVxzfINhPJGW4jfJE;t?0YU5xs^^dUzy9nC>Re z!|vc*4g7ZA_2i(4Dj%ET5nBcsVDU0m3{wsE0 zrjnO z#oZPta=A8c33ZMsV0D=|P2__g^l}BS2kKcpM2(_O zj#-zu=$UBh{pmFIVaI4{qp6Lio~)^bj)vER9=S$TZk~&EwYs4hQG0r{VB#)MdBs8vOkTw*l2pA>5oCxL)1bp_g%~vR``dj()dB* z6IhPQZ+E8Lowtq7H#*n$C|i9tTSt zb!*`0Nc`96j>i?BmAeFdf*gS@m$>!4@7>#5KoRqZ%TUaHVc^E|gBv!o@CZt6-+Erw zhTSyuIGoblV4w#u(8EBF$YX-}!sca2UrI^F2>$vRy!h|L+VH;a}p*52UdOVC=x1CZ?5_3G{4APY{? z1aBdB;*;q~>+ZtcICL;%r-7cmkH+pLq!}F|!&QGn?m&`Rw_AzpDDUxElxO5u&IwXf z*C~|)CixaIh`1W>;t+< zH_1T*BtcQEV$i_@_k^A22lNV2W-e+Dz#OGN#5Q^}^7viBy?-J~vgM87ZO-!}?B-mh zqWzz9W2a+_WBe+%(ciHx2hnZ<@i0PVPX^6N-Fl6fl`T%;(rmpH8F10sELM1ykp_0r zjjmZ|92S{IJ8Op$9Ats5Ja(5t8~dW%Zt{9|9#Q1lh$5*G`VNULJ`{NfX7s{D)k6RHkOpX+pTGJb!-5N zPI{|AV`5B;>z{|XpBe0-a(|~AQ5eS;#JHH)y(Ebqi?5)1(~^2Re~Av)&aZ!D zs(%M>$8EHt@lMgdz0^pT1Z$Z}h#O?^3Nn-#tV$7O9#@(ub2$sfi}!F*JzuS-mT_Nx z-VC|R%3LCYoM9@j|Awx^T36zQPQZK#tq`IsGJdLBrjxk9-H3SKiWf4roT|I{XL_Ow@9yvs2 zb!j%sz*(P=h46KPY+|X#!2*RA2(S*O6rln`n5t(WO_DFR-pI|g`_ z3wVL831+brwUFlr?7clH%5@mY<^f=j4zeuIUlpRt7ply53r9G`99uAzGou&DOiG2$ zgUMW1sRaW*E>B=_P{LYK)UJ& zF||5Js)8jnN*Xr^f&Pl1Emd^#j#?{X-C_|ZeMmrLPs%i&@jGYSgpF8_~z{No?>_wVbkUxQzNJsypY(@o4X zwrY`cj;NF$&g&bGB8KR7^scNSg+Ui{HEOaq3rrFqzK}Ccu{%hX=DUfEVx(W1h;DOoON5^ zH^`A|-92X+wbB4SvqdZr-sy6MXLDO0{X3HokBksALd?qQbva~pWMnm2R&UB6t9Kus zvxp|?t+$T_I9|=9(TvD)M0TDuiA*`8hS;`|yBrAr@yof9OmL%sJSi)F;i&g-AG#bnXsgu_y*YJWRWDeSLnFAd?cM92{P^;snzUt?fM38t zwII)FOn|BMH6?CNK`SsTInN9kSfRQ6bn=`XE8rs|;BpK2^`ZhkG6G(wfNjJ$c7slp zXH*vp`_1&io}SSq3fV8S6es4xE?0nAXt{#9`*z9VOs8NxF?l}j!urd6e%n;?@R^E? zzzzbRw0wX-=S-f&eompw`iRQh;xv`5=!IrB7~mSZEE)opa+XEvSwY>1lS1&&>QXg*~O&*l8|zSDv<~ z%)c)m8~@Lk(?ho&rl_aGyVTk7SN-jqmk!8j@?11tf~%XJT@ldDlCkl&4m+ZQQlfUqKG|^xBLWXKCa%wFss*) zL-08XE@DjKInYDK=c#TMtqHHj5C8wSkEo0S_wwo<>-cu!>HyuQR>ruZ3@Vb=>50%M zC!n6MHg@9WOz7j2wB`&u_orSDCK!w3NMj;S%kO>cT8?j{lUxe#Kr$bKRU=;5$agr- ziR}vH<>vh;Cazo|$yfGNK;J~t?jdBu8pzwpY|km;oaAK=2U3pge}g(!lXo+X5!dn` z)&oejHg>~j=v)<)ixi*esEVH@Dt4u_46gK!cBNzNzBn2?G5lmr5~0~KHoJ8__914x%NN<(tRg@@=QOXfpl?_Y52cFHk!<-dPa z^uSRG4z@x9h~DrZmNl&dDyy=_9*>h-6g$X*A?Gy2HOewcRcT>x*Pw} z8AX40V}G~fUv^vZJFy`1PVDctt)|EyK^w##F^)H13O;d zpe~8-`;*92pXwzgH%u?(F{Yef-hKGLK@)@_uz;2-hSs`zqPyDY5&mbboX|saxfVd6mo)h#L#{0^DW6;Xm1(X|ed!kpZLC0F zym@(!sH`h0hg3f6j!K=&moabI$o;G=qS?-r!{I1i=bY5{FugtN@-C#4FG_MRN^>tt zr7X@)kEdCtXdmLu$w^XArK!hTpUFy3opBcft@r6Q>9=sI;B2 z(ZBo;h&vjB2jtdUtHXZym%f>3RBT{KA}P0zP^ZiAimX6dOA?J!5#vnQ}Oed z2#VPTa&1N##sTcw_!`?jEHJ3#GYjN$3yfhGDCQR^Vi*{s7_%$`$tq`=1~PLamcySJ zO$8bJ#X0V8w%cxFoZpMSOQLurdo)nHfjl-viOubm)BfF->sB{Ixt|UP$c>@}a>r ztHm_g1LfC_4-Br^TwIf~J-#IrAZQKMl0+`0LD8-O%o!LV8$&lTHWqGW)^2lL04yCE zuw=ke)RkohO9ghHjJ>i3<^fY>vHd(BJoSFu`fA{nF&Pz z$N*J74c_(pIP5r8k6$$H+ zMOj7X=R9t@=M+Y&nbSVS7uo5voTxKx0GjUTz<;VUAPc)Uy~^ZvfrB;olMVL?_l_`la%7U-H#JQKSg@6{8fmX)*nW^)$#Z;Bp zE8<6GMHKd#)3Q39;6~=Wtn`Ul@iVhxy!F|0vnFzA=btz+`TVR&CupTcNX|(Tlh4tb zbdpx;EG+U_R)t$h5QIn!g( zsUF2gz7(A7G3IQK!qYv9h~*Qfe5B9$h@X0qIO-$$`_yqC6At`H9{G_v^doibNAlp0 zwdCe_&rky;4h^Yf&VML8|3lwsT8_^+qQUpN z{$6*9jo8GaKjdfp`5$}x2i;cc{Exl8R?D3K@faU5)pO7B7;|=K?g<`Kp5IY)dPlx} zd!~~+raiZ#_|%T{(MdVSazux61mHD&B!|?z%9PZR)~nQE8|kAq!o@aay^HF5f3LQ; z7J7IP^?tE?aCBgIGlzoa8u^M$d<6!+sm@`?$gxl5F>=X@+NckGhi5Rk}@xxbK&!jfp29hG0t?Tr5%3Fu^Q&v42vy>`S!wm zYhkcy%GjauO@;Y}!lCz2xZ9u(;Pqe&ODo%bPaY=Q>%IKXMOEn3p5^(W1x-ru`wez&a>L{u z32x-&i)w8|Z1nP?ifsp7R3EG&7+ZkSpgZGL%sK2j z-`-VZ?JBf&6&`W96`8j34x^l8){5p&zDX;`pjBwjDl%r} zo3io^Svh8`sESz=R?>bS9de)COHIykEVL69S&8&QS8bKTV^at3A_v&^v@so#S300304^fWO_T_|$a~%Naa0vit*8m^` zfF=VV=8v`{(s2m1yWry|0f%~G=1O$46~G;K$2c1G^of0h_Wtc7AN3qrTQT#N1#gOG z@?@k_pn7s!+Mc(i?IpLgJ#9;0&D+wsh1y=WRNGUF^|T}aR#Y3Dq#gX{1$ zW>*Fj0RxH*C^Dd^6i&E#K+(PdMFtd2=Y&&q1_E@0uSC*jf?TdX=IYGaj%p z^)xW+Il!!p2W%dkbzpFo!CAB7EEn+`LIqN?5WYq|M5zV^F$0f7zF#9gZ0vluiFJi& zW9NI{fv$_3#?E&OI#~R3NgF#;#e55J5>_GrC@$hX4@#XD)Wz;eD$t-hF1CLB&(Dn= z%piweW9NHg=l8P@^4~Lte4EHWxi6}P-9cmL=M~1Ee^ONhZ)6Qu4E)kE3i_!mVv_YJ zvy`L*b|n#JSs`%>T{gnJ6f=qV{F3-?nP<|u?ys4q(&@6Ml1+)XZXt#9$i~cq)DC7; zx#SegrPq`Q9WiQNg>mx21a)D8xUkxeQ76V}6ce0^al*nlk6@gvFr+F>wltss)T1vvpG~ptfhIUr ze0yN)1F;cP+9e?S%jM*&t7ym=@!xJ9@t^+8%l|hW^Pl=m;Q#CH?R8T8f4iOi1H=FK z7$3v`S62Q%W6fB;HN(*Q8Dqv%p!J(zE|3Fc%B^kb(r^reU!uY$3+GvERcp@)3&ei7 zLSjC{#Ah&z!7Tc-Y9_vUE$QK8oeW9}w8RR?_7-YPyRb3q;?rBKImN;;lzIA18xI#t zhqMeLd|rsK9C<*73C}>SO<}@Y3isVpwj&vqyX9a5V@fi>!2pNwyCJUw1~)u+^1AZT z)jeq_W0>iZ6*fTTIRY7jR}8P5!7C-lE9>Hwn?I8S+RM(Md0gtad0Q^Y;!2*ZV%XtK zriy_&fjAA+G2Ctj>y!uUY^{Ub?c*mv+^ubJjVw;+R%?43`~_^Sk>M>{p+&Q56*tx5 zsBD6KRz{$+h=^8^^1OTr<(SjviE=6s(27aj<`BkVJ{5hm)!wO_H1DD&RSyUP!t&q`e1yg9ed}iQrn7f-cpsYYNV5%#+Dy!8PvbL@|7}P{C?tQ=yQ;dlI6VX0c>cG8ZhNnr=6~DU?HK;I$M{UpOcB_Vw9hQ`txDEYz%KP~Q!5EHctdf7g2E_t2NMeGi~;^5cq?G+Szw-ujg+jF%p$IfEFN=o z$!E?L*iQ7QM;PzpBqR+kz9laIjQ^BHc3b}yyH8Q}H}YEHPh|7oLvI{mMF3~xHa-yF zC46X}q;8P#2V?qgGjz4l!3S#QmC7D>Z8p1+JIj+ zY%YrLClWW`LW|2zZ0B2vQnF_pd2%9*k$l2-RNe2^z)`E3RmR2igjzQ<><%2nIE=02 za@NMj`(b>%%e&*pI}z2dzRVx$t%7~E+|fjuG8_pTu$_)5D)87#YiPWAMmR#3RPU zu_+)WlZrGeN;xnnB`D0G6oXRcLn&KZ%b?F07fii8TAl1W1QyC~z>H)&GdEx!5}v#U ztOa_F5U1G61{^&CBpbukV*uG?K$dgMAhH!AGMBh$6>zK&ZLJQEr9H5d9I!{@;8+HX zwc7@b1%<5xjRA|$n2N~5WV;5Fl^&A`ys;#Y(~oJF(}r4iikwU=K0{^#4_yNu40tf$ zVVU4zZ!z%DG2kI6%zy_29u@!(SN2#X zmlsvb^DnA3-^E;fAynXsF_}!I!wbozDFYY`U??A8koLr9f*8^`~3}!VGLM zuwiD{u$-9AdhR)ltB!4*D>PJ9b(OKQ8v=+_fT%H|7&K(i5GV;6>MR%=YVwK0!VDTR zXlPAnXgwJ&n*=+akrf6y7_!phA%#8Y#ogQSs+arOubdc)2 z;H_i8n*ncgfwxu*?10VV6MNz)P)lW%LP)e{DnZZVU|kgdV~bJgA*Btd5qKXW75BBN$>W^SG6=kl&?H^4jjj!Vdp4TuZ)5CPv~oFX^^U$6X2RFC~mZXPDR&?2Le z?_z$}93p2_mn~#XP7!L3Aa*6bBJBlc2Tof^TW3^L?6Gp1A4I4$}(1idx#2qkfBLD$I9FTr^%MK?9uzU)6^v-lAOx@q9-)#erl7AtK%L>0G z6pRQ(0J}=FVB(7YACjA#VFYa4?<2}UpOUqRr3O3)L59FJMmMrsPJcc5@XMRuKK{}e z*`g!Gwd}#TS`acz6WIW(0roXGC|A8Rx8})5wgE*RVWQN#;{Jnj7K$$91XyxG@+r#c z&unwP1`pZ9id9J0Le@y!39EJ8Y1}rE+uWA5spep<5u{f(xpA|AF4sbk;KeRRu;YPh zc5#=%gKA+nCz3z!6Umj`Faah?dP;fFE<8fLxKt*+C0_7;yx{m-I(zXiv7q?_J6_*( zU4r=(b!_fg>X($v45~ z+??dMNCl>@VvD%ROqW&-blUA2=yba^(B95di6>EWz<+}i2T{)QHI~N7MG7k?Rd1&A z?sw%DZQ9fBQ;x}~zW4WmdGX;v?$LG+jt=Z@7L1kOgE^OlM7b(ki3_v2Di2bd=}9>Z z*|Yhla8yQClJ$hZWwIt$?rVH3$6_aWeWM+V%pYyZ*m|K%doJQPgkJS5M-1lPfO=Ms zA`9VbWGk133a7vS7rA_SQI#87@xwXd>EoGclU!<|GT@xTeji)loeQ?&OVIY>zP8Do zz{FoA>41+upRNbW62G z7^R}uqS2u+ICyLnm`;Y62L(=uRWm5ipuhkCN$fMawyd6gsX&g3WWlZiK1%{KuU~=B z_)j!DhFd3s(Umr+Ww8+SB!>@Jm(n-j#DJ6F{`!qQnYi?M^!%RD^G454r{}djgI*)r zV;oYBY;DsB?_f#wEC7Le&bw^PbkfU<>Ls>uPsOw3q|m8kz4+UP(6yb_ zylq}IB3}eP%Tn1&>`L=>_iqR?^l05Z^V{a_Zuue*@AS>GWSv+XU!!2W7&y3~5i5z9 zN{7D;PBb_%xW9g!n05*t%mEX-1|}MqIIB}o!wHWPk1*)4o<&9@%tIivI>*hV3s^T4PB1EUO#nhc|cm~k?oaMb&^4_%HOwAE@H9o0b6 zq9lorZlKfavpd)7**=vCPa!wIQce>AMh!R_Zqyl^xhKB?`TnIttXX>D5?WWtwY8OW zO98pe^h+O)G6z@Vn z>e2*4^?|Jq#71m8mx&?D)1S!4FmwL@Oqu`cGa>)KyW8GR<^Q+4-ImG!e~gdG|6gVP zzsv@jGymVD^#?ahTEE%0mwns5(zO1iCGwXzbANfc`X%3Uo8)|xoS&}b)8BX3)`oRT zm2+=$-wjmKx1Kdrnwa~(V79wSZ-2Ua-=w$CSx=+hH`tb8veu1;ei~WpE6!72N@jZ5 zQ_)wPe7?MN^O755On!O1!X`JT9Q8CnZG(ZDNgFpfZDZ2LmzOBM{><=Y<$|w0`Fly} z-j9*ry{wFGlfrFaVrk%)fr)9DXwaZZz&2=bQ_$ez3D}FKT$en@dRbZ3B{y!FG-{JZ zJs~>EAQ^*XOcJz#G9`mD8<_M?1~Ce8UNq&XL-H)j%? z_sl`_b9`pX|Gqk8b&HHV;v$!`sZWBRkpJB}*l(xuzxVb#`zHVUF+L{$dzJa$>hBFc z|NXfbH%z9uv28rIn+AGV`!y)~w~rW~@;HDRu!ucBHSGPp`1!-bMx6(Ga`p~tnt^x@kfX) zyXC*DVrOagjIqv(HJc*O!4_m_=`#PMO8$&ylKG4Q(u*}-=jbiF<)2ltvo!nl8k!9) zi}=f*>$>$Ps`Obp;mu=q0&;D+<}bVXZ*;k{H2m#jHr$42MBEij_}8-F8Rz>ik2>%< z!*m=ycT*fbf_G1OXy?9}2if-)acwLV`^$@}e8D1A zCHhF@a{@l=7nIxuaz^Z+s@NTX&$4$65->=>AOV8}46}&=0#6MHxC89o_OLskh_PN| zR_Li49wnB(;UY`=?(U^^MVc9)jom@Uwy=OzK^C|Yr@;d4yn&c6wVi)g1+K(Put+=U zvfD{#Xy=R$0SjSg3%eFYkfAwRt<#%>wnNK89!EvzCM>B#PV`$=;wxBkkLWb;)pp3Q z>Ls(g4c56wc&9oQ#LS1MQ8${+!s?EhF#Cm)>=#VtlgWJA^vtKf`Im^gh$FF*2kACr zPkhYF%$a!h;}47F|4e-n<(HaRP2sku01OukuO z%1udgY?cva&CqxWt~Hfrc?mQn&X`$xo=hndW0sf>Q_lRCWoEsUHO*zY$t@+%W?4xN zi%D7m&v>(MLRHOPzdU<=lZ&G6m~HbD-Zq-shoc}UXLEeVe#3YFro6`o{oZ$6f3(G8#`(95hSH0fL|6EjsR_$4yA0FN+rT6^? zyEeID@{T|m>heXk27>tKmlxHag1-y)$1)mav{;VGXcKZwPS3u8FC07gPjkZSD!oy9 zV*4&$&bG7KuA0rUaY>aYzrS5&TsDzr}tzSQ=nIJo8NxC=C>Dae!H;wv%2WQ z=0BqN+l$M;T|59zOM+uXJ;BK}qGHW4Aa)KATRQw_aO(5IsU|jB-7x_0i2}ejr~`!o zMrcInUGF^_om@lgz)J`9F7G&EV_`u61PJSS!#jp8=maJ8DVkKmHfUzvelEtz3~#ow zZ=Uedd2xrVS;=i@?`01d^BGtWc^LEAlY%8tG57a2y2j>gf!GBT_-(+;=7N{dh#{k0 zlszH_J=ARf@jn+;N(k?>@4rH7+n8M`W{<{S5?;75IKv?Wi7~`)<1bKjBjN>H>7u`X zJ*ofmkK@K)w#U6hIrdaN1t7|65W|12wn4EMEVn@mI4i@2y4;ZKHD*@^Z~+6j4B#?= zE8VO$xC`e2T>A!a8Nf9IHLk=b5;#x5TyZ5h|Ek5`F@S~u(9*qCe{_Ajh*j( z2f8kD8av-D=wR{BC2j1Cvk6Ut7XrqFbqk=3i+In2Qqcufv_&0BSW6dMKmO~0KqV5|^ECRD0ks;RrtxV$b;N2ICO3g$vX<>h=s3UYUP(>ooIb#= z<@h!_aq`e({)n!Qe23$l$AkU~G)OhCGh~@PwS?plKA~HWP>yKt(#MY7!)~91lutSL zE~*ca`r6L#-p-`%%~_K<9LPRo{~Of1n!H~j=v+;X1Q3`18h+3Me$WrZ zMqul;B-T{2P4j}rm^ZVepu2OEg3iJJIX&^G{!HTkY`3~8{?AUk+cEr~kMS}5pJnC$ zG#0SsTfhvpr!j!7FspF2^qw^!cC;@qs@QglOcqa>q`(k-CMs+a=AMOZqV}9XLhOfz z;L{L%8ZcuBJ`KUAA^0=|p9aL18vhxb`n+&zITHR1!KVR$E3{^XFkoFApE(sUG@pj% zbN%?k7|{$kG2kTpZumQa!6t^kb5^>`r@-0yq~VgG=uB4FfTiaMmJB8`6q^PUl^hdo z4#npDiB!;DcEVfPnKb9^E6JW&^87MGm}!#B44ewYY2cJ0yfk>KJa}rWRE(D&KLO%y zZG&rMaYDCR+uPtTV5^M8m)V*wnmDh>^%h5d6D0chTwRL@^-)<6mM_zB#=jZj)Cxqp zF4oIA1iB{j%jHQl#eq3RvmUXuO3_9X}c>N<&3ys3;8;rTWFXs3_IVJO;`oQcs#s2|g3~KRraH z{&g)B|GRFZOMfsBkaZ#*=ve;Gc6+b0dywM)?21c<|ML+(hW~Sg{GXC-=}FOl&Tb=Y zFJL1)OxXw%5nxN>_go*rC&hyy|C~GU^Y?&jHy=juI{|$W<>!GtAh}z5nA7suZM54n zm>v@?+X%~}+@{fth+~E?PQ0tFPOHZJkvxfH2wmGjRATpSplo6eWQ_DI;&MtHag!p8 zxE6Mhq7)4nN0~oT{B z3P>LWekW~nF~K{_M?{^uVs}akh&m=l$Kau2JajZG9NO#70f&s8#Q+zusla6nWRc+E z$Cnq?POEiMtwrD33!OXi)G*6ih#h2uc*u-_FDrvDd!0fTQAsrZ4De;IWBf!0zAVpA z)SbsqG~s-csm;pu014Re(Q24Y>8r9~bq9<)8%CPDv zXO{)7A+mdFFAqRV9kLaBNiyZ8AOWv6014W)RWzNAooH=bnGiWdhC^JSWUmJ5LCs9Ci#wDGNq<3Q**5F4h?XMGO=HWriXk4~x`d z2qeTN*tIBvEQhza0Hl2FNQRn4kV>i^4-btELoYeFCGz`pjNfM|Sf+HEy#S_^uAT;g zY!U)FnkVYFV;nXHfq*h2kcEvQ_e02fh*CmdUR1^FE~*dD$Uq{`70ok{++aQgZ$!Ts ze6jqvlFsfE%CByBc1>`J!75AS=xxmta=U9%tPNNJn+8^vV=c);F~)-O$QG36dTe>= zw0!|~DP27cD%mJh(w-~kbwcK}K_v#2tQnOcCOW3a5WNl&2B*+vBg}i3cO0=rAcV|f zxk7gv6XE(AatO|K`#G`&Wc!3hgd^}8i@^v!s~$6+mlaq>j*6*xZDh&OxJh|8M}~{H zG$7%lwTF0zy`o)Pp& z;*Ha&6C@J3Mkeid1<<2?RCemuSx`r(Rra74gE}?}b?nX+(`v#*4C(+IgF2vX8_1D{ z94EG|{4$y&%|MUkLyz`B$(8ji(4*Zh`+)!}gdP=LY?WCSjj%%1!Hma-)P0_=Ovr_!TiM@dl#CD`B0sg3$pM^tAWzuxJiing z2<>jtcl$Ysb)tz>%|Sv5CHJ)wUi~BXz`uP&<#6)y>Mj;eu_h)*MU|ofx=k(2;bIZ1 zNJ3U(?mnnK$Fj=0xBF^kt-9o<-tLZckyTFg#cJYr9=5KeHoT9n{Yj+Ss%rU$Ypgzy zK?J{7I8VmE~2dQFGk!c-qC8cTKju@^50ggmHKzD-RX9IXz%TJ z4)$BSyPftAt4pPLu=u8r5m>)AF&Qpq3aS3lZbrGLVW3Chre@E z>r!%aSg(U$DC|QQf_go^QXy9#=8?h~!`$r)wumctv!YBoxP)7+8tAm!HPGpHYoNWI zw;G>B%>n-nPQJc{;cTFO6+k-=aiP>7dP{hy{^p26 z9Uai#Zy$E{yPfvl?yl034tB3%I?;nJaw1Sww6Tv`qoS$hjC^&?19QjF%|tnOK!-?RFx$A^?~?jXCzkVRaZsjYl8|9^aWQ9b+k`l4DBOQEd&%l}+d ze+s`{RBIlwy@b#x&PrSYtm;nCR-j=x~;e&I>Y@WXT48kUWt;D*u zJ%!1jU|%+&IJbTKWtQP*}+;L|x>F-TNcgbY9VZ zmYqDMOkXW5(`0XLgh$$6!za<``w=IIiIiRfm@Qq*{ZCg&H1=#Dd?X zg<}W*sc`c`=GW&MbY>$9kD!ySSi($cdH(v1lm#otb26sZ%yv)TU;lRRa0%PJ^?8HM zc5lPBdlA+cp%I~Xy&FQWusi56r1RvKy2TTCCku1*s}<6*$SBOwTk5-)g=uErJ+2eZ zyqA4z5Di!!qG=bvnK6lc2GAHl^GpFvBIIpeL~~#ejX^ZiAsU>r^SFo$Ow*&tLiif> zJSY{6Hup@4CT6ka-~waLqg7;HVDm{;49sGKo6$&KB%Ck_a_!PteNE;zjp~}sB%c`# z-qTJVn-Vcw%QJ~qR-r{F(`!|+)tIy=iA$ywmuLtFouYGoXEzKlV~HtSr-bu*re1lo zqJ?+uGyCysw6MTto7-@z z^ghNG@WgBZ(qg;*P0Wg_vB`FTHA`mI^;nw?pbWtCUGt*>oht&JuMFrkpwocP@OxfJ z%{-uUeY8#nbZ!{vOhKDI>)lW+P{bTq;xS`&|1N6zD7g`R)}sj8rSFkX2u^FE!&gOaemjmlA7W+wFHz4Ub`QIKa&I|4>( ze*bkQ^Asx`42K11hf^yfmwGKt3{_A&$$lgrB_#(1a! zkJy_hEcx$L1S`X-`Bf_M=7*>%t~IOOXj!PXTd^;@@h|OB^mjM*cRT)Nw-vt=gAJY7 z-)&8kkZkD$`j;j+BR1GN4LL)$6WlNPydw2>nhNI7P{FL|Gc*6ow4^Vw&jkLLR%`zt z#sAXY?;8G>NBJ23mrdt?F$POR`C<%~Pt0JsZpxRXaJ)P%PM7kMxGcun@}yW=N=eJI z5D&{!;$0~#(aIvsDuzkrS-B((lS=8m5;e1NDpRC~Ax|+ICqtgH6!Mf+07=7<@?^1! z;Ycx1#XuGP-O!>Ktg>#b@`$u3YvVy#O2X6T5uL1$*M&Sv)3M;AQ*|uF&9RwW9E*@|lpGitvJHcZ zo&_p0WE+NTV=ZJG_r0EYy4Mp*CsJCCo$uLAeq$%W@Ss(a;o?{a7smo}OEGguVvbOj zhNR;ul5`aEau_m>r~H}7|8Ya;l|x{R1SGg3MhBaQ4rBuV$KKvvH^u+4chG4Y{*TA_ z82*n<=l{6B_lRwHi^B_s1|){Jj4^a`jGX!J4xD}v;i|WHt$V9-FUg(xUhg{Utxx6BCcJU&7Ax~PW zigje7IA@4FFA`@S)yHE=^BGLf@c5Ila-1Q6y;zPJrjo}JW5_`Aq$T-WEc?WD;uu4CvUDh8 z2u}=>F-XSno){=oGAOeo;*;r&J4@y~c?t%e%xTg~wWKeLXJt9!UG7^qL%!J~sh4V7 zUox%BYNqj0UcMRPPEp4!-3GrL7MZ8i(~7rcCL8%uR?g?s(QNW5 z)x3SfBtGpjX1IBtm)XW}^BALzG1>q_(PL~jCA8U0m8-nmt=Cyy}JlyoQI!0N$qpco!ogdJ51)G90Zs(PlQXqo>fVpAejE z+$;;uGki)0+AoysQ$6DeNkH}Ld!pmYOI{%d&63V6-_uBQOlIwLky(ob`oCFrc4 z9oyLg{9n^hf631z{;%E6UW)%~_n_M`{9ljpG5lYK|I0Wb)OW-GWteiPpmeiU+WU$7l!B*c})5* z!}aw9e3FLitMqQkn%VZ1VB2T7zRb4ovA2ECobPKTVAPO(Jy|Sc$i57eF;K>keHkoM zLM&qlzt#+j5*egRG@Bc)ueFOdnLxTkL%N~)Dw(8HJYP#RxIaCHuZ25V@lH))5MSC- z`HVU_#qL$m%QJ9#&90CPuh*jsd;Gd(=)9hnVaCvT8MBNr%K$^?WvntKwaOSuucv5{ zF>GF?vc{AF>Kc9r!|z~VuDW4ht|9RG6yE(7hJ1++8NU52{LPstD! zkpIi~1{B(;$t;RJ&YCuI5JzFpv-%C@!#Z*7x?JN}gAI$`F`oac)oyopQ~Y21t^HlY z|MeK37e6*HvD>_aY*?u<#6caYe{L(47r-Sps-s(vP>yIt=o*VVKn(?8Lo_0;eAofd z^Ik?9hkD|NHHsVr8G4x=Y@lAp1D8-#cL}feiSODkp}gHuUmpfINrXjgZS( zP95}*=6t(&Rh@CjQGJ9s#TJ|V_J(+Zx`-pzz@&*?#-VHFy}6csTB@n+TS2{UL1sZ4 zy(H8dLbtHesp`p6`o694opPcpI8bZohvuHLqPsRPl9UM7RzGKq*zuRi3 z`5!xbU9NlA>=^UV!rG8kNexA(zn68i+gSSegtx<{fJ@f}#z!<0cffuIuW5LIcb=y*mcg zYNdila3IT|$RkV)?7MJvZ;mjfgd)3MxL5)30)&ALfDaLn1p*ES;ATje!2YV!Fa!7Z zz@ymZeNesrFIEK&g^x}`H31$TDnR`97+fob0e~mtI~;7?9fNoM4}_ml#E{D?0I(s9 zHR8;7oX^Ojh%@Qbd;#iu<*my5bx~5C$tS8}2S=~5C7Z5R&Qr%{9MN}Y;-!T0V{mkE zv@h#J2Z*1=FNsc&7xc3qhM4*(8k9JwFe6UTYyazy)3tmuK;FNGvPY!nGi%Pa~- zg*1++yZ|S*jm6{80m!|^l(=Fq6(2W4$iWSEoY?T;M9L%FdsdKpEyJH)yby!r`dIz} z!0#uY-krSq?ai?o*Of|C8&NyG+^LG@j4c?}?Kf%N2qUawNJGd-T>2Neln+H$qzh8( zc6s3vGHjjrd>EE_U;RHj=`5tzX8$*P+d~} z!-m9nY|%r$=pQkCWmjW&(9k6W($J*@wR{l`a-he}d(|y~AqQEU(0BpCwcs~V9VwC~ zAnXoU=y+G7=2Om{z$hszqIye=jB`9f#OKGL-C`9X#8VmDhW|uT;J#-=jy`h=IU3vn z+y_GV$*a+*|3n<|(l7G1SO((XRbP+b?H3nbL+l9QAA@$KLQ#NHl$2h8zl&b=q2t^E ziYzi3A=d^j;bIsv4_UZ>7cK?iuK{%BICT)b7L^x^rC2X5;xY>ppUI-FA;A`s@{;Xi zh&I&X3LQoOb}cH9DHBN2wbZ|qGeCV85H~(!7`VZF2plXHPFZ$Xl^S5aH59LZk6kQq zdIP-ci{%&C1l&M3(UREDd{?!W%gXhMlgNbhR0 z{S=MJwGy4>Q|As`QYibT&k+SbM_v7MLl-*2Tux0feJ#&F2Ax)Gq`xa3t7Fh@y~nzR z|Mn3}{h*!w!LIm0uqvKd(jlRK^I0|;BL;4UVqKsBg3pQPVpB_d6NGw_e}#*l(1%Da z3Swf4)~3qfM2J?k@m*hz9fr8Y?){re zg0xQe~W$xpI)7uM&r7o7qgTQJ?5RHda&|B&dj)pPp-B@F{Kk*6#Wdc z0r5E#YpEUxmeRa6sWvMw#5GJsLnvZmUJnsP73A6~@KOq>@&dfM^#r~U1?W*5sJtOz z9Oy)giuj^~`E@fYFTm%az+K_99wlN*f_JYYttEeW_U?7`qxWi<4pp;$W`GjHC79Cy z3v4yF0py@5e%h3Gv8!4bNK%$6#Qv7O4`x`XRKuC_8ui7>0fyuTxabDRfyhK*=oT|B z2D-jP+^|A`(Cq6(X`-nfB_15eiK06xd`A3E{Jtin?;(rhYvkNDz!_l-Upgp&yO65N zeIPEXfnN#Z=j1dfL@i~nu#KrGLBIjIe#$jn25XDiN=8@!;F_p_8x|(+2#wqT?+HZ! zy9y}_JV;zMmV=FO(#~oVi#09cTFB$9$vB~KfST9D@kfX?U&t=12Q3#HVxYS1EV^`@ zZirNa%5`F>R;#;|#TqAl1m_u&BFMoRzED%6@+QjOlFV)35%`E63nli#O*hh6v>gXC< zJPxMc+;R$ma>Y$o%NVc~qeEyVF^Q-zR~%^DazbeXnJO7D5`xQMnbWJ32Gk0QNP}Q( zcO`?qfmp1VSS5p>UXt62E?)y^t$_mY2VQ^pET(iH4}5ikO1f!9TR7E9+l}Mt7it+q z^jaFjlrCvEI*r}NUPW{Oq}C9HcU;zPuxqOk$iyZih}{Odg(J^F`k~QX!**Fi{kFmU zejndfUH~0z7ZxQqQm#!n)1yv~!DCSER1=RN+hwifWR1@h0ejWZ76O1^cy~#$9gG?= zY2#0a6ZaBvm>Nf-y2k*H?ESr>MU+jufkEo5!i^RS%QU{O@e5uDpFCj5mPUs-m zQj2QV>lSfT$2Dxst{8`$HIygIuG?b5A@=hBY*9G+?hxXA@7v$EhTqxu>uvihtpEAP zclUaPE-vi9xZ;1_w)p=~IfVvs~VOP96+&4{h#1ZeMe~+bu0W^-=hfHA6@58srv6~ z4#$2K*Gob(eorZr&jGz}RYqC&eH$*d|nSzr` zKI$Qgi5*^Qi)+lc#AYS-%60_+u8cy_qp}5`5VUc!2{gF032j)n%|HH`!~YZ7?VBGy zp1=9r;J1AIJ0|%5cUya{l>cY%plkd;kMe0Zz%NMc|6@*8X=%Q^1Jrk2ZGd88b3GbC zF@`Jm_jQ59^g_!(^@DaDR6#?9&NlSEpuz7vq&y&6n|=8B`c3cbh6rHlvqor064&&XhpMcdH6(KVI&L<9(sexk zDeQJBJmfHxl;Cd!+OMDkT?^51QUm~qX@3mfaWH~+fW!Zvy*KP`+eR9NKVSX|TsnPX z=US9xCw8LK^m)Bby7k?(^|904XHQO!4Uv$=HAS)nWmg;LIrsbhm-m;v2QvT&kRT=5 zj??a1AKF*~27|$1Fqj$4Trxt*gk6(aoX$(>LKF;Tf+R=B>h7j&?t$?ziu^sBlaQw5 zT4+^xv4nYwyQ0B}PfbemtB~}rZ?%d~t5rOGd-B(#L&rh__&YD-ZLqV}{^=*Rr}JDespozA)PLrwZrY+ho%2S`Jpn-Oo-mIzChC?F#L-9%7Bjal^zb? zjFJ^6g0dMyU-5;qtooH1R(xHVR{giXLb3*nS}ka(@D6K@Ih$Rv;xL=#v|z;lb{Gtw z_Xa)*4htmdJLf-E{Ir72!|HR0!$_F{>BhGv@gXNVp^PxSxAL$!Lu17 zf6>H60)X!loSztYp%^#jWOKX#;=K9kRJ0XJx41}c1>Vs_dxHRo))z6>8G{$9fGr3| zg1{Gnyu@~OT2~seL*}R-HHlT5)&?uBf#OB+2Rxr8qt~l1vw^234MMSXOO31b?b--d zva|YnuO@czg+@7L$t>VgR5L~^scM7vvo~0};s}RR#qns9HFPp52W_8;@PQa((9UYX^eJANtb)C2a zNt-C@cz7}MMwc7%>pUx2O{=eMC+!V}pGcYgoz9ZxuiZdQPpeZ&7h zU>U1z6-`ezH8Hz`-?wTYN_tWMN?_1^z>U5fDobS1K_adJW*|&ufI)MnOtBfU6fP2w z*~75Xm=T0=O2TY5j#C!NXx=q-Pie?OKC7)}!@q2>V{d6I2M^a26r7IHx|LaLVgNh3 z1cJ0VtU(0XqxThvVIG&&!v={^1xoj^hgzviW85XnU@t`bvUJ`5?_~ecuQ0kFgLT_~ zc7}r&b^FiWaDVV<|9Oay`M8zlIqzQ$y~{X_M&u1F?kOuhW$+&9SD&|WYIxS;nOj+} zJtB2N=)e3FPqbEjL*g`IX-S?7FycZhCZqI4^&+GUr!p$i*nYQB;`MNQ#3i0;k6EBe z{V7XR53lvIr&R5Jd~GIaJ5rBv^a6^{U5WUTSOUtvNtofXx>@6K81`)5G|b*NBy|s{ zr%8xfN*JW`bwktNp>(jWe$~KDEp$~bcImHy;|5iGcwr7*dy&y(j(T5vtvarLjYvy9 zsw2YEprNy4ZE+gx;fPDB+QxRgl7>*9stZWPDtoQ0LX5z$kypih%b2;^3sCiu0dv+$ zx0=Vrs;;!Dq*fkOxaqNuji;doy-{ba-i_KlX>no9iO8VSoR(9edV~}H8lP{7Kitju zg6md%!L3HMRjj@C2;@@_8zTdBk!7WZE&gZ#w^pg@ea*%JK058m2#41no^1Z`^XdDu zqvNfQ%N*Nv#yco3qz%K#Qt-3hH^Ut#Lo19S&RunHd(vcgtM6iywG9TwftSz+Ub>pZ zLME1^=~_HRDCc9|`XWe|!vH+lgXXdcW!2T?`y!j!>`2h&e8ML6?+LLIYz!MeLq`m^ zlz|X}ksPyhE>uC&3r3z?vc)!e0?3ik>uTZCE4xdt(r`qd7J`Uh5tdK7W304ni3{Fn+9JOQ$uYvY{_T3J zrs7dQeIoNbXT_4)WAi!~XB2DeQaW1K0oI~lTR~av>Oh*|vaN9HP3xfQF}AIM?1na& zW(;xl>D&o+$FH%BonvrjQM9IGr{m;{ZM)-+)v;|m>Daby+v->y+qP}%=_pS2zgOTb6RWpK*rwk_*|-=+oUg8rrt&jQG%fxrThvmH!!b9p(7 z%IObFn#4YiBPaOdzGZpw%VA>M!DVN4=mS%<_s5~GlU|lab)9qPynXZe z8MgEOCBkGjPMQ1FaQzVWzgMUkKi^vaT)*Apsu2ngfG(vJ{Du2At4`8AOVOTImb{HK zpej!;x_-NZ{c5`SX`in0fF638&w)eZHO{G)TtSRIvgdVYju`+uCK#XNT zE^kd&-IeVTy=gMn2Hg8q?I!PrAAdDw+p9ZPUL4 zg2A`T7XNkk%ex2MRX>DGD?Yh6qaHSomRC@^gDM2}kS_}_JI;_Qw&svwaXY;ESqo{J z=KxvhIT5{LH>LJp z4#Y=zT^U(B$N$xU?76l!**d?vNIlWa&ua)>ywab$b%@$nPPfRns- zbh}iI?H_*$xJCfyR^hKvf$V^b>&1J4i%SH;a{=Cw(i-_P402?6yH^utx{GSdTSfpT z2ceVd7`hpEPXKmp(7!cFI`W+TPFsU&Se0?4&Jt)hlhE+Xi_5rO*EJy5wCA0JNj!O# z3>@7ikmvDb`g0R2_8ln5-~6c;Wl^RpOXqo(dB4$lm**B&PAc0Fq;UrhGVjhc8S#>9 z4}@pX+j}q07fyhgYA7+SFalmxz zAvOj`+QFfZ?E0(S*F20U*}!{ObSs3|ev0ZE?@<}Xe?d(~yiHSe$ktk*;NuahWJ4(3 zBoxmfd#BB~mH4WZ7~P6CmO=Ws{_#?u9A(fa2G-Op@UcWH{hpPo8$-e*a5q>a@z>)V zVWu^87_~|!9Yct?@DoQK(=neDBF?o+AxmqQ*G3gPI1LB!jAO|87un$Izzvi^1bOtb zXGI=s%WJ8z_CHmt>m3)&Aw+MiB7zK7WIIeMIe4J&wIsC4Ifl)wZz>A!EKHN}&ALa; zAg%8ElYjXmg4Ga-P`cK?Iixyf)#rV4D`z4&~@*6NfpS&0M4;bo2dyLzy6K_$Dy6)vTjk4THN3r zR&973gS8Q2NSAr4#BGhwagVev(U9cWwYF)JCcGwIQO<9@o8>o44n=LrqF3z z7EsgCmP=M`N?ey~zy9!ff9-jLbtJsgqj<95gGyNgKD(YjeZSmI@mC;nZp{0(CUr{t z6pBen`=!I&o6py?Q(cvZ@ktVRHRUi3N;G;FbH6xRUgWa^YuO-5BR62koc!4s(m}UCoAC5aIU06dD^c*=$gj* zZ~Ch!b*74pB`bC|Ph0*rdpp>{Tn!GaQ%YiiuE<(cg1SFu^wpyX56G~rbluz?gWRst zymjTj>dLZn7rfBhm}aY*rSnCxsP@9p8Wvqd6%2wFus1vlO5MLyH?)Y>Sgf2OwJKaK zBu2K(aSU;`+KhiD9~ilWRrogB3r9T8T(c4@LUsLXXo|K`np8A44+JWA-teKBD|*y~ z)5v%2Iv8q5?l7xN(%9P*hvw56{$x?*^IABCkLZULuHk|pJjp|<31nrD?ld$u#zOJC zp`{pV32{e~76wnuat9U-hCHZ)hOf&M5%{{4mWMSS{Y|V#ac6tjT0h2o^JOv_MSw4? z;TURo2S)BjW_bcfY40^4=5)M;av~ z%DyC#!=m6~I`{qTT7jC?8&X5i4F_z`v)yq>j2&bSPj>1jzHnmZ1Aw_&p#H9@8gP7& zQe4A{CH%AW_O%rfOHgWyd9OH_2%l%5IgOJYwk*yc~z9`;|0AS-uzwaW}d}Q}a z>|3H<@h99&T1e`BmVWS@8EDT`msww$uE%tMV~XeW*ZMkHL2_i29CH(uId@%IZy8^d0MF?G1o z-7$izSIz_@!^uyB23HlPH0T`Y0I0^D*v6^@6G120GKHILud@7i2c;Chb47Ge zCFF#;bxaudpx|ar2l86Qj`64sK9Rh`W(p>@t ztpCROdsH_AEj0h) z`Q%I0v|C1t4ZdUdvmEv*@Rv^g#)}9C5**lWma)eZ8uBdpsYa%fVs4g7ZWKRU>Ab-B zO8&Z%np<^l_f5y-(_<`1NcMc2w;8|1)Q|pMBxuTOfdpN-7!!udRR^xWE4ExUxW=3> zGUi;{lks}&YMS~deR2xA&gK*gpS zC8FD99O=0F6!{tJ9zw}HxJ(=br;-=%w6PxG`m2yTm^JhT8O0P`=wF$8Wp285p7A$e zI}9oDWOr3E&3Q)?$U0)?BT?+f0;W)FQx1PYZS@%@Yl!nOB?O%R8a`uBELf=6CS5_y z&#)4_awkIj;Sam#ThRr*7xDq?u1&z>;4w?Wu-8P_z<;s9ij0qI09mUhi=C6ox4|k! z5pG0Abj|t?8+^BF8aMy1G1uGmFMxPek;6Zd0L8q6^#i!EK1=G)q(me&^F8E;w^r2k zpj}-L5CD86%GEUdic4V|WeBKvIzYEvj|yXas=Kx(`H_&rApG>uv1|-Lcn%P>`_)a} zuYX+pa;#*wTrp+zSZOR-Hair73P%tV_OY}%bpuCY9hmD0AUnIFVlU2_m?Pm7=0BeA z9Pz0XXP4KJHWz{FV*>HpQQ%%TfjE?e#HbhsP#a6gG@gC3ble{B-Lgm%ZRt$ug z7i+?ZB&vtS^(LC9^UU`DWGU2aa#h)FLPJ<;6sl?mA}Ta<#5!LCcO!T2-@XX1F8Hs2 z(683W8kTC=#AsncEzkacKL+>UzMcaKw;w4qmJ4?MkQHy(fY2_) zJ8xBFv3}|{V|dh3`2k(2--ivy>*{8hwMz1I99a!FiuRH6jfNVTuDa-k*^`w_YkE13 z-df=MEc;tf`KJXH462!oC!%T&L5Jqw3d%%;1v-{OuDwqunjuVUVH7G|P*NwW-YLJ) zbO?~abvm%p@$G=N!PW|RQ9O^mIB~Qpj8w|sli&nFb?5=@>U|^)*pq^nSUYGL$MR8F zrJ-`PfBzox*7e}v{(xR_BBLcy-3b#gA>Svp3w#*xigeG|3lfZ0&UDQDAo?1pP9}S5 z#4zFtrfS83;tABB0aMW6^XMv5e~55zr{7*hhlH1g$IPt|TH9sYr$gk$+K(Cujb&18 zZX(?a3}JrgLQF2jD^YikOEspys_ZD^0DJhuj-2 zST&tW$A58CuL~8!~}Je6xy^C^-~|zzyx69E=H`S z2h{2hFn18H0_%}*qW>j_t?C=Xas)Z4aiEd3zPjygGRCSJc_~DN%3fn&6>kkPufR&s z@~ibKq6DTAwYpb5W3o^JqsEqqy<~4D{$2a9%HL7}# zCl^yH!}@#%C9Z^v{v-<8jAcW7ax#v}g9f`3QJA7Ol&BnpOl54PSzn?nG9ac++w!#R z*pg3-pj*?3drsPkNV$}rHNotaSK$$#e5bTi3SMO94~Dr!NS>K!JO){2;AEG+#Q>bX zWD?(At|`OMa)`>8{xsoy*9_xR$k;v@RHvk#btXXUJTgt(@J^UtXhax)ld=kBah+gD z+(cpXy)d4#<&j}0zk6Y_J1ba>*V~1=W%DS! zu3Cp@Ey&(%&;+F79Zc}nSyTe+>)`=Do^tYDQA&zLZHVHTM=rmG#qFyT5)O1OKWtx; z6Hdc)F1Qay1xOUps8B`}zxO{EFlLzH6{1p2x9J9$%K1uGZB!*ZE2`cbU9whp-7$9K zwOU6$6G&EMMgZe0hvTR71hyogD*#gVSaL6&SUnE~)O#QSRuSFlugz&>fNEk@Yl$et znoyrlb|ff`AUnpn{U5q^B-f|>gIVOhu`3OlDPgz%GNvRZvRfGiN)bf){y?XhtbZ>nudKwax(#*cufE;uu==O0CnaCqt9Mk5|`U ze*%6IB?jq^I25&}=G7}ZO2eK>r>I))x)HZlOpHY7vJ@zK)=5Z8ibB{e?&Hro&|P73?;M1AS#~UkkXcFD0h|e8L;g+vh)r~msL1axa2z4SMo!ZKdlYlccT@e$$|x)W1*Zcg{S$=!`+m}ia7Wy%~a!;Z2p3<`)N9#Ucw zEQ&-Vq&x5`Dd_i`)ZnGEHm8GYB|TtHxoVRvvlQ@$daWlM-n}!-0m2F_-hfkh02u|k z@F1rB3=Ee$PG26w{~T$$5j~08AH%Fz-n*3mm9+RUE!EoK6MwGXBN5RA{zZeEEW;)B zyAT#3iybV^z#vCjAZCEKS1M#4vIHcwOzdFuXK49>IqH?UTh`MI)~m<%D*Z+nmVy`u zIi@~Iz75J+7fIyxs9Ef0tKf{^7er&Hh%uT6S7G-iR6ai5F8O_=In()_-M{_}e{HI% zemsaLz~R<38Vxo)U>ijq!7YXon2F^+Yf5G<*ReWK5FQHPTD1FFl5Q!~P`9*5PIpEi za~i(ruCb1y$=>cYI@)&|j|aE+(4@&vJe3@X(dc{1%_S(gprS<+Bu9jj+xqG`A^`CF zo#2Mq?{7`E+i$W(G{pbQ8Q;q_%?Ui~PzY9~k;D2%h+~=M2_9p4ECIM%XqJQUUe+0FhmW+=MJzF>O zv-YI6CRj!{`Pa^6puVyz!f#b&_#sHkqr71avY_}*gcPUOZq!&Xsefdd;+AqSr#f|3 z{_47Qv>-lNqFv7#*U%0=5}9?%3q$t_MH=OrcxEWF2cZ`kI`xKHn{G#YQkir?U%J9o zrlavGn75En;r^&n(H#_97P~p(g|rzVLU~|DPCGBpeX+geFQ=WQpm_pXUg6E>IxXi;hJ+K7P$+`m&l1LrCZq@N# zgdirJVd4z{E+Q=Gw}VZ85GYL*xHht&9NGl(Uc(T0k*@5dYc3C)MvkzQ+`Vh@q3)h^ z>cdl3Ks2{E^?vC7;1yCu)on-@_Ub!Q2oI-1!5R?JNtDssk3h9Xj4w8wC#heUX7GpX zc_jpfDa-=-b5%Zj77MQ%sihrNZ}KM8{y;L`0{OSsYmb5Jo14B9zz?EaL`M>E+8yfT zPLL+GR#&p%IznqM@04~e-2n?O;xEq5>uoqc+IasK!7t2G1((TDYJM4%#8ZL$g_gB{ zR3aSF1!LxLU-Awxg2-@ZRO>nBV6**qR*mKg=_n2*l z<{1#Sm33rC^NFcetiQ#yAC0kfFKD~<6Jg=UQEq+57GyE>Xe$T8UhI6c=LcHK7X!pp zLdMjM9ykb08?{f4zgqhBUphH&B<&@fxMLDqqp*>_6y47Q1cr+p&tZQWsm`rs-qRc% zCsq(MYPxVZ*1qM=N?XWyB1d9pMOm|FiL7JS~`3L8CgS zHc(bWl|TX}eK?YQ)%syyW%!6mORJ_oZx&@85U_1$SGgot-8n1AUE96e$H}{+wtJE} zLYb*z2%VYYKVem`Vx?%NV=doqsy3dbl*oHRX?@64#7&|g@Ydw>fp~2(rv|@i%;ZKC zAn-2pwXT_di9*feiVVo})kK-jxmZtxdM42HB}9E6SZPjWbdNo9=lgEs$1sj@bQP8+ zf_Ti5BLUJ4!oG8`wklcuSJ^5yK<^Fzc_J`wc>5O z2<1l5-dkBwy;LefXHdZ;(#W`q4J%CRLn*lr!?HqDE5AhsWVH36c0!NTttQ@k;|IiF zQe6{SR@Ok#>?xi2aKDT{GTbJ;JnX#IjleS$5;4T=_d3Z6Mnlqr*2y-fM$wS_iC? zWL|wu{cY)$V%@2>XYq73K$i=PS8j!OR{G19@j4}#l{}eADL2gy0j2Wh=DpKRi(3Lx zBFDc!K1qG(YXFZNfLw68w=oG%O`!r0{EbMhK?uFoP{ScB7>PVWVSU2Eq;;9>l3{*6 zVXl*w8U1l|d&V<;Y?4^+SMgc}v&t@@RIzH-Li@8KC$i@LPi6DtHCr5Mz^SfTpn*{3 z6tvd+WyN^5LL;sI;!FeAtmWEZ^kaxm+sLvuCN6!lmiORplse^PyW2t2+-#3=WQV>v z0=9Moa938<5}#ysE?o4Zk}vuYXN?hJWqWZD010IMp=vis`kv z$lbP)7V7b0tO41&bi7d&rdpK^b6mjtG@w3Pb^482Zr(gR_!x5tHO)Pvm}cxSHPO{D z$474{;C1L55XGu`5ctc56vuY~_+=vA=p@2;y_-KaW_b~9aN{YM<9*om^hSSf0nc{J-sv6Xe-e zb}U6@R-M0Jnlztroo`gwwkzxRH{pb~oT`UAk{qm4C{$F$t}XZuHCt)2G7cqnizy?0 z;`GYRmYT8%gwG7syT))|(Tw7KoYKo3E8qAMq`)0K-hL30?5<wMLSMYH-w7w382+56=&DF`z=9x2 z?z4AWOe6?Yq#_=vwhjv~>^}DRLWUrB4Jy}kquiR5#qpaglVrcX^*I~>vLA3`JWTRy z_5{x5R4^A69+HTngHdy}&=gZ|qHd4h)|2; zYi@h=#KP--(KD0^=wqoMlqzW3Pp(@9HWp>5SB{yfF)Q8+(m2dWc84e zZZYBCcQA*>8L`|l^2fmn-Rc=OS|&iC>#C9E!YRkEB6h>>gLQ;By<+n^fj;HWxekl# za$F6QIb#vu>_or}%k`QXT^CCWx0GPpG~z30Emb#TLwfo_fW8m4g{6&GtcH}ExnC=+g1or3h;{&5OND7m7$U6lXf?$_O_)9cDmy4m?w~0Zw5&NVubI~~ z?0nWgvD?jxH4&k+t-QB`(BCc%DsfOuU%|{ru2ll-&~WM0$Sp5ZPtp1Qm6m|OMGM#> z$m;L1k$zDZLZa8QG<5ZbXE65WklyEvh;NR#_<%t!>XJ|@apZBeURVLRzmJdXnoRiog^BS1($J4p4f9*td;^ho=NqLBBdYtxkt`+Ti z5YC^DE+0w|BREyDEPU#c&L(4Gbfu|#q>xG@Os;uK*AzH&)HK@TKs5YQ(O3x!B0>WY z%EvE{T=@3jc%(t9W3&gAT?b^29v$;lu{0`;T;1$*7v|H4vLu#T3&IaHTf)`~Hhz@s zN6Z$T&Nn&w zs_F{Z6@x)AV(`nh2xoWYSBsQrugTl?mrvK%c4S}S*E3UYZ2_?SrbkR<`!oE)Gi*VN zq)}2jkD$bTj>P-2tnJC*n%KMszZIWVypx=w=7(PG`n=~fv3&w)3RExG8sDE6+=4o` zm+Dn(*a9n&8m{IU0n43?%jllSW$e)gGRsNpCZ7q-8dmR3dc(<4rSoB|h4*S?ow|#= z+fdXs&Dkk9=i0=LWL&S{^7VD$izw|()u8N&n`85msYnvqMkZ;ogHPfRID?ErBX)f) zwX38oTDgwR15fBrxiP}kMqw1~5+LI9`L)2)(^HPXnJ?09+xj~a53J5TwgjnUwi14o z8zW3(fC=*MdhIxNvb-atLn9gfJ{B?$JvOPr}~7g!s&Pun^%ndtZmf=Q(B);3~sZadsc7Aqvmz&a~WP-uhB zD47}4Rj3_eOpIRUR;$&2KZm-X^@yJ!36#mH2{?`OV)(8se91-qUO z7~L2kwC_7*vFhjSjr(4Ljp$;4YH}GrKVJ$)taG&#wc$MpJa%NzpC_BX)!XaZuCK}~ zp#E5bhF82Pq?Tut$vbe+mJfJrZ^q=d0Yti8B^qv`)9_EewD+GaW6BwZRW_NlaFb*N zAn|Inr5eTa{bx}5j(f@N;8Fk0&&{xO8zPh0c*)I=tP@dp^d_`ta*-(*q6~=vr>jBE zrOc=w(5d3H1P%Bx-TE@;8|PVQ)4or`^R3}t&v&WA_X0foM75uGd&0W0qJSC?yFF)- zr=ofv;-90nb3TyfzlDi!iHO!cF>lT{JTaqAi=XMJ%=Wf5Q1smXdsZo^YYfykIE8c= zFel8!S{3Nri3rsgc+>NCs=d*02IhA1x!v!t>tk*jMqPdOG;(fKF9Y!|u6T2RNO!B| z=+94`;3{a!*>*`r?7okW&?*S8`xLwifX8<%2d?bc-M11;m`X7%_4ka%ghY4S<>C;J zh4v>w-uoDN(!&E`Z^P^nsS-x`p*O5~T?PnUJ;`n# z`~}Rkb2u^Sr2#}!+&SeQsPu+bKaj4{tfB_8)-lXsV@F-@AT99xGE9|u1|r2OfVscbPUHvAz$Xv;|U5FpY<`LS(U>bc=87EKWQ}7 z|A9(REoPS7V@RAc3?MlrhQJx>ppB^fhEXnd#6Arm>Wr#hKWXW#_)-8R#%5mgHEf?K+tL?;9YH*4`OXLjI|JcSlPOl>3C0!C&X;62+=j*;@AsMOofma%x+cFxu zPe*blB^+~j9{@+q)A&0nxRcP-V&pF2S*24=V7egZt&oyYSIf-;hczW3V6uT0lQtWJ~f!kj+rVEb1(w$LeAE2 zoR6C_n3WB^1$_VhZGDg7^?XuNZJ2E|+j2b^Y3ChVYZMaDHD^lGh2mUgKMZor31+sZ z$iX-j(iW*yCmbFdpiZBm6_j-tg7c4K3P2ONDXe*g(#1cA)i8FF=@R8&d?;QUEIw03 zI0<_UiDVrMadJpk&YK#g=rAnLD+B0@&2z~lK*FMlDdjVrNZP{kM@ZA{XEQnb0|h}6 zjYY(;u~x~YiJ6gDS)Hl8UU{=f94PO>Do(%SWZh zp|t$-l1h@5bAPt!lkXn#b}qPp_gZHjggK=Z<5k4VrKfNhYVDoRTE*E$06>;A>VE^s zr`CC;A>aMYPxcO3ge&hk6h&?Q$l^Qq^%*CS!%3JKAm`;QSjt^`=6Y@d0D*IbpP_K% zy!*|!8j&zMC5W0Z{N?cVff6}5uJMM-pr>}!_y!X}J7G=nrOxCKhOB*4D6|$?>e7W> ze-|^(2WM@~bbfYwz-2X9?d6bH^t$KJT|kwR%aCe{;KhaubFtjevZ4euknJdmE_zkZGm zePUzm6#&0oGB3&8jm>y&|3*aK~O9L*)+5WjE?u3qnK znTlU5<#aIA47Pj;q#`51nA1m`mQ05?eBuusin<;vkcht$a8XpqnZ zrCDT>p|i}CL|n(O)5anVa4x55%@Fi)G0?S!*6~obP$S|@W!IUA`lx8KZQ`-?^!MQ- z9ah2o-V)UIWhGT6qi@3e;NAiC-eNOl4viOEZcpApl1P~r4>RbCm)gi7P1%%KQk3CI zCqqgh_8W3ugW9SSo&HCmlaRsT;q~KTMPVNi=?d~-;_=X_P;SF_l3Fr~3 zc1p?yIFeC8RUDUiine-#LPzE3irV2i$VrZv_DilYdbPB)$glk1_%DLDCsL%e_S_j| zktHiG>(Th+(TOrV2zBC<2&8gTV{e8B;Tf%nzZA1@NT;~2&fomvD*qNlHoQ?~qC_s- z%>5WhCTpUH+dR-LMZcgq$hSW9{z&!eyxQ=N)>Z~V1gT9{CRYuB zlowaZ5q%z9CL*$nk7{F3vQwIW<_FyC1=40U2lo)TU-1Sy#y7%wbFSrK3%$)*D>jx5 zoBxUDYs!Ks4C0N1SSK;6?hDUWu~+-|oIoEY6$sy{tucdjj&EVe(vtDmzHaB9teZLe zTRL43EQ~2)*3HpZd1hC#jPcK1c7IF#zaI(?ze`K6AI@!2Wi{N|f+kIHxI2AR3CPN4 z)y`xkl!%du&Q!)Qexe!eFR(#ysdtRDCEI9%L5sS(4 zfw%P}|NcQ_=`py52E;tk-))v4flYIIi`3NhwX35=+7w1%>uTqeqA(+@)O$^kQUUo_ z{>tXUk7eVBrlC#;y2`gKX>d~&3!WctO-h~mQ<%B<^uJECW|MUDRyaaeEVb;#S6OVNbEk(8 zv~%@yF#y;<<*2tQ>W_HC6b8VDE|%&%R*_X#lw~S1z)WZO@l|wx%ZR7q21W(Vd8Fo{ zFK`l#mRn%@K|j#Zo$l2oW6s_o1EpqOjMQg}au=gM z8f;3F`KIqw%U|-(xc`#;c~CN5Rw+ZqlegTq*e$eHWQ1mf#-wS{k`|fiN}4R-q!ukdFmaiB$5rWJS6QN3vH zIAc?0x*haaaPu+>1hAT5mZDrcfiyr4r0}mmW|O}q|3n&j*6^Pox+Z0q_AQ~2y?XI=AZBbJGZ>7`%fR>)4%^TmJ}!LKc=|HyK>_yMd%e7T9Sicu zS&Eo;+{3_HNG4tgiAE1<_`>Zui+D*;>{)~Da;VYN56@rgCK@npz(aL`zNR1qW~c7u z%4O@Y@{`1=O~0r#6uoGS2;XSMDElRJWXFGj8{mFy^jq6s#mtFp_HW=vmm)U@k+V%d z<_78m5IW1?TCwxD(RF`{EWtyMSNA+b2mVsdfy)sYuSe_p-l$M=xXrg7YG@W*0{SBE zTqi&6@cJt6)t|ozy8-;Q$G)4Zp0}+)!h^>A!U21W-LTqg@XtFWUVC4Gms=gX1PoS9 zI8kXT_JY3ehH#CEQq-IHGpUUKx;|k& zxh~Xu}m#lc$S4t*J;7S!!Dm=N0qFVuSSJC= zGVzi|6?rRXHekxleu#ME_w;{KLvFHhqt$o`X%n*urOpfh#p{ARqnd3OAs2+2B_oK| zE$PB8iiA7HMUoYZe zCmt)yI0*tvG-b59C%S&Jib{6?yPk)InlxE7>WpWS%D66=^w& z4PCT%n(-hzJ;@9vO<=PLhyXX{rgIZbK=NiEW5AJNa6W#$c?37U<9T0r9=J8_XH5x- zAp}CVstaVg$6FR^=meB|BJ((NDZ#!FDUwVKGME+XESEWvRNUHrM_N z(EvtY5d#A5LC|d2_$UZ2Z7@Lq*YUshGMh0xCgz6?@> z@F#va5+GhjfsF0KI<#G;UeuZ%x>~bM1MV9UaLe02g6f~_#q{qnu1;?ujpdItP-jpQ z+x!!hB#w7*K=0=d*h`8>%aL3@xozmhSNCTLGQo(q1o*)biv(i%DtSboSZ2u_;?{0^ z{RF(TNHNOYdNXl{zR7LjuX-V*i-wHWi8BW}C{Lr`pux)@F6Lob%?5#34Q;9=FlY4X zkpBE$4`Dxs6@(2DajMix$4AUvg^C?;;Z^4%QQB}I`t@5V*DsZOD*v$@1nQk%fD`|t z(}Usajcpgt8QdEhEYL-bh1aC>*yqxV>Ki6f06=y0>z|y$N4CojSWv5qZE#b@4DTv7 z#!iNLuue!&(jrAh-A8<#X|5coz=}j^!H79S@0p1<>(7DPK&HYPkTC`xY9BiK75XJG zjUL6Xp$U0lUEt{6U|zua5t=DndoR5UW)aR&&|x&+b_SJMukQVfv?mcTUcQLHSl1&u zL%eRXpDvumkP#-9zct}ZMA~OM4yRoyM=8cC?L-&Cb@VrfgZkfyiO)?GrEBNDJm*df8PYGI6t!KkjHUd;6P z{oxwQKR8PTwXcxO`ANX11K-yZtU9F1gO;DgG8jAC%sFOJzYC*(9q;3Pk6a_jUFja-$UH~w6o?lg`!0NeQxiT6F zjRDXmJ}qy#{l}i~k1w%UfxB0Qz-MB9T$<@1N2z$pp`oyR1P9hfrM;1FLWj{cBwO^v909(E@qK(!8@ z;_!z$*8aqtYlCCFIaOhA9a{ny-0R2cpIYvFSHL6B;-;Ypjj1jq>EXzYRb;b<3YdGX zl;){62p!aYFe##h>cRx72gm|-v^DwSI255GBgX`Gg@IFAHT|-|%t@FHLf>^ui^_Sa z>edNmw%Yd14lcn>rTv4<0S~;AhNxJJr3_ASZ=T?327Y~dIfeU-+{&1NX>h~aZ0X~^ zots=@x8?~&pb9KeRKlOPYhlA&K*!S^`SmQX zUZJeE)_(vaMeZ}dI;J)|{uz#+)gcMeci>&n3|KB4b+K1l9&&FEhc^B+Q=M{ z6OOAb&M-rg0<@dA7VVtSTeP7%ido2eyXF1jbB8vQ$czfQrzkVOGoG3av^wLpov*o4 zbbbHpYSXm9l+m5-LKX9u;_Y(jP&F*O%R`0ycj@p2WS7+=P} z3!@Rksz*4wV<`{9y*C!m;tS4*n}YmaFiMqwAC{rhFsh5k|0g!G*21^h$mNArA=gKL zj;2Qu{it>p%)!w-&Efy|YZW`XAi%-L<;boYqNJ+g^lc!uCwFkxQUYLM`DPEgZFB+C>(gjXFbEY7w{N|lZ?Khqr-{=qj!LT0=t9UsY&uyT z?!<#dmbohn^2~b=_a_BEV}^zh2XZ;5j%p~?t3p?j&!)P zjOmCtoqN{ewadgWuG^~r;zo>)BeXaOtuI?D8C=CC(+ejj!&T=QY^yjQ#zni28raZk ztC_P<&Wkj#VzX4a-%_m|un?>?mcwdE%C6wc0$Q}8RuAOwdt%A$2|=!IzlxXjo4H^2 zavOGhl>B?Pr8--lfQx&IZb);CT!bME?!Zq&plH|E{?}F^@Y;aC*K&UCmZMxt?d&7XHeuoCD6_w<6_mjLv}4nnX;sbBP%~Htr}#6W5GCG*l6Ez61ihFio{?l4}ycDZEWGWleT&L)&y~t3Je%+vXa#;w)0WofrBrBHE z+H7i#?Yz#x2ZIGwaok*}y%fJ{6u?g1FxB=gc-9%yW>76eeI<+xbe|&X=40He{Aqw? z`cjO7SSd_D=qf%O>QcC0$hn9$$ji6WW3M{$Yta4QfAmDYJQ3;eR+|Z$fA|dV%kr6* zRoCD-Ngl6dB2b^&oVvf%)zSrDzlrP|+)ETH1qmI~^?`JWxAJXQwY6mT$*Xt-TZb8c;W0Ag!L$+V><& zhT400&`Czms1%P7==Aly2#d1ct1hGDoNHJwx-Aw4_O!g~iOc4hy!t}*3toP;d|yB} zpl8_havN}#_gJXK_H^;L7#+wCu|?+)XSBwh_+vxg2Hdk-7b=lmfp(qoC#W8$O*2k0P*sh!bgJ(Qho*lPv-c^!YgPSxbj1l0Ps0X(0Zlz1*YXc9M)t)qU?Q{W^{M zaOw60izW-R_!>;Tu)=7`v?7uF&1uRb{H@-qCffxWnvZ9_<8zrpw?Z^lZ^-=`Zfygn z%fuANNuR0hVIX?%k)B0o17~o}TgWJJN3Qn03gJDAGH;*s85fA)C{beh++iF^zUh%V zv59fsPIrz!mt$@!XrAtRNt>dZu7zKmw{aZu;irbajkP-Q+fk6ICyA|@qkEW+ytC*q z3)+0DHBtH3Mi5mQDF+FIAWfu?OXPn?VS-z68@8nq8T4yjsig;-x45GoWA(?0`}_3e zOXvAEPYpEf@4pnR(YLhS$z4TTUKmUKD`R7UyoWgbMrk_D;Di% z+LoSWBq~YNgri{^y+GLxo+|`b!Vyy{QJbH}|3nb%Dk*^k4-s-gQP$aKpa7F*NwS7c zgX1qDWf#2iY2*YQsl67Wt&chLU`V1z66Heeyxzvw zdt5n$-C>cDaG0W(7VSy}fR2tYPmhU{l38L2&7jBpr-FVHLBGJP_c-gxID_D+=h2o! zs0&d}i;Y`Nq6%;S7jyU2U0L|I3pz=~wr$%%gV#=7R3up8DZm)HtIcqP$Gu6c5A+GGS-Br7mwdjU8dp@{$EPxN;R7A4! zt4TbWh>6t{CF&N1Dh}1ZWGP;i;>=j0p#(2lqi$74H0jhCv~7KQ0*~e)sDqd}A5s^$ zlh`oVq`)+_k80p`32#o3)c^$Nt3R$i!7oaxiI3XMv;ryLajRk)YQ>Ldy@YM zc0k1B3mt8LMhxJUN4GjarB7rG$mx!JwSWjF zYAO~3h@PGGl~;BTPIh)3T>%W^BhUXU4l@F{zUK9P1n>=>xP0qAzz)zF*=2A@5&kPF zLO}Z7a2xEN3^nUX031L7xOMJ)t8!O7bt4(@@7H@@`G|G!j9&3TJGpDyZ_N70Cf+=u zvp?S#)wrDIKkwP=61dG|x22OuIs?OdT~tSAQoV_icN|0@rpS-|d%QEVC1U4_(kyeI@H^|x&1_@>t}C7p;b zWowvwXR>UYespdLW22C*BG8q)X%TYjtDhJlkTA0p;h|aVl5tSAxkW3su|=q3aj@1v za$z3GDM;gHi1?q(2qtxqd35xWpS5EewBiLm|5xEe34&%DroU1TOZfHq!aR(#|YY*0pWc=RVu zxO7dL!$%itO6u-(nguILyi61ep(df7ONf!pXd?cD(_%{dc<c9kO3I`_EXC=ns@qBrG-WZh9@$`k;cpXN3 zw|WPL5n|8GTPjIp;%AO^=lw^63{=`YW3M5?zw58r#XrW1f{QoeW86pf#w(5-IcZ%TFeE>8+Ezq z!JY=hZDA_d)Ks zBq?3=s3(TvE-lE;?s%^+?ov9&6||`2%5a=X_QEnBIA&veU|8MEKb(eNr6W1`TG;2ATde`p_s=(EqY9g~`Py-XU$1 zGGogOl`B0Bx6vn8(da>BzeKi{A*b&jvA|)u%zL2VlG%$Z6WPtV#*>M={3DxJryisu zaQ}V(VkM(+JgEvoDX(0w$+(VMUJp%XJ9$VfPp|fk&0ZNuNWq!PF#}O z078pdAAcRCV;~|q+_&ys9WRo@kXF6QwARUrueJ9Bm9roCfz?;?aM_aka_!uAzJMP@ z{G&XQ={wAhrj8S9n#~Barcpl$teZ1}dUcq)NsP$)-5j<^1~Rj;O4y@>cE^(Nt@_gk z9|GKWF2ti3y>iLm@Y^OJjHnAQCWYWmoPePzxR|6-!npHK&c_~+-Bh^M_+*RxeiD<_ z>1@88XEmWE1IsX0r4>R)B4xVe73PL?1Htaw;YE$;)OBXmExtD4{i-pspE@I3Rh&#Qw_-6k3s_>(g&po+ZEG(fa~O5 zMyVe?PWOnk(m9oEij-Wc!kk6AfWt6Nggek|36s!<#p)vtgvl*r_N@TXUCqqB~nwuS(Fb+{Uu`Z0*1QI89AL-LcB|;W4(1I?` znKx3n@+tD&17iX&68k_vlW-fhJZL~iYL~)kMD$X?Fo1~TWxFOGDg~L}QQ+=Rh3Z2x zCy~97fE+Xl$Rbgh=d4Q>^|Y>FTZt^#nr^rAdP3ziUK#u?spS8_tG)^+BbhGR7{-Z@ z66#>ItIoSb{RFj09UV-nAFO0(PcPKOqyimBCEM5@LUTfdDEOsJN3Mbc0U8BziL=je zRCQ}JT8Hho8OsqBDguk}6MJ$0z{xs-BH`GEI}1mM1Kh0n3I;k>*=Hu;fQ&R(Kb1Qz zGX;V|ly23BZT@|NV&wuJuydfHR3qo9S`=7ZV|q^1#IA`IUqQ{fQ-BgwKGsSu{8l+e z{;+whVw!hNA}3*7U)%C^j#ji-E9M$n5)g3xUdxE<@b!=F<4t&Wa*PWpt?W>gWk zj#uFfZ#Q_1;rWDrP4e|$1_kw?2tXQrdQU&Dw^jCmGgSn~MIi6ud){4}>CvvKQ$d^U zh6eT7foyTGPiaeg)V~#9#A6#743uPoDG3V+CeakB=DAUY&j=KvkSIW>6hLCTMjLLA zGTN5Q+E^J13yebJ;P=6uq2zD$bB?5-wds)$84kh)VFhBbh0eOxLQMiWquwgzxD@GD z5v<(WAN_g@RbWU;#pl~tQbiKEcs)BuW^ot13y#Myae{bhDD^}|Vauk^>^6jz`&v59 zt%54Vl^#e)sfNvT`znEeVYY(eu#K$Y44s&NLEFAIfKZz_XFELXcCILeJ(qv9GC6#)^GJ{Q{CkH=u!gjhM3ODBL!My!=fl?Jt zTF5VVh2ee1vPq%%2(8Nw*2u&N5?>CfU%8Ah!104bmF($_wxe$qTV1M8?xR{^2K3zI z%k%b1FjbdEDBeP+jFK!gk$fY3zR9wuyKH{y7q|$4|6MbeXaXjcv+kCS&D;NWCySh> zsU*XbAAa?*b}^Y&VH!{*2PI7-eH5altB0&sNutuawd|WMClp~w5e45&Il0#Yb$-y= z;2s1zM>f{iC!<2G+Ac2bXN|{%Qny~`D%|Xj2%c1=Dr`mWAf!B^-L1UA?a@(HA-wkn z=e`ZxpGy)|*>z%-$B?eOYR2J_z^2G>!PJXM#}duXOnMp`MPakY@hjwH6U}toTzLW} zQnePrFn*F}CqXaUE5Zw-5jWFBZ69lrp;}cQO4WvU;CQ%6rlN(*k8+jdBBPg&}{i{$=vzQfIEJP@k{Wkv(X5I4O7H2VMyNiZbU?9 zLUfRc4wLp9A^zva(oJG}Jkb@Jb`yRCM$sLv(H|NVLm=2>KYJxVcw94E(z4kx%3>lu zwOmQ>(yCaL^LB~9D&GGhb16z^P-+|MFj)hh6M38sbR#Hnq}Dt z#8^{>L35Zy&_7$h%4S2(*+ow5x`Gb7^vHLVpVkLp%VVGoI!_1!WJz;OX? z>qI+Ewip*8jB*j`H){V_>*ONz>QT*kWQ$h*?UgXZj*f09HE9*o@SEofBxFI{I7Za8 zhPkHHEVe#-SWET4K)GxHA^CMeb+kdcoC(1c;nthFg{{JX(+{0dF0QQD3%g!YFdkWU zcIsdl8z-l6gPa)f4o=F&Hbra~xR12X7g~aAt~j~CBU_{&zL@>TTz0-%IDW7}Eks+5 zI_pQ&d3E$>cWfhcy0hidz9X@+7G7cg&P%EXo*8Wg*%jENon&S7%Ly>B!KWFM_wLb? zgTnTOg-L&GNRQKPzG|#LQQxLU<5$uiQ3;CniMVWsMho~Qam7RDl_-6Fw0qPmbmU?g z4%wbkpO#o+O++I=>0n8If<1_q?7)((WO1k>f*V+@?wza9L7*12F<}%SDb^0UA?7%4 zA|wyjkeC7nrVGLos>_RbV zMnBEPL@f+O+-f~h3jY8a9blfeFbssn^%hpN{=haxp~hUJ_i(!hC~B0Mw75y)M*U+5^8 zTZaO-9N$8F>9K#To-y3|@O?FS?aT%^b9#NJ0xo{AGHwaj6wK(?cfMH$K=04Imj9a; zl4}f7Ai?g`*C$Ra`;moCTWvM^3567Th!A*duHD&urP7ui`nlV@eKZ!v44Ry}iBtu(w^II*_GW;{vYFnU^@f1?>Gs zyt$E6n=;_TwmjUSf3tNPe8QH`!i1|jzRDc8rsh24~! zo?s$rpzSAIaGMWn5(8FIS3aUqdr($E!;T8Wh)VIoY5V3rmg0Mhkc^*B04*=t<%`Fj z8=hYc6<5((AO^9{V^-4yzb!DF(-x~BR$IgSBsQa0!CXg*EG}IjYQfKhrzpiox`v2ZskZ#lnWq*UlGJ&Bt)L9%zt&9H+B$_(qQAs4G%u0YASCDBoH5ew&IODly4;QsV^vsJ~0!R>U~AdR$T zl>OzS{(ak`Pavf#H=3jv1ZaO+vVl$^B_7(BY^b_+w**ya%L6LxQk8%WxzQuX$Iqu2 zVOi|pcE<%U{udTw`1vnFBqGB-_3FKf`L=`-bA}~~7D;UkfO3OGm^cCR~mG~nDwH(8!a&A+$p&_H{IP7jj8J5Nox!dvT zh^GT$F{gI8Y^2M%eICy3QBf`wO4u!RdSqRgaTS7$ur8kUTQW}_)U<#8KT<6FZn*Sp zAFENN90O5K_4off#WL6${6n!2`utEV_EiJD+M_>hn|2Wlz7bjXE6C%_NQR9k(?vHxQr!P>nW|Q+`vS|~zLkM_;lWlpW_3z?)j^Dfd8W)%ks~`t?7^dNzhe^STi(_l zy0+$uLglr;#>Ds&_MFT_eD6@-Jp9ku1r_Z9DX+dcH&-hM{4TB#4ESF?Kb1H+ zcN^ZHr;Y*W3hfbBB==JIk4#R{b+N>z*TBbj6;LVf<8jXq)z-rKuijEdhKa2zRSJ++ z(!GQ(N>p!-sw}E_8yfKkGC)pFZ0%V%fl0F7@~sI=fO_GT^nX|@15}a6;SOR(nQ4e! zrr!R=;=9cT@xM!nV*UrW(tFPz?>&TJ#jLN-Mf@$hRwgXqRaJ>hvfuONEQ7LBtOA3r zd?ju8Z*Idab;=5Q1X&KL4bw}*Q#>ZQt6bZrN+{@b9)Sesr__u{OT`z zeeuIkLS0F=8T>>>r#U|wla*0%#yfx7s<9v2P@U=N5kAA-iLVCwX zY?2(MNb1kAR8gG&-BPJoA`w~oV+I9$M)CGV_)4yn z6QH?F(Jh62QL_kRZDo;T4PyP(z=ACTcJO&QxuXFkrLMg26wctwA6f7pRAux1|23+z zSX57#xXQ$;)Zb3DJ4%UelyKYz)V>DVBEr7)m1S+~pJdXjxo7lz-T!y##-O27i}Tj8 z(p})&_xZc2!y4BdaRgwSk^@3wim!{G5bt%he0JD_Pw_h|i}b2#frW>H!txI(sl5~u zZa`y!_-Cef&yEBC*i9}pP+tap^ZJ=o#);E9u{{+kbg+aXyYPvEYuR4{o^5(OYiumKW`Sls z!4|d#!YK!=OIqYpe^vyC4!M%V6OlkF?4<+s`lI5zHLa^}K>+q%W#ScYX#qGw zaZ_|LE{)BcTdh-xwv#)f+0RJTbgDWfCww*)7?*nW`u)bmUo};;Sw|N}e=%AzRd`*h z*Di?=Rr0IjqMvg$R14a>hImV(;D3KRO9?)J&u;GYJmJ!&Q0RV_Az!b6y*`=jYp;E) z-iaa62Wh8!GCB2q-3%NISbUEZd?$R}*y9tj6=b9u^4uMpc`91~xKcO-92GW;vQxB{ z+u#rM1S&xrrW1`>bFuI^gLacniz~QN_NF}#n9!@0k*D(AcF#a- z%$&2!9_aapoZycmW4ccoULD;}yy0+7|LjZG>%d>~SWMJU3^VR>sp2!13y3tex=ZB300oL zDLXB~vP&AY z5JqA4UEPidlHVh)xP*BIO;Sv&7%Mj>1~gNuk-~m%B(Qcuk+3&!G0@qn9avE9bn)@f zVQ_nX)w{(V--W-5)GBdNA0y$!PC33klV)416c8?U^V@YSU|>@>RJ?1+wK-7yscT=K z3YwWv2IO;egPe$=IgrMt@PK$z{7d}QKRB|cXYQRM?*=2#_AVMBTVTYG&2AfKDarvq zJiWC8%n})q+Qq0VI#Lz%&DgW@oZwd1=T4|H-Cz!=vmM7vckVSdzbQHH#Y-3Gi`t%J za91{Y0dxFTzfJHcs!?CCasV%z(0_)uJmeV$p&>9w+Qz;iIb%e`z_5+TofXnm_&odi zXaJuH6K!RCUKG-BN^jI*9|a)r0qBrW%;Z0Fbb3^|0C>+a+55h~#~+-heObkW600;V zmI9F`!0i$mZ4NfZX`keGtZGyGu{q|zD**3cm_n1-hB?UDCbW35(GF4FPFL>uj>}7Xl>5b$Pd@rJht|lGIqx1AZaF8r{NyKc0@w$JTu8?bsHi+qJlf8b7)fJb5 zY?vx-JK$RBfFiE|wOm+hnJ)g2ve9pJlZ2ew2x>LH-cHT0M~wnnGV*@7j0WC_TiD`% z^lC--bMYClp&CVP*cy{xTN)ETdm1?5q`a1GkXyflm1DT|+lUY0!)u#a4cdYOknMf& zICpu@I;D$>d>88?#0M)|tqvj;dYOef=d>-dYMysfM1J4qB8O$`ly3i%$c&P?2UAVc z6MjHVkM6g_w@JsM&#xxcc!Jff3DG6f6mUW6jjy-QuV>y^W4zeE+HGXKHIdovzC4iP zRKM-*r(8!H?KbSrFqkuo)l16ypjB{0twXuX0&-xEMc)rP>!z0iS$7?6-b_B4;8N)K zA}$#1ZF48P`ZNoR>v+JEB57n%XKs+r_O}UDrcuDtA^Csb)KA_(Bl%axEZTmZks(So zY2Bnxp({Dos5>f8UeuR1%WA7vE(%Sw($uJV88SE@23TZie*UtN|4h9v!~20}yr_yS zs=7uf$%qTo>3Nk*dBbQ3t>V#?fN^5UMa$!TrcH6m<)>QC>1G}vMJePv1g#r5kL-u} zjM;Srv>&2J?$vbHqo%}Wk79jlRx0+*rA<9B(K9=D#)FB^S!&*@D^J1oE&L(R%Isut zO7HL|sz!5clbUWmn>Sdbri2>LUyi4r+L6@9N0m`Xegtu#P=hLDVXoRf1NnRfpE#*K zWTrZ-Slh2x19^}KicBYZOzAIi%$O=OhS~yhmMnMZ0Li@~8`2J$DzxxIVE5;@$ECNA z2?E_7cjve3!>fu855d;Qm&*rToiA@2ro)ukeeo*k3a7iP{S)13S>4~h&1}#I0&(oU zn`HEyqfwBcz2sTkiH9&q@e{50tBm@rgf>#|8!G(Z+sf@_unzVN&uXgJvT$P7-s)M# zQbvPcs^;^x3p_viOy2RH7-#ZZxNa zC3vEmTs$C-oAkYEm9k`AP-o9O7q12EVatX{PJZ>UCy${J4H%{#sehtK6`j}EL(%2; zo$*Za4qF=YSH$WUaFQ4oQ@)9CY@>h3z#Dn(z~p};vZJk3Ks8cNXyCUmBsd$)2xD7< z)(fFI6RL8HD3KreShM3Yonv;HE=Zrm6p83JTCzc!S=Lfo$Ss?wUf!@#ud5_gXm7@! zgs&`tpbaeNnU|5-lWavVdrbnhtU(1wLj5WlGm%|?K zlbsOFPu+<#=PSp%ijK}@LY>eN%U{?&zu;@&NV(9pEKx&%+aZj)UsC`{{-k*3FOAYhcyc zLfvacm3fU%Ax=o{N-Gz^Hne@_}# z#TBc(FqAH9_IK#b?Hyzotc%Vbm+`GoolDTR_s?Ek(YvTrPQ{02Mcl5)N!?&j&9u%G z0K5A}R10AqpGvu|swlHMOc#Z2Ro{2@e3q=nlyk}=YLNn-Vk%zvqv2x^vJZPk-0)oh z1-L=gK5&KTh1K%7v}Q98?&F+puPzl{q7aE%0b8_?Bd;{IS;=8>Ri7oJTjG4aI-A*O zhjoJRUl0TVf4L}`i-00wwlSC%Q5zayTAlBfM5jcw?bx74MbV64_LqIEg)5@7({PId zPYM@^lJtz?-E?5)p$>4cgH?Ftf;X(|E#PEbOY>V<<=Uy3Pq2l_lzs*XH9KuCD7N-n zIaszl-+ji!ZWd>|NCY6UJCF^v^#NYwqsFVLy-xoV#9GkZO)Ch#Y!iR3&=Z@j?Chjz{$`8TDj{-6ZqZaSP{a16z?8z^I>z!$g^F3L-lQI27WuV+ z>YtZHNaX^PK>zUgNml$f=-`z-|2|LM*fS!Mmh6QQC_yAarI^1)b?OQ zP2a_~l9QJE%^Uy)@tkMxz^m^~sp<(pO!hfP%-sj>yC*{1cYx2Y>E7{%g8FZX)yvH? z?Kh@6x(4nXBXQOXN-?Q!0toG{lz=IUM_}8Yk&hv((Ba_B6{+%4*9r1LxcaOVEt-+GlX`Y{%;G@aV87 z`984X^6&tTtYUlSSr6L?YjnGjqf-h$2_|Q78DKH54DJ#E19vt-GryL<&Lcg;O~;;m zzt0bkHdMD+TV)9zRsh@bzCG!HhM(ASBY=OrL0)=Ql~qcGfl5T5ZjWMwI{M$_5?XVICe0IOmEyx_^)$|g`sEj1u; zyO^*}0*W3Y+{JuUGF{rj?RFIEZPu>R5*Iu>yCao-LnAuF=h#qY?& zL(yO65a4k_@hW`%S;C@U;h9OqJFSdk_0r&0Gxk1l6v{$|yk5h60}`S(0ac`vr81!O z7twC0cbZah(*|QUL%sH6JMJI9$)}FoG{8;m>{Y3HC$tW)s`}MKV1Rh_ayRugv z-q|QgZ$vyB{fz}{umPzfq!{n_)j)D;ge?-srnPSzWf?=Sk-FGY}4gnnocg7QI%6B*&=5O z)u&ws)s5(fZDOr9M6@08G>rwpzWY*r50W-3v@@?-^mGZ}oT+xlw*5`)4dF55>;tPG zc_t$2b#oXXy|cmdDLc|OYL-4Wqp$VRDan_4=XXj1$ECOe1D!h_XAE*3`EzNJLzZ@G z;ag+V|4S8Mx725egCPB9*dV^1sz)CiZ;H18fWNoCfgVOf9;N$C@Dkp#;Vvf~FGAM7H|9t$+vyI5dk(U%f^6wOz>u8aQ(4(tNe zz4dk8dB~uhBce&uIEaH}Dlq%&!0ONA)*x8A0+mXp8w3pG;580^EGEwuS&U2Q? zvy%7>n$d!Rvy57ci|L1$cajt%-13hNqhiO&g;4dFFo7QHyA;C1)r@{Y5xXXq#EM|7 z_tK%bp*Rt(v0E)`Lhq`@9W2=xLv{oiceh-LOJQkI-HFFCO@xg)0M<|8faG+87zeBm>jyUqT6(uwnRB=1!<2g|+jhu#lMf*(_3`OUahs+)!qA$4^8e1s{BqY1 z`fC)fPaf_gR3KmX-t-v^F^Cw1Z}>BkZV%r!aVA70mDO5KXkI)iJPXuf#SIdppgr!o zITTGRWZR3(qEv+- zBbZgG$(|6XlUF1A5L9Ftpv6Un8e4onztG^zk{`5w8T0v(i&}dPfqV&7b7#>6pXVqG z<+YXiloeZXDRF#QXbbW<+)XsKhdlgs#k|vZl_2Xl?LPE@$A}~y)J_lDW?iM^L1)j> zJ;qn1<OG9g+-*ND=#P_vf{BluP?SX^R3JPS!15Pu1`~fy ziiRXcK4ys5nj+n{G+?W&CFb)Iy}=BQ=!iMe$C)AS;*N3+&-NN?6&yYgho9zKmg4|( zH%1)TLoW%vELgozOPAQAgbhOeO|VcoyOgmUr$e9I6QyI>Q{&3^qfjYr6R)v z`#W(@d$R!A48})2oeQR~4kNzxM~S3iMK;al3Md#kJ>YCxMs2(@DaMKYg+z9 z=x6ce6X1Y$qlMOaTJEQ@hvY6*qKK7+*PtAI8z2rYE{cRVQ6_@iw?<7IEa@SQWr2oz zfhIBLcX2Oq6-j8LgQXpMDL{}fA(t3{I<8P-fU0jH7)4ss{Dd1}yAu?Da)ziN)S#`G zXn|V0GIQ0FuBrlZPXdAUD!-SCA*#~<=uwD}?yz>Xtdgi8v;4#;fRDx&wax`|6^TY+ zu~-SDaJhKSgwa}EfhSbC4#J>uQQ#~}JG({mJ9aj+wo*PdrdnB{k3O(lYZ1bL%xTc? z4=rIMtyTk2mClQQHIeUS61ObX?+mqt&S0VV+aQRD@RCWfq7|ux+T^>(ZTJ&Esx0xj z{&@yD=J5tx_&2vr_g7ptePB``UKIsAH7IfcTWKhSWs#L$!W*QhwsFOqgauc;ptl=A zw0~_s?2HFKZ-7;DEy$-ytsryIZIPGKTxn>jlRO)24ro!(dL-!@v3`|6xMRaEJ=hIf z1`-}PhFRNe?u^RD{Gr}8+qrVe&OU9Iu}}k>F*eRyRW%R}|m=&kHc4>i*fiyM|gqs_p;5sNk!p## zV&j55RBm@e)-`Y7Vl_+wE6#jZacoo z?bP9Osr;*g76(7Eq0nX%_+ryNuQZk*6X9rvNif`K;Z`Z5iY9eXZQ?ByM1M6V4G_;Z z!=aJQL#{8^eF2BX9&KRyYps=Y)F2rq89^Yc`(`jQwyIf)TM8yG12@)|7322yR!vq= zlcPPN{Z8ATFoN{?@c2fB0rn|Y%bXJ^Gcj`%TW`v1U0gCU3H3hYVa?8`$b626`?p?< zJgPfA1wAHaHh#-IfZ_18X9(v0)T!T$J_bVI)>0I@j$HuPOH8HC9rG($G`FY@P0W!B*nK1e!3bl*486|=js(oNx#Zsfr z5qh$kl5^!cZ1D;oZkCP|!bl-bU|5iZCk7D)D+QrQ852S;Y=oxS2PQk%KKaxE1VT}9 zIBW8sB3b5QVdUTixi;-YVeZI5=;xKiM!X3y%@p5?z44gZMx@R7dxWOMrsv8qnpxc1 zj5IuO@ZE)~)yNXm!_s!0*)q^1{$z0JHFfKmn)>U;*^{J7<#pdu&S<(d?sckH>pf~v zoJ1YhkTGxHjuc3}2c_k}`$lP_ImG(Si1D`hePqQpv}kB1hYo@0U`3rP zN|s{?^xgaV@ZEMIEul40$G&0(xhHqp`ow@LFq;&p`e04INsGXvBRWjQVtw4OYCq9t z;I(mZo`QwZO=JxGx44qOu;;ww(9N%!klg@Z9fvbmvtt?RC9B@;)@2MHMy$hWD@(1 zB{>u=hIY11*H6(op8$d*e>oEB^{hW(vMktlb4&cwW#riAYeMMr#1%d--M_L|90U5_Oax?R#Sz}}jWX`7OwVSW zJ5cC(Z1m=X$NJFrTFtq*SFBtn7JzUMDWIISP`n14+8wkO)|70h747wGP0+kSxbx3{xip|_5FIGLeU~c zNoMB4Uswe_?DD<*hl}P`#Yd0VBLi=VZ-82&p;U?H8)Ed*^{eLPW3yIeACL^xEJ5(_ z=IeW-2Pho}ydXU1y#8l1IV_1tq54n`j(_S7|BhPmk2q2Za$T5|8?=}k27{>uO7N#j z%%z&g2~oIUk4v-go^@cvyN5(DLm!_wx7*112li(IRdk>~A5&6S#G*N;iz{|9m8|+K zkP8Y21!=)SfFnx$p?u-P&%3+pU`>4-G?HZ`{Ph*CLx0W>mos|5=0pnrN@oIP0PY`n zA-^*&>|zj7ax^^rGoESSC-E#_+;QD;-*R{=bO|zb%ME7|W1DMqLLpHoVkZ(BNI|E> zrnMEOEUD9Ol?QGDU2WpnL$qcm?ILF`*YlebB!ryS&(fnwnLJ${u*|GrZWqzN z>fSrYxVy7(G95kUe7N}gzt;=5+9iF}&wlnr=&UyVqxv0SFqWIjhhBw9jln4)m2~y5 zjzH4>mT0YY6|ETlfBt5RwkjgDqdPt>>Mx;ibQk>b@!t8sFNK~L7XLK=I`n%aBUHdV z_4M>^-6nOy^{PtKr(_!BC8s)RUP!Y0nN}$Z0}B2zBNvt#erE-cc29dQxVD06D73+v=$n{nz?h(5MTrXfw1x#kTT?BMFLF9waVi+e!kJya)vJNG z`os``lmpF9Na5Q-So7^5sFTwfZ0vX04AR`IY#{5{>%BLpxi?4kg%P9Qz1DOp^f$RO z11358fl*JJHF?s>PDPdNY<#*?oc$lJ!9XakNnmEb2QkBrHtGv#FL+J-kfrVZdp!tV zJ<1>m14-%-q*qW82X*WNcu>@(r{{LJD^L#@>5B34irS;KtQg0xg3eQ7cOPoTbSEJ? z{_=}3A9irgE{W_UlgU5?4jzP7ULxw+H@9b&1QYsTJ_Uva(r@)TnxeJEt9NmDN#k-K z*)`PW{O@g7+dpoh7IR1A7r1|fX}TW!i6Z{=A5#$R5LOo^#)bpZSk`)vPbMUxK8}ta z0Oy~!eI=L3ZD(YYZw=F1#pPxY!R$4a0{4fZ1f7n``$I5_LW-^QHA*_Ya}h1^WzNM7 zlAx;oa;S2!I(@YFXUhouZWxTw7z})X+dv4dPde7D2n>en<}o`_Bcfdi-}Ejw;5K&- zNE3YFhf_-2o}zH(nBqw~bUKZ{hxPH2J(pE5QP_XSU1FZ!jRW-)H2119c}y z;?RkC;%OHu5$_3lCIvw}jWOQ^E!{BRr1P|adDJ}~(aF{fxOq{jxo~9~OVKjb(M+Cw zy(=!fb!3fgFv7oH`CZEXi()s6g=-X`L0|>#M3I(gS}~nXtnr1Le7-iqw82IQ98#Yx z`E@^}40}pTAetvDBRg9FLBbJqkeavkb$HnX4=V*}G|XU8y!S70t2(WYCrN^)C=Y?Q zab4Gu%rr1_kY#LBQNoOST7ETThvL1&;4OZy!v=kfgc)dGry-37;ttng#w0El;J#v| zLJg;`aWVACO5<8nucUwCt4vi9QJvHnAcTZ_xfr$XZV?B5`#P5dxVUenfl0OHOk&vU z2A4JEFDW5JOyNc%1}PoN$9Ednrrdlg-2 zFa-($4z+KQc(6XOm?p3SH=IkLge!}$S*0?Le!G>AMu{msx^QzyBj>G^squ^5|GTBs z?h)&9*=U!fsLW}Bycmg+^mDM0S2phgmv#>{Bk^-p?Ji@J7ZQCB9Ktt21=SA>DQX6a zEIOPxK(Eap&6wDCwg*+^H97%0K?dM?67DYu=is@4Olt}z77QR5;o*}g? z(i>}K^KOI@ZEdID>K=vt&M6=EFS>WRvS7P`iio7}@SICu_`<)13sKQeoO5&ixU@V^`grxKf z6+2UU;8i$Kz)b^Kf&-Z4S2o5j^4`)(NTNnOkV@!^D{h;{P5_BIx0wY9-37xEu&V(4 zbI-*Sxy1&hj<2iNV#X)D6bQ{3Ad<57#>WS6-ao--Jn>k^_L4j4YIEq>dB@^UMGk%NoYDl4KN>nVVU^Ic~SfV(mMt$qq zbhImZPUw=6U;bVJ90iF$Nd~8oMN0d_ECc5D1lhhX`t)z801E}YINZ~4h`H_lv)KYQ zL=Z|`({cq(GfE3sHs6%(S)OdU1G5{xl5ub{4FOtuMj9h;pKTonr!Cx%VeV6E2S=FEmqIR0} z;us2}1MV$`o?}j#Le6l`M=p(>y+TK(RGXPm{oBo$sbH1u|Hgbb6>|993MR zBO{7@(q`j-PeOl|NOnK(+kG$^{nVEeMnzvCp7<8`qm72(2?W2Q2kuF_Mb-<4h%sqS zyxcz8sMN%W9ND9QX9_C;pbG)W8a^`WSotxn!hX3j2QU?lX2DI6t$fDW*iae!*kuL9 zMq2Zz2HnFm#^<5mJsy}a1 z&qNSpEv7mB+G>1s+J&Kefj6`s-2r7n3D%UO$qo#@Z`VS}U?(d+a#hRwe^bYyO)PZWG!!24L;Pjd|o!x;OV=SrX3D7KvK0 zY0@g*M7o>SfNOy{5JFcY@vc2Ym>&*P~05X6Ux$9Da6D(&YuH4^4*kxZ(>OIz}PiKmE*lq6RF`&)H( z-o0~4Uf=Il;ogGtQs%%U%az$mEuw4a(KU2(bJcuohy&qKQR=iP@4JYs$<97c?f8{u zJUE`1PvxTiNY*z?(^)22nt=o(4A^OAEKIH2a`WD?rhy)L zdGkD9;X9mizjptAQBNm2$bQJ3(FX0tOakE_SqIs|1R zbAP5|V_E9L#5FjrY!O)zsY-YnY!{q1_Ajz%b)iGHY?b@+9147dtdVh>(Dal2i3%O)vqFdrlK`@2D)bj_Q5?jyQ!=OBzW_#i`7sk+3~>b zl5ArBnr$rz0j}TjAz#c7k!T}=DM?x*k41lc253+ApjRe1;o<*u^YWPnmbn|4qj|hg zH8-)hi~V1`ol|h6Z5XCwXJXs7olI=ob}|!lV%x^Vw$Pr4!USV2GEL0Mvx$p55>fcM3I<>kMPAgDt(*`Y{JTk(-qy?#Fns4r!=-+Joc z_*#mUnXNTb!^qNNP43xY7csgt@cDullMz_{D86rB4iw@T1L!i5##`W08~(B_leyoIS)2@*y|Ep6~SalYZ3 zCRWjZuk{a@0aRng;=i+L)0h@D#g9YlsTo(j`EvB%Yra;enfN*_7{6}1=2zuCK+;~g z)Rma+o`;#ov}omaqQ9#}*B;;gt=~mmjcGcJwQeU-6)WKm{wOj=bRS<|Wt0r_uo#;s zJ(5u6Mug#H5h{^DOmZjsQA*U>Dvu>U3rpx&u^^ikyhkuC<8kYjS${V@m#@);Gr^CnY>2`{VbEyt@NsPzMpSkP zf7kY{%QyvxP3J2KsO&HHk;NH3P#Q%f`XI2VuHyLWV) zSt8&&*cRp7CGREH$bs9>ADWR{rJ0atQep>s~ z+xL0d+>Pv!+C-&MNU5fPr}{9rUAUUx|GfzJtd^~LKHzxgZ*VCV@mH(~kP1c@4ea}v z_`aseyrTbTTQWZpitG=oYJB+_?)$K5aQhWrthk)N)?*G(=T75W-+tycV3PAA3*c(eegFP zwtm*UaMp^QIrGcR>ZuQVF6fzM2F9lIr|zmh79YrjA@H-id*ILF7hqY)=f9($%t0GL zy=0CO{^Z_CzkKmGU-4O-mH1pE{Z9lzh7q8s=AFYhz!qjcLCfW0dm^u^ex?O~@9eMr z*qZrx?hf@MfFxF(odeOUF$MOpv!EWeeuRoP=3`6D)HT4qH$e*Aa?e9k_Ee41DU8c+8;pLw5;4luBcJJInyLjQttPYVUuJnbube~K0 zhnpv^_0Kw(RoZysTzi!=zt0aGsn+6XGoF! z?C;b&^a>2~L(hNP+Q{_XASJ{{f_LD484WTZ+c z9~VA~TblD6^S`r1r+47X%HYQ%OLf1+4mW6BnodfK7F#ngQPFu5hi zw*0itCb;e*9fBq3qbujl&uWHs^m{Q2~3H8k3B zef6H}c%iXhb$#Tx(`oBm`vBP9s@V;!-(CkUUHfe)*6B>AYj|M30Gmc26~9sdk$%Ox z`Z0#HyA4odb3f8RFyq*8+hLLPGA;F9*ejrNd#1wVLL@#~cZ#-;8f)&i`H^PCKMcS0 zzZqGfL!d|YvF#0z>fd&+fv%}&^T-4uNg*#^juSRD+L!cjP(@fk$MKtD;O-sRZvVT` z;>)!6yQJc)AJK97x@-l_mwiZG_fgt+F|MkwXGr2v)z+*kcKM?Pla`51sj-if=s+3w z=DOn;XKQiAIVguoLp|)vt~J)M5>C&iBrNB|hzo(`}=}T0MMp_@Mfu zwOAcr@~5Tk<}!r@unVlh`ZUQuMm!1g!jn9XsMAVAAbY9C-o7}oaFYAn*$Er(K;F8$ z!SPlK2L?VxjW!(nxq%CFX(LJGCaRd0?6hJ>p@FIU{*?&xH>I3uy7g+?b43zINZgw~ zhpG~!i6udh0ice1cbiokIO}pA`TC~$Vk3hku2FJ=M4ZljJH<7xbr(O820W4kZ(IFe zYD+qkx`j65{&=N&T+skCMwZTRfZW^_M}B7n_yVKfSGoyA$>)Qw?{)M49^&UXs@gbE zPp7AsOTqIsVAh626M3B!5MX!@Tv-?Li3C1UPs=-IU-4b` zW4Lygy#fHYS-${b7_W+7k4;}hhvjE}gsH%5=?*#uCtKsD_ns})B|2Eo3)O!nJhD1S z<0>cFL)s^zkEPMIii-2jieGUO3l#don72TK+B4wqIv_gm{dcZgv!U&lWY_$6aDQ$4 z#5G|Nv4%XCZjSIYPmfB8Qxr+QRN-!JjTuXxq#3D$!CO`R!I34vvQ#?E|9k|jUd=t5 zCjIhj)0_l)|N8Y|2J}rX4H?k9ES3K)#*V0E_esd--)zs$QReZ70m?yimR6SO5`Fwq z$#y02Yx9Z9zMg#-S|5XL+KGChF+s~AOnQv%>fTCtD=)V%E4suqH75n^Dn+1?=p?jS zDTL1k+~ZD^jN=4MCgSx?Zk7LW`ri_52tfY%8v4#Jk_rI1-b?zmEmU*n7wZ+K_)0{0 zXl(#{CbnGyTj7AN<9^dj!0bohE4B-%zWu>ou+$CB*}UE&I$7S}QRx*dP`^7}YC7zA zs2u_HM#GdJ_&uTeod!f*_=SC*eNK0OZmE97NqTaQ;7qcn{tGnA-q}c8hm9I#Mj{*8 zO;rMB#%A1PBZ7JEiRJpUOAQ8#qu@yTgr}YLdvwXFV9wHa86&%{dMWPrt`f80fsYgG zN{>nGQstOlA*~a>^2AyM4=bq#ilL?^Qb%VTdoVFEGO^~eevkI7Fp_qr_&x zSY<_fz}e@Tke;?$K~jCs;sibwwEw}%rw+2QFn)=;xdxKoGaQ6+2pUBXISYP8SU?QM zAo*xvIHXzf9nIhR2v-ZJB`mj-O# zPRd!r@Ix>5;;S|8$@6iN1bC+oAx#ekQeHjO^A&pT2O%QB;;De`$;BN~1r4&e(BwOU zMYh857ZlOz)rj~^X4>U(ENbS0*#NKmvxB|G!_K7ET0zd(Hsg#H@Lt^ro-N;(wr1tG zZu$SlT&T?S_U=8ux+*qMMZYM1q+z=dav|Y`fwSlS2Y3aJ{7md%Fi8xaHqG~@g?JXt zv|bfX6)S)Q(T2b{SWgy@hC536=eVO44yx*egHOJn9LUd(1+5gRPW3f#sz`NwE%d#X zN3f=9r>Td)nyQO%1LZkA+OvG`6?@NWnV6;=G2}5pF3EuUQl9k2NUS!1L?>t-sFxCY z*g=;2b|KT!t2#Q)`fbCa6$r5F8$*Ec;A@6m`fW5gaysBAZP)JrlJri+nb=>dV`$9p zAR>WnNppQf8a zt=%qGTQs8YM7C-u5}p<4)7IB`6Rl7fr)#kM*4{1aAv4FHiUC1Z%u#tPhj6w)M){Fy zBwBeJ851)nShK)-ppGwmWj^dVeQ2PAV3by;bwJb8PHXwz9%$~SKDz4+iPu0VV{LuM zZvMvtJYNmE z;~NzXoI6Fcbnt|Uk>n|1!gzR!{cJgFyB2k6e3t2vWXgscH8SSRzD%fkQVV*JKL29j zp-%D%7-prSQ4o}b8*=EFm6V*&N0}h}5U^fRtyc2GAi01>tG0gv#qC$jd!h`sy<=)L zNhKmACloYkrEMsRrR?h(CypuGO~NPKLZM!q9wIK`Sx;*j;)gBF?XnLupAFG8=}5LJ z^1ESqQwk#tF*S|SWB0daF>^(>9P67iwG;z;^?Z|s^m3%H-=jJ4D?3y#l<0dc!_~D%lZWsXD6sC51)BFvXT&Vor0Mb_b+uNS_Z`~P)@sST$G;kNJRHPK4?_?`j+BJ*_ zV-4MoG|%DlCdeK1rpM#xnQ;=f?4Uf+(Z_Tkpzl&6Lu zT*soFVkd`Vx|G|zD*fus=TrmNQqJ$geljEUOf#DUarv&%LLn61qT01#suEFnKOQui zoS(SV642}LnJVweTj^4W0TSnSP#UN>j&0Hx`6r4;NO+Pwjam zNl!Hl#kLAo(fVJ@M3}75D3i0u@P&#|Z2Ne$CA&?2$Yw#i5dXEymEeV3A4D*jux^ZN>IX!g}gnToCc`o>Qx|Z^5?0mXW1F-8<&)0cE5xi zhZsfsX@6Ov!jOGv;*@%)=!Esc4jzz}n3gQ>=@Sr1p$~?SB#LR-?~JaMl2UespoLoP ztm@x<{HS5HOjqS6JJ60B!%zn#d_t;t_In$CSN!_Ep4!Ey$*2}QOsTAu*MRT8qQLze z-B#S{`K|mntsdlq(kvitBF<`T1W{g;Cj5jv{Y7(Z^4WrubdpZS$|63%|+sW2d+wp5xn9*4N+t1q32&ZOz*i zdX+r~Qn@81*YqY;BeG)^Y~EpFE4E^8>JW+Q1y|0I2<0z>x>H113JdiyeNxu;Dacj0 zZ))rIW6l3>E3t;c05`HeXpG$ey_QdY{Fi{`z$&rOT{82T;s{5Uw1~P75I6cVI^kCldNWNKvpqZsTw&HvXsfvjbjzR_j}o5hve;#= z=2}8%EiEksFUr&ge>+I4wxmf(c4=~d8xL$TcW=!Y@bmK?V@u&7Sgq5#VR!zd(k8^c zvD*?){C)qvha#*)#!Rl?*5V2L$uLsMbno>gsttizJ+0q=unY_~rgi5~rxWt`9<62! z4eO|Axssq(##DNBE6$lPmcoKQ!99`X1o1zlus>rPMZz=FK|k!-Ej@PgzrpYgRicHb zig6V2k0VQa0SuxGW^-N%?PfVftyQKsc5~3E?@2{}l@eL~$!=c*l;Aj=F1MF;a3Dee z_aY?&4cSAwq}CDidMu3qF~NPSHT;#CmL0~4`1y+2Ig6<7Ts+EdDZ6oiYX8R8?bkD> zkm*|23+CTG4RoviCVQ=h#q@e;-hu^)GBb`b^&;D>kW(8t6VW3Pr4?{G;lFvYI>&22 zWW@S?tlocG!yh>!@F%>(E8=ahF4d;Be0Iww&16C0S=2QQ2;n3RGCEiRico&d2bZ|e zcFFE4t|VvFYvT_ggjk}b(vm_)(^RdrI=Sd2l)#E!#HjM3+f*#(;O}bP!X(%|aNvms zw3nshGZZ<(1(}F&Ltx*?$zm2-T92tVW&9&I|J|!XEPWtwr?^g_58J0RiVMfwHy1u^ ziUNhuin0UfoXLPO&7)y}eotmLY%l~NlCjBZGU1d$j`WCj`4gE#JF54zOu-bx>}aS( zns<#q{40_2jid6~4fQR_Nk}I$K}))AB$6rQOIN=GH~#uLt|tuBFiwgwKt-2&VaYog zca-Va(Qf^`Gy|3cwn^~Bty$^#&zD`>&}I&NNcVIeg{jOppR$=l<>2ExIU3cbSoqz| zsHFe~_2&!kpr84g25G0x;^|k2TzfSM_<(al(GkC@vsJR(fm?sFC>U5SYeIy6(c8Kb zGb)Q0I2!)3@&R&VY_b@{OM)t;GU$E!<312C@40GtNuKeOeEU-l`(!Dkt_ijOaQe6d*L5V%B%57?N*!F(gj*^Y4MRc z6r#nQEnP9joiS%C;j)5(X~kHcutVS3O~#EluNyfECZIDP&1PwjSERDbFishAD|AGhP^;{53}FXeN_rP21dfkPIrTauifn!44R*kbJO^v_b_ND|Mly0sVZ* zhoUItPs)l$9aC2nl~7IXNz4DlmPmC11iayn<;~i$66ahY%6D25R(g;XnC#JQ|GWSr z<&CNjijm%_Yze6iB#JJr2N8Y7RiN>o>8BB{R)HotPhaj4-%6c?G?8jkj*X z1&8?w>frIcMIAmR5HS2x&-v1%b+*n|E$SYFwo)wFK(#{hO zHGWr0sPHV6Rfi%_{KmzSI+?G>P9rrYtE_CXa(2{Fpnn=4i)1dCeRMY(SJUqkU?5A4 zliHu?i%tPdtn-LSsTHu*gFqq4bHDOQ361sRhRh>#n$fu;D%?ZWFM&p$?_9(GMP-O8 z#ceJQO6$^2?Wi;N`N>LHua+T2P8>WlOC3$Z%IyZ*A)DeYW$2%d?^j z*RQ>Ohw-%@ev?MC@vxoXX)i+KjS8Y;E^572M9T9284IDlsf`P9)lrWb`O_y|^=bp6 z7zK-l%HBE8Mkr-b^dm^S=8AG(LpXxymY>>TH@ZK0Q>HgDc~f}blKO!a|ROmN}6%EZDpZCJhX?$8Ec1q;e(`5zl`eK=xaC5TTPhD+4_MGzs(+_p2 z)k8f&MnSu6>B%JD`D34&HrU(hB?eXZHauAAaHoE44A$-F9`7ezk2t<+_kHP#`L$=) zQ5%=vG??M2-hjPd_bD@cjf-4=58i&~B0AOJ(-Cur@q)O_+_(ss;h<5mj0sGZYimQiuw5*@`41|wkKGAeW#Hx0hb@UQdN~`bty$#VTUHOVU6+Sq z?UZ#vRMUSAjFFg_hAO&`HKYbPu&qwPj-RUZJ7Paeu3rQTU9w9%!mf@f^~sca?YbA; zJ|}AHR-hnSF@&?t`36hoEd*Al;>{Q6Z0+Ilh?4GPQ?qr;JePpc8e@0yR^MAa0QME6 zr~m;r3tLtXg}|BV397nEOvnRw`4V}~Q8&GB#A^KRtBSypa}MO6?CNuJu_-gkR*CUH zyWXg$l>L8Q7;|g+xLKRO5T`kvVkE;cNJj19DAgJrD@?S0DVL)fr^_6lwOsS*k27wH zk*JSM_qy$ht{@QzDPXD@7-1hxUWIIIt{$n=!RZ+nH215YH-->QyW)g5krb+CR8^~0 z%JN>jUeGzSrG@18FlWrh=$9lP{s>l|2OUe+HVz1u9rUOp*KPJHDAT&O@b0u`D^nk2 z#oWJcN?W`ywcM&Gjn?!--y#a3a`@;Y0_S7VhfK?rX3}I-q0py~=ZL96t*PkuoKaYj zF%EZyDCl^^d3}lN=O|;M>`*ZJb%`eAe)iDX&Av*ebZBLq@BgQjJiDgioHnF0YHfQs zSg=9k^3lD&SXu=ZahVmKHd@!Y4^{N0B47-^P50B9)BTZ=Wz~bw3*C-Hs*<>}$&N6% zvVlxu{7LxGng+sl*pq}D631Q!3 zGK$3v*FP6{`zkQiXT5^UARYb>K@&V~0_p96U6) z^e(rU(S*x9jgNi%g99}V`jdkpto>*^ae7MK9hr@p$0yFy1SSXsoYQE#S@MJ!hsCUQO5Mpvyzwn? zT2cS!z|k53q=H`%O9>2|@(tSHK8Vg&2@2I(rS(Z~O;Ws$+lm*AGvy6I!_c$kS9KjM zjrZD2t(0mXA(Q^Ff!~&{Lx&iQtOW|*v<_MEx0PGBQ>*zAt9r-=NcfKGjLTGyju<1$ z1Yx8i&-nEa-p6mRPn!@EuH0-H;dCMcSIqR0&0%-vD9>0WH|pf(CapOi(pgzwkacyT z2-jcp;h~w^tl13 z?xPY6e+s9|6w25J^!4!3iUIn%JeUE!v!SIXPbiO86uNlU3$kJH1Dq1KeCgNJ-F?V3J+} z*)GE-_3BjQnSFPB^Dvrn7wxyEo;Ip>+rt_lBna=dyxTvVz9eEB3&9gzs``9SqP3S^ zS1kC2@2KBFYy@;D(1!YPUYMOB^#?*y)0rX_b(2RCvl#59>DeveGIWX-3ToSuD-7CuhIAsMwe8&f{MPo;1j# z;jpyL4eO|2xF88A6y=w2kQf(Ye?e^n7~nWF_U3LVqgPhk08L%SF_pzW&ub{I+dhJPnwe&A7`i~DQ z{ATeC1MUmqcu+^i1KYu^iVl$Qm>iuGKpoUTWN&5{HNEs$5%)n)Gb;I2->ijWe;_?0 z5&V<@drs%zNMptQz?W=vp4z2_@9@F6{}ZH~UC)wcw})8TJXx9vpJo@>e`lWVaE2uu z$euNV^=m!}Qm9<>Et6xC(lWjS%A&STqQ{h;*`2W+SM)cLbXY)Jn@12(YK!31`dnt}MK)Q&dQZM#H$C-tNv@1vHP`%^L_@R`qX|W&Gs!rnbQZLgGZk~?(&9KTPL@JGaPr!Q6hiNz&M~T~SLaUK08?*0 z#*4AZtq-!76E^h@)+B(niRbYPyM**zH(mZuKW&APfaOVK=`yZ44GW6HO7@|0uqCs- zOCN1YTjsl6 zd?KnpqFBe!vb)}fE+pO3dw&hF-syK{Z1wmB%4rxSHI8CyR>_*_2MVIGJR+Er9x;{f_|f3dSMhFwqn zhlS5|5hXxch&+nn-j0)9r^@ zm&=@b9l>-p1B2ZGtj|!637P+{u+NQ_>r2Ck9>muqqBEkPWr4mf#W7)0F#GjO9|z8&Ct~T&8W>5+TjWpACfwbs za3$)OE=7vX`SZW~yq$e~9>&#f8!7bu{}tp+prro>Gs4a>fR6NmW1DldYY41;!mM4j zD~CES%K)da4{nq&E9DQ6LMmbZ7wD;RDa~ou?c?Y4b*y=9pZS;+D$|U6FrM}X^EsLf zeXz4*3K0m^`+C!&TFEI;&U16DoO zkQ#RNV3xDOmhcZZph`)cU?fEWB}Ka3Fbh#Knpfg_)n)$G)LI$)QU?)i4VOL=H@SHg z<>w01$L+{A73#bKmWU^#;5m!pB$p^h5IcHhXLjdt6xpo1dAZe?pad%0jAi>bzlJ z0h(nmqU^#Cz3;Yr?i~jhzG@F2I-c5J7WEBmk!uI`n?Vr)pkd2UWlNU~ZAQF)vQtTh zM0m2D3TsT)`+U~fPYbJN^+<<08*uRIec!LTncm^L#`45pIV1mU_~rkM;{wM3yq1KoGlC6h@9aL^#wlh=aohWg*mp4S-J?1Qmjk{{)z^Gty@Q6ua5ae zZsZ)?9}it>ixA=+%=q1qgeM1)s>0Q^?f9aI%Cqow`s-pD+AruWKx!8H+o_4P5IbBh zOEjIGWQ;2FdMNM`sK824MjSU2=P>vDRkj)eS8v;-tX#Tr38Rga1uxDbb*>Rn`$>@@ zU?Dp!AP_<~LD}kNOEe9H8MQz299Bb`F4o#2(ZvB01rA>puC~-!N&?tSJsH=jM^PBC zwUL)C1WX1U_cApqiT|SAFp^2q?^Y0nkeU|DCCI)UGcj1KfGiv$Bprxyeu!bGuoPXBw9+BIi<|LCiy!IQLbRH21TZ1n+hs zna)!C-slW7*8WrZI5&5D?4J26FI?>Uy8;o|U1*R4BobWx#v5NbUVyZ2pNQ`uOJmdB zd1b^o4VwVn#urmFZL8m61K}=Pis^|y@^E$jY@j*vCx}O(wH%_~K?xp1g>e(F_~YtO z#+ZkllR7PCR?&tfef&ndsztW**Vo!Tp#SZel0GwZKfTnhSrD@tIPhunG5z(s_W7G1 z*z>ag59wW^R z;X=qRs(xnwk&TGj$WpSfDMK8N3O~d3_Kh$cr5v#=BoikDizJU#?M7}MfA;#=syVD-6o?E>Z)KPnC7{^`HQebG4VIn2()za{Pmh_*EDN30L7 z-iAJ`t==}2e`J;JuB^T?eXlvDl#onAdVb!e{OTe<1RDK0@-XhT!ip;WAc}P-^C7Ie z(lp$V<)^Cx-fL1ZZTH-j^i}wN1Df8FpPx2g(|;VVt$}{YmW8Ps>%SFT6qLFF$^ok% zkLO6dICsMgZ3WCHZo|f$qhY^%=6Ql$|JqOZGH6js&hzi;y>EP%~Tj8%rSD0t$ z+?f-i>BykOu6G`D*DB*<9EiFaIR~hVB>1UdP@Yi)O)Szq&Y*0n~?$giGcg;!73I5Sbs6sF$ZZYw2?8|kS-F{B#_a6 zp1BdsMz}6*>5W5O^_m+=ohBJXdKoRE(iUi$Q1li7lns}!Hz3wO*Fj?Vn`D(%N`nVj zY5h;G3A_brj~{K7^o~VCpRB;g0}VL7SmLA^9}e|F^Xr5iQYNO2f+#fK6y_IE%j^{f zkxW%1;Z%4jX&^jP?(tKc;~ zoHgRBXH0Wt&J}9T`V4DY2#&kO+nh*fdy-h2OM(Iql?NUqoK|V)UX2*oQ@{kXT#ucG z+BsSx5-IqHSHH`s&~a|tV}RdLiu+4mtaHLFds5y&+mmOS5BEOktR${eNevdda0=Nt z#0Hjr-msu7>+Ir}!DcXY&DU;4&@v}sW@z*fxw-0}0Wflu2vqpkZ!cBuem2a{Ll)Y& z2K|P{!XMJFU4Jhxgi%$5->J<_Ocr#FL$=8%Cn7f}+Kp&zM!vtTR~!}j{<-gfeI!Uv z3c8rM?2go^PqSKbkxmIo1`g77Gljn)^bVN@nLv8Xp~wt-Qm&7Fay8}n0|31D%fT|U zJSLc4F|r)?6fsEq5l8!6l^kbTf`h;m6m^jb_cLVY4@`1|w>WwbIRR`hZCwuiG-C)Ok9M_Z5S=&_hl88-HQ4$Wn znrHDq%sR8SUs z@5lS+e^gMRHO7F0Xn;K-zRK#-WiuVfxeNEir^&ro!(dH~V%?pgy(tOYH6vJH>n)>?73NDMK_Jl7n+^Ux%6n*1P7R3TBh|Wn?5n>?pb92pdrAw) zZ)I{_WM{E!gOSnAYt&@Uh}7uov4V)Iat+WEql2v+fiLfL_$my??NDPrAo9^`FOiUM zA$qR7nsAFkmBBxql|NPFSDnfno)$X--46EBY+VtL6TVrHfvaR0w^`rOPkcs9>F`c= z*lz1OWj9(oUciJvjh0het^q~O+A#*4ZM%_;lPT>KM_8GJMQtAI1dcTHKn?KB%L)Yen?LVs$!(Z3Ip>e|Jcv=40Ick5OI=a{Rwb}y|U|68xRPKQQ?A_ zuvLkar*99)=hy7>;_ou8a3}hIR4wHDC#VuDyJU@`*j%!Fv{9?WS62%hylsd^$GpRn zJ6BWR%k)!f=OQ4BZ=Vo{v|H`89<%D8&HZF|=?bN+ynGgw_`8rv2#MdBS*mIjd0?Zu z7+l&zChGz%^%7ljBIxpA${Lo)Mi*MC^wR>($lN#o1No#{T@Y zHrUP1m}^*c3JYc0phYX?i>8df^fmc{wVELfnwBKbqqXuoHNA-c@)&CuG85@n76s}< z`L>rK4a%H^gNA7dTWPmhfa-K#f$NKYtWfvWN*6@(NU=9?v3^5Eayw%y`dk1c+>rVLPXk zZc&w9^TOP32hrj>e>YJ*@Owb4(i=m*Od185!R*SumK7)?DW4YDU<<;^z&2w^H;DJ^ z1fU*U^y$NG*>aNcA~4Q4{T0cpZYdt^+pt}zWCRRpdyNNM*pAVo`-drqk0m`C_q9>| z?P5;5!nMPbtwO#WcKLn#Z{v#cEBJL8enc=JBFAMnE zTxBKD{XmVs>HNpdQh(2qm&rNOOrvop6Y(ohjmBo>ig#)-UW_+FHw_`BFzlKTCD0Zg z=mNFG@a%`kKjn8C@P$>ubYaUQUO7-{^QnA^IiA;=D^RHMG@>OLZ?tghl(A(qx~VtF z7mK2ytmwkT^X>6yWxh=h)u!*@$Ry*fEMV;o>W&6v|7b^lkT-mB8XNtpUrAgFq^}0I zGFFx6@ek0=&{UVBJ)!er9ZDNT{&;%)puq(DkgMk|3{aVzIf`wv75!aaIXdycKg zrjHSpHKmK4@=#LVAY&9h$(~D(O|W0rw`-O65M5&}l$|aWg(^{;nU)EsPp4A5NSsk3 ze4*ZhGS#fK?jB>HuP-~(YQ~YP_2uU6PA80&=7K_jO?hJc$!Mh@9I0YT=#PWmI`hP0 z2RA66IgCUoBadcHA10G$CKo{tSNhkflRCl$GXmqHw%nR86~2WLs4XGLRkSU*A?7GhzBvEvUdxOIQWt)M5zP#~)hB%sYJ6Om zxkT=N12o_bDTru>@C7Fjrf@{v+MkQok|vHT^A{ zNG%2lqX?er6)A;pdF-2xU_>GtARpO0!gU^aj-=A`<}_(xsD%F~;I0noO4bANoiiFN zuLN|0_>nC&k|2(I3eq%W*mifK8!B(Ppx!KC`yp}~Dmq#PxQPvd zP`iYS1(U56qspSN^VdZ@W6#L~%N3|?<(=2J_|$#B>bCm!dtMWLGhnxd-q7FX*l*T- zmD#|Ad%SK)b2*^>6&kY*kk{qAAM3$eoPpj{q(lbaW%h?Wq#Q3Z)T*u4w!c?&ZiDO@ zzx0FR*t=5JvB5lJHwO9*gZ_R6qP4eZ#CeRRqB^L94vyM_v3 z^>%i4R^(gjcKz$ZTr~hyC6=-^`-@4zD z0=c_AP}=LlH=HPGR_U9NWAOXPYFwsZHMWp~htHC!)(16H%-uOko*9Cs-PCjGBd;}^ z>J_sBj@aWgim^V)c|!;_3L@a=UnS~=l!EL?=CvogbCmN{zx5tr&x1Pk?%vLb{tBt*SDRer8KK!gHblEZEP14>U}@Q;T-l(&@1s(xr3+WyXP`xV zs#XPKF-u1FJOd)oP-yiX*NZ>1j+G)TI(?;^r_hR0D6h+*%xH41p@pmB5Hl93!a^=% z@2}RLyH0s2iLuc(Hq&t6q z(|KQn5rG}uYYho$0W^XHojXs4*HiXfm|R|WVu3|ETVLA@4Ej3f$A-(3P-pPaV)rV| z*?z!8nimA6Gd}FQUB~1SA0VrdHg3?d_|G-2I>$ML8;HXgd#oH@=n|r@$}qgV`niB= zw|tyCM5(E#?GI@O4aM$>Hm9ljKCLw9q;}?q)rg&hf>970ya2*%^1UWjz>4AyE*H&w z9Wlw&?E{%&pn@WR-r)FB@+VKs!p%Our>q#=ecQmb^Pt0Fcsn8AK_Nz|`uBx&5Q#US zrtiDiO4Zp_w-cgFkJTVpJpnwdE9VSSsk%o~wEYW*s3VKTA49=<%cef74Qr5K#77xJ zFJz#|3A?dJE1R~mb(2|dnByc3U;o5ocHe3af_%V10*JRCh2DD4ynNo8xUZNCpCbK0 zKdYqN8{Qz=|A)mi@jUgj^DfD$_YK~I*fTU7Xe30wki}j};`hMsO~ihDyeW{tPMaxP z16in$f)Hd9VsYdBAbp4b?-81ha#qQ&%>BFB%9V++?bTaKh+{d(OIu#g0u4WEYU@O` zmn0K#h5NJ7&E+POi)##Bq?@9Zc&5`(z+&_FM7&lI)cN8nE9+9WlJv$@AchaGNm^$g zv&Wul=kTJps#a+CIfvjCDCy4(26iGP$x6xG;kSPjq(TwHJkwZTZE8_0E7&l>#y4iz z2m;>5EXxtx3=d-`y2nLi+}Y&sE01OFo6&X+cg$%r>MvZ&{hZY*R9+i;Xf~~6Jrk9X zWbiJN?Q(2n9SE4kqlX}ppq9p@<0BWalikk0AyK;8+k&U!%ntx5Ur28s-=kAloBalm zT05KH^kms1NOv-Nkv_*rH(mpL3>)_zE~E2GY--ULnCRGHJc=FJIXVVjzboJ-|Le6H zGBR?I$A9~7t@LQb#6Q3+^l(4Ya3-zvIf_Qh?YS#eU3h@I(9XMK#)gr6`xr=;%Y@{i zF{T}=OFeb^WY)%ZOBAAeH#yr>QjX?5tWHCM@S{y z9=%5c4W^6WWyK_Ys3DJ+Yu8OiU-dxqKca8-12LJ1uo$u48+l3q=*LDYF=D;L-%6hJ z>eyNWMpb*$S!1*|9>{s7<@4S*!(jN--|aOjFE|y$g74$PnDul&V^Q#+uC=LregG^} z*kNU(?p3$xhukTxs9|Qhcwo)ewfYoJ2?twyY4p;PX_rlMIj(FMfn#}pupDnL(g4OY zJ@#DWQ!o);<#8cgz^nerabwjES=u4Z&F#2_XBW9$^^|-t1T>gJ%f3m?u;IE^HNvnT zwR2-Qt-C0s3Hh-t#wpX+RJlkuBrh;5jpUBPK7Mq~tDX5focf*SZZTpNF>eBqF=X@_ zi?vl%0(omnTG?CxA_nzEM;Z{BH`Ee#0F)`6Y$a7>Ll06fw$vG>VSAYE?XJV1^&z%# z7=LehTeE&<<(==yc=eSEzEXj@3`bu@z0Sf^wIZY2jV7&9V9=oGnjNbXQ-kxMop33I zxfRNqN1v6Czf!(MeYMbS0;>{0I3?-q>+Oxn9)YS?4F(1Dn+@SKeZ0tYM8I1gw3~p!emf-MhhJQ+3!+fvtKu*iDets?B?ATs6vdb{`*| z#W~wTq7Uq%#*se-`j%Al^r3`d7Y^4H9K9tjHv-Nn4QdY$WT+>v;S5>EBjLOWDx5qh z6k4eV2dQYH9mu+jf@s($Urhf4O#fX^$(v@~cXiolJ8Xe%G3@hsDwZneAHW?S9gcecF+RF(j51(W;rDu_(| zE-}xFMF)HxYaXy(t68sekoK}@I$wUCxI>^*We=|ps@FVWO=tGxa~^I{)kw_*!9}x+ za<`_$CQXapnHiEu9|zv9U2QrUE%v~+U_&qKW=)ypI@m>`&;w}|!r5rsyVF1&9!&Q-n|iLeb@Rd2z2&0Iok8o&(bi3gwWh16 zvDvU!EV3IeOFQw}4Tf`koh<3%k+T$!y$i9p3xtiVYpiEHsP3y%)8Bd?R7Z!I|9vb$ z_W{s7-hb*f*sM;z{wEdOZwv7ETZ#8(_P>TRit-`n(z?LVHSyoioyV znLMjKXL9d)Q&IGOU1mJ!%H(GBXvh{f|Dub(`o4b$a{ODIMsYg*tW+rzaUv#qWJ7B! zX5lf`-cvV*S(OmjMbw8K)~Nw3bvMQYV@~#lCOTRMo37=vi_`$;P78EXpzgY)dQwZa z!)BXr2}blafbvFk22a$DIXA}H3LWQXK_g?0j3#XLw1XQGMuUso z{I5m9Dih$}p5^gnoQSFGe(89_T31W`?CAKQ-st$~RrMe*(|z@`5;-|4mWdd#hbQ{! ze!6(r%!1H*f5+p_1T-{Om&RQ4aU&*!SY0bJ;lpO1Uz>xk!uei<92wN*PC(zu6Oun% zy!=?I0&w#XpMsCB3F-0KJWfY!*M{=U`TG&ud-m+fGmo)}!ZYTcy2}NReoB+YiIfn^ zf-Yaxx?>J!>da>kF~aHI)|1x4XAcEsjeckzn6%dV0YrL==m&aa!mN2QJBHHbuzHm> z@{JIYQ;)PtZ;kOtCJn^t>FX*fv@$`8KfHZ=+~FC~3Ae!>ZKQR#N#|jQ{C~Im-)1?1 z*5&`%9qu;ne@~x2d-~=6_bGm#=l;jDOljGA$ye=U#JU4KB%Nq{SD8gGO=N?+74Gp6 z?XddazW~FS-Dcu_Ci6n%Bi18VA$!W+2z74sL&uTRkw}@K z_pnQYPwLWHDE+%NB460!mXKv!2s;pFQ!CRawdk&;_f7j%$Dj?g!40P#li|##(*d_N zmnE>f-LAeL*IHh)X5TenYtbT0V6FKqZwjvKM!gbRx8WLKy97)9i&BL$w7vCy7w4+@ zB`#(^;3w3uet~&&^V@5SvPevX2%F|cFM3@aHn7?; zYd6(N6Z}#MKQS*?&2^<7qywKT+jrZqUARmOUtPqRe+g}F4c3v(KosFXsRYD*q<-h0K?ccG>)&07#hTCj3(@^4)^%C;b;y0)tk%Pl zc{d)3^wQnx$OUPiJNyHpD89?(+)Z-9FuxS3T3gRCONlCYQK|sd8F%6e2%BaFO~6L1 zi;+^%&(&;$IyHQ&-SaAsV0037J2WjKKs{~O`Y_MLGXtZA^Q}|5HTm}?6e_T71Ai|ksy;H&Sa2BUi zC)*30sM*3)o`U337}Skq*K75nku+itJ{%mLygmEw==I^*oBbaTKeih}(BFd(e?R=^ z%4Wzfp~wAJpHA!Q-RVz1pBx;%e6Tf%(}+kLU>-0Z9AqX85@54l8%lP~l;$$NzBbGR zkC>mb;kGl5h4&`IG$y%z6nCySnokQy%f@mBD}*YT|9frGaL|wJ-~Nr(;X58vyeEKR zb$UH$@;)xuu(QhszLxiB%CfENu#K7(Q1VVVTSRssGsyH%m0W{dFb|p|>5iR;Z}(5W zKYZ)bY?yU?)cS&5U$=PoF+r*?uYLwyGUM-atD5Zl86--Tb84ejU`yn=i*V zZ(+Q(x`^}rwODU|*zwg`%y05Vr-)VGcb#HKQ+eK~n5emKtLS!Bto2gl>-FZ}>5e+& zp&A3)9Jq?8?g$&@_38qx$+9+w?qL5uv1c=lbpqL-iZ+lpxY^@vNrjmQjs<=7e#w6%2 zZw5ns9`oM@2r*_&O8>*^*M&_@gI|9C1%9`*{~}Y|$oqG#|L>EhPoFmZe}}{27yIw0 z_}$L_>z-klg$!=uiYiIQhT6^KVBK|L1agz4x{F!(Com5Z{N# zYX=(s1P-qqP~Eq;;F{(9GdWLwer#;)I_W;Vjqjv&+M~P4dLPPraVxA;1^41sxTL$g zxD~bp;{PJD@m=J_=wH=wt!|CKi`5y?vc_x8ZFF;=wajMvx|_#ZCr#blzo7Zu2HpkhE+2m> zrfmgnnVWkZQCH^B6{c=q?qcWU+N^olQHH;cX6C%eWg$bEjM&=)bQQuhN5^KsWuK}} z&bjflm#chlDgP1gp|92mg6rhJ;qzwv$KLMV^Dpw>r}*71{-d6Rt{n!_N>bM;4py`m zgfbTrJbVXDK4TI}ZoqEVq+|R8*v^uSkxThcRlAdY_^`e|C&^=n9QrRg29a`9RZ0OC z7rm`=s1dVs8~APU)kV!|TX3}s$W9gH62#qm*oaXbJ+L)OM=x2g#~R7VH1QlM){E0i zp2SW*P{QptyXhB!8IZELbWpl&vLXzkkt)faFdc=+uGVo$9VY_6t`*D9itIm zYO%b>whH;bluVb;X4}D@oBhtt<)!YUCPys6$eO%1_ajF9DZ;dt3KFIO?Ea43qV`WFjW#(q&T(s*>=cfW8ve=@3a4ICjS9nSwP8a30j3jC z?z8{afUDL-AEHiwCjDC1UzBtCY==D%?_-4to2kzlL|fuF+XSXogQPkwY`M&XlUGX@4?BY2l->n8@6=t5;bIVCmM*?f$dQefj-aez)ZRnaa!= zaA*F%ySLls|9fBTKcD3HNAUlf*!BDF1#Brl4}Rc^_3C#2LCfNgnPs8J36B%jdpi>( z`Pvd-krWCBoAFDm4d3d9?H?amEvjX-rbPAe=`mG2wzVpW#b&t-g@`nF?$sIRihqCZ z_1|Rw2Q!h(<7p~$ajP-b?f+*_n)yF>pYA^Svj0EDug{KoQHUHX+urz8KYFk=ATtr?;Gp+Vv6M=J>q(+r z8tRf3vlbo#w$I*mTn;SgaUNlr(zJsyMmTK)Q)Rw2)+! zj}Vh6csSFmQq>z*wpg~UfE{?=&-$jyc#4 zWzDDt|5(SQR3>;=0VpEUN2Lf0IAGbiiD^3D0JYtP_+f6Os9?2g0=tYk3NNUDs%MsoWT*zAOhb_sYGC}m2APa}^M_9t zuWh_EjfF^AF6Q!5L`WhUB@P_>$47zZ?JLIhHL6UC9roxEU6RUGQ+xCXijV@@C%I5F ztKNzm{2no3cjOK$mB>*;Cf;WvERbq4kxyhkM;_!uIVggl$%}vmpD(?to`ZT&Qz)qG z&h?cw)|W_y=*V($sU^c?VH*~CDb54$|NWo;>wU`}J$fSxF?#gK`<5NmE{{}uR?Ei= z+YP3d!A*9>72`_DFb2xRvm#ySi)iKR)3Gvf$|n3$qf?);njs{D;RLb!(W?0dUK8pG zTgcL4Zz0RvUaGsO)ZB^z9KGIfaYo*7t}m3Pt8=y`E=8&_st1_!wB$*$U|}M7ss#o+ z$!$$Xae-yL6Jaj!Kj_W~jYuftTwaJ267vW*Q-^_9mEjDLJ1KLhq6>ZCkqNvBhR|#q z4(lxfI3;pC-)T%?hlwH#wgZhRiz^bJ6q|~IsiF)otlj`1+hG~P9LiKdHS(FT+(vIp zj2^uL?q-ky{nlno38VCRo5@s@AFzg1v|LEkI`1G>Xagh((M|_>VTMR#iE*Kce#VnY zwfl|yg%`vagdS@@G$7#_ZTf8Q*ter^ zoRPk*BpD6S<0sF*KF2AwYMvXQ>zxy-f1-jyD^dN5mTWUrcl&Cc zLqB@+>%Dpp_mROkk>kOfYj!bsCBqT|i!{Uop2fjD>eqb>yY4S5uPcrmZkCtCRrkKa zVJ}xqO&?6v-G%Mz>rrIqO(0-5RF0{TOfOI-BUSBbD}0a#2c#i_uTcI++NzTz`b{9^ zg3z#JJ;()2ugE^`YdNVL$w*kxTwLFJq6m=!IS1+pw6ZNovWF%kh{{B`PI9%2c&mjq zu)>ab_W`|Q2o)cHtxwx}7Ma~)y-UpEYsL*Sp3F*c_Hm~AP_aXc7zt6F0kV(ktP^>z zg`*M}K`zorfW?=LT_9ag*!dFIs&lx!ih}V(rc=1k!N~@K)sB|7LZJo3SUK`2(ja_@ z(>L+>5c00_*Qfh2JVtBoq{Yj*zilfAE-^PpUadM?%{wkG44 z7FO7i6MI5fH%m7wi#^9aexztRhz4+Lo#SEf@nfJaLsO|XfCQ@T_ylql@10k?=)8iO z@Q6I7*K5qq&DpAFcq%l5WRrwX_0l4V;A&Nw!Zp3-0<{XndKD=da9^I&xd`Isxz%;( z0n8<;21PANp>B1Uq6pJUVx)PstLGrTeHm+>XHrNh!*+pD(k$StUpaoWuB-;+I|(xI zCxv5G8d(j~(ZpOd^|sq-?I==Cw)HST9t|}Not%~S#mN&j=~e&{J9S1Z74GZifRL-U z!i&-R7@d)b%MIcB2BhPST~@=l~r&zg7-o!m`Q^Rv|MS>7dJc7^ki-| z>zi)B^;LT7gyw3>929_uQH;uy`@@2DPeOTNCEaz{Ha6HH#*gXr^tFxdX5mbP7i-9z z($L}cv$7zOL|=(JAx||!0Iyf#D{v;p&Z+4za?p0M^IEoP-D#|aPMO6*OZCZuG-jbt zisuW;o(D-R^*uaghm`WpbPKn`@xm2e1FN6{#s$4XGkwonB8Ac|uS;sK=NPq`B|TLV zVCvaSwcu_P_;I=nyMG9flx%bvj5ew1X9TgzHo6p({7-?)dIC)iY$K&6*4N0mf|1OH z)W)?NRU=R~?3jR^;ub9&{htcz}Pk@OQaRFooo9Y5TV;IV+49< z(`Z2_#u~*AG_rn~2{5=gY{_UrHr4?wVN+c-fPq`tM@>u;^bI{3@ntp!zkRcQ#6p0d z5sR}LDC&lH4`f^$s-UheGdr9kFVJ|3o^ZxXHnwV;Mk0T`Ha93f!gGPa4A<=Z-LDPg zwd;ntS|kU2;N6ZMYHNx%0EeV>>oSc*o`9CdRU945sG+z=T!7Fy=P92e+jfW8Ubf&J zWy`ECy_}B)m_9*2t08h=o^iFqVoQ18qlgEL`&motRqtVgP1VM#%V2hwPsfxOGt;r< z#vmghY3~x`g7yYPl$y#2%%FILE}z3gg}BMf^v+XhX(FxVkn~B`l*p2J9+TTeO{8;3 ztYUv%UAwtluy_4$ny!8cTN}ZxDgRJQ+3L^9xB}e)yIHttU;sYTi@SMhNsgNP1)WMz z`QvEN$j<6|e${oSl|HX#tup>Y)pcyc-qB7WuPRp|IH1hFv&_fnq+aa4*sXzbyR}3B zyO*+^lxNvORvjVBZK#|ZJ#Mg+Tk)h4BeyH$I5?HffeLGW5D{)Kmk2I8(k0YwG^b*& z1=Ucds>nImS|GYiu`N9EVO2XUFF%4YWge%I8Yt*w8<0&l*ae>_HcDIt>^qqwG0=Ua z3$mTM(@T<|><_1s5Qck)Wu9wtYI|5dRO}MFM7AjucngpfSv6-`NeKGjxDX3#PMGsd zpL2Rt@DjxCc4|s>m?uyaOv{}%I>^xEBavFmSD8hXpNuiH!rsu&?4I3V9yE3i7ZJxq1{Z&pQ>O@_4PBx$nW}wd!me zmjVs=;5h&;5EVT+a#8SCaDM;tTkZdUlumP@)SGFXz8@Tb)6Rw|A8WsL@juTWH{*YX zPlu1c`2T;3AEX+gY%S_9Ytz!@XeF z-(}yhVet5c{^!HaPo)|$$gAhPC6d&yH9rKq<;XZFb+>cUe_-DZJ^PpquKPn$QTv#? zqZJC#NLa$N%>KC>41--SsvK@cXg7rjho~0eP&4bVjd9R`(y8DPSoNLg6^l%a@W4UM zrgvFE^*GqQ3G3Z_|2wDe4QXuk{vSRYK5gcI9qv8*lKNZN4n!1Jo%A;F@#FKD4_EDI-` z#CRgwiw;)|H4lY*iK+Y_U&-)7c1ZPuZPb(ujQSI?zqHbi-fYbW-2q;Bk3AhrK@HP zITvB*gtAzUSg+U2+THlp>wo&&hu8P|pXmnT_3v;80k}WyJp_jQq<0Y*@r^8m8hL%@ z>-pwon&O1Plih5}9ds_QlzLpLa!yxqXE`-1xEuzLcZ1y}JsJ;I-;eTzpO-1wqGS#( zLvCM{H+DFaN~D3;MK~k&?pGMATG|oofrK&ad3{sL3#Le!k&Tn(hz)liKYL6s=y)~l z7!kkyv$db97j2tcyUi*ktlgn4FJRc#z-Kpca;7M!B8;{@+j_)$_TRQIP#!sK)%9(L z-c5v6r4#aMn0x&E$=5DFYx$ExF9e!uE4AGmW5&|%)ty^0BO;-ka!L2^(aE*u;|hJB zf&y!_=3uxVcUKtjZwtdsb}C;%h&|&upR3?ZfxDTVo5NA;R3unVkjQH}Br%zz>rUIb zV2HhG&d18ec0zUxW!7T^u-=Qit?lWiF1O3lC9ZO~g~9%Kr`vjhypNt|IY}4&IzF#P z6W_3Hk@KrmMfIlA(t0L=;MY&~zJ?x;Cag@!AtNSsZ_X&1$V1?xs-Rj$DZcrH|-c980? zVkIc#D=t`CCJ9ta!F@y$QGc!P!$js+JdZ$Hs1(*U1^Cgi{@i-2Vz=};*hNrXY#^+A zEV``Dy4-qjB3K98O1^7|i0A=>(aK#;rk*{g^dZw{ucW9M=1edyGy=u84L>)X{m(98 zadmHYiu!mU^Q9hv$s9+*KAUc{_0I3+tjZ(tTdCi>Mpex^fgOOGGL0kChM{#spVaX} zQ`LIURBzMc9l_~X-kL*_o7GbTNd+*KrN{B}W%JUi3JcGy%!5o6-Nq_#(yaJk0g_t>pNV z=R#!?JxerH6fmOfkg#9OnB=-;|E~O`9I*ZZ9!Q=c;y>6A3H2t;YRDfw&)x^k`FVFw zB#nyZl54@JQzgToSfLsG9(|HLhjIPEj!87sAL^Zi$6now3vT`%(ofCxpk7uJm)Qiz zqO&*p6mW>VMNn_aEeQ1PSC3hD95W==puu`|wv3E5CsY!*vtR|E zj@adHFbtmf)b4Th-V~N$?|O}odw_^gW(&;+)Y0mZ=dr$ViO5glanAFFKQ7f`3^oP= zVHi9a21CzYSj=@IP7A#<`qnw=2o5L7K6X9oT{k0}tUS!6QWgHqwXjXA)QKK9*M!>b zh%fVM!N>w#b1u`&h%SRgX>vKhQNUT4$TBi}LUZq2TtNhHE-s<-V;T8xrT>FIKS(D` zg4(z&z$=?t9LoqN@`Jn*mm=T!1H9hwP7C)u9$oS@)C?6Py9EWtVa}5-XQ!5zUYx2z zGi}HPVlfz)TN8)AY``<#tOWr)N6 zeBsT-l8NQmri>8Y-Cyj!*!39OhpUp&)iChImwvj1#js5rO-t))J`k%YQFZo|T$**tn z+;~u@gDwEeE9i9;fvP7O>a%YL?BTs8X7OJyah;;DmxQ%8sX-j71`^=&|cSN@;u&sc++jcRa_zdw=&K%R{zvq!ejp|j5f&rXdm~Q?0R7l#B}S}w_*==7 zUL9LYz^H0(I%|y9#sfLew0z$CW*7{g`n$bGldUDdAviFO6PWGVQWSF2|MaB5*A450>N2MH;|( zrpKO(d#W-d9nkpCRhU5i?rIFlG*vF5qd9^cthf}}P+$~0|BIZpXGKP#^ zW3jfXN+54dNh_NRK*XTF=tu)1^M+c&4uCSHldYtRZ0JGi#g;n5G;9yEz1?*fv_8Z( z4&(1FZ)?`iti1Cb8Lz%l!B;9!m*MEEsMlGTs#aulyV0ao3Je+)U9)3#Vrp<6v=c6+ zFtXsIL~fO<+|52&W{SeZ9Rg*&|T(s==UuezPH*rs}6p3U?p1zn>EN z#M!kMFO<)->;`q>mNl#~gn+dV8fdq2_inJ*R2}wHV5?pZb`xZ^YV+P2SB-L=-N#2~ zan81o=mWc`apX^dz9rQ>eJEkrg~K%kM{kMCjev7XgWAIb8S2SvI7624NH}kT3MUUr zg%(^_=IK`L+E1Eh-{qgSTG~F^ww>j@bhLHBU#?wV+T+Tv{HhXWC(~Sp-*v1uU=G?* zReB$(hD7GGKKnWvv8JHgV%dna-N3p%-vraO@9u)@T1Va3My9%14>!QL zix?N@oqN-k$^uv7j-jCv)-?k|M!sNMJj-;aTkY<)*;e=Woo#Ocl_kJi!Q_6u3L;a# zOU$!k(E(q_ng^`cYS!xNQVT)0sW_oQGRfHBvJ{aMA3d z+^s3GNz=g*q)-+Oys?0=u; z2c%|o6Dw>d!3u3JmG)l|yaAe%DwcXxaar#oPNNZX3e0+QQSgWteB?2PA<-MF{1`*+ z4Scfvs-9RS%r#5nRBv4NRDTZ%mqx2io{f^WKI2*ZeJ;yPjo7<~SUlwVr3s zv8(Ld2 z3y-n(p1Lv2s)WEUqCV`fP7PqGyD=shbFw!y(a|#4bSCM#o35st0+Q?yH}b$jMQ$OvH#iJkd}0 z)5XJP7KGOOJ05o?prNt4H0GL*8!;Kg>ROQrA2$2^+8lfp&i5MR$e=EF0{Tv#ko@7| z<;PMLfSZT-6nu0|NRQ9vaXMnVHk4=1-;dbdvu979d5ldIo-y~-T`qX^Q<^MJq=Zlw zbor{*9dkHSXFhv~5l;8Ep0pM|dnhPt^h5K&q_x%$AktGrKhPr+X3dM)F_bQc)vK(L zZ-j`PdZbl)Ym7%SX&_EdUsp+?l?hV(;qBYw4$p{AxDEDbBdxnlIuASK|NFTAt;-3t zF8|N&aJO;)d$zat{LB6CQ~W;9{f}pv(z5lEuiD9obq9DzI??#9GK*fC$Od;S+~XnI zVfDX%0fsTV!NmJa=7q>dtVgaw_LRL5>fGpujw7cdkus}!*5UDDf(krj=tVetecP$- zVV4G<)TOgf`gdzYzOctFA;2q2uUGS5}OGj?99O@?Bwa{YYcUwJo1tVIB z8dF~y1rPwoZ><~j5mfO+?BmDL(q;vpR`6C4MD{`D#M1npnz~<;LM(8-aovtYy*baP zPB6RAe1Gy=l)ikhMK$ZTYwy7G#V_;Z<%6x`lb`I`X z^C-%7zdxP4-ChN<{stscUmpoC+oGo#-%UbND^Fa7`HqDVZaad0p~7ii^tw81V6|h` zZmN?e_@xqlVqUJA>qJ}@3vpNaG4gqx`;FX658Avyme|CcVJMt`1(fi@)nsc zZVh7NK4GCr@YTzKD8hkK2?+W~{mwsw43eqVzq=5NHJ|wxqWe><>$2|ako&q>t%oJ^ zZafm{rMuOU3(`J!_y@b`Q~|0p?!*-kHq8o}fQ?ue zBc-CBtJwy1YWP;W=T#oT=p^cPXlxb-GA+dW!o>@yYu?CIU+Zg!v3&`;95&qD-EHFO zp6>K8Ex39As5gpgue{;A`a?g8a|c8yPlapRL5;WAZdLt!r>z_FyWE~(zpO;7;qJ<{ zzHda%+4*`#&CjY&V3UzXu=we)!Lo z&5&P0kNd4Yoz~U6)1Q7mIXHazU~3Ym5s@~)JYYUJ$V?a{z-GNRlrcRZ$ePXNQ} z^m@?beO$0%XO|6pE$`8kWn0%_8#ODSr z_|~P_FzfiJ^#!}WZqG2B&1J-%K7G2f{Zi0vRXc>dfp!4gKI?|N`AM_=I;fX7Uyg6y z!gynBr^1M+oQFGl^(e0{O>!rxo>&?H@9d*b< zH3qafa1~MA5jM)})dgCUWo-`K!Tx<>&t@9y1hPRDZ6I%Si)bv$?fUGvVze=O%dWfb z9~+CpXxHix(C)#aXh69)yW;S1r=snqaA%okzqXKD#mVOHN-&PR&IvZj=PX4d&{M!-5|GAuA?|m))aF^E>#P{Lx z+JS~Yfx~MDRQK&IxMn&3OwN;^9~&FHPPz|o<2z}c_ULZ1-iPvD+zKmI!M(T@F6r(r zZiOv@_`is3e0TY8eMP`^^5664&H8`Oo_>-4KF#m5$bZKDujSEqe=PeRJoVP_Y_`Nv z57RtZmVRTex#h#&gVWsd&g(vN%v+=4T4C$$w)Eb!3tYQ|e78$kN4rn${?@8ZzY5YD zxW3)MbFEVf{i`~z)vfV&u{tAK)_ASCjc)FN2K zw5^~mb91jF>dG9t!qn}{UF@7(n>7zR%JA3G%$ygwEMzE?5qo=pu0ojR=-3Rn>{Hdr zIX9m6a+U9$ZMfOa1px^1EC7M?DE$I}D_iq^?sOtY|L? zWiBLm_zs$U#w3*7fZeP~$M^@Zoh2C~m-3&gb|?GrVSRs2lE)4?^j~reBIT&6lmaX+ zdRygCBWC9|@Y~|6i<;B6;A$0+ohryBh`aZ&5u-YKU~80)Ub0?~HIk8O;yF^R7pIpz ziJg3)gxhU)(=P%uAZ2mspmf`0MHoaQRgyhpwoKmB0|;TZwY8Uy@$xGuZqu$iMkBn` zVtJ2k74m;6nJ%Brwu3!4`<i|I;V=_34nZ=|dAKFUpMdR+$HTtmoL(diwb2^&rEjmJ$1Kdwc+O zG;8%W@CD>+#h1NhlHZ^%^Yu`L?{q?!>{>OA{o^A#U6j)@o-j&8NP}(_^0a`u$@SGW5QS{8rIEDJqOc$~1_+nFHA z*OmZ_q);%}j9+4H_*OS;|MOyu8}5DC|DWR5XUDuKM2?m1aCuRwFfQW+s!z0*@;9DlD?6n3{bIH91k3_y+?t%%DhLxm|u90f?}@!E%SKlJ^J5XpZ$a9 zu`CrkdUdG0Aj{>yL|Ax190@+a=DGZr7hI}PMq=>UH}?j=&G-Mw;r^>14}*Dh<1tq6 z|Hpg7C(ZhwyU(9~IsZS&uiq-}?RopACa5{zxWa1w=J7jI4EfhB2vHaf2Gh8hm1DTR znHs{T(Kgn;R-#a>%<6Tip&qp@t&FZDJdNBU)w)fq__9nHuayYL<2~@aBa}xKd)J)U zN??iTi(H7obmnK8;SCJu8#KGx26bI7cp=)b!E8gG+GvoOh;wkzd#G4SCBgM1(Jl>j zNsCzv4*}a}?>eq^ozuYE^B}X$eNzNob9ro^6M$BAk*fE6{R6x};%S19%gHWcS$O<(u{$7JjM!xyOr;FD% zUYf>2q%0S6c_|_!5seZDj{W1K!1MMMIi+Y*-|)fm+S%z0Y!Bw4U95j@obgPr8I zrlYvPGTwu&n!zx-XBx;>^5G%9+l7widgS;?9B(lV~&_qAu$)wu- zM*hMJVhlo$wI3Rg@Qk<0XUigNnKo#l>7>cT_9Hqq?!v{m>142NWQ4KMj3cs2o~0|H z0h?=bjzq!Zq_#BHYFn+jT^wt6wuR%XP~cQtvF=%-2%KffqTb-N;AzD3NZ~PYKqCXq z;;B}NPasw)EF=jlWE$&p5vq-eLbFUd(V7x2%|io%@~c@)tTqk+I0e}I`Y&Ux@UGdh z8Gv20S3-q(3<2ltnjOC9^DGhU+Ph}H{xSN$U;hgloLh^1BUWHxvmzCEQM-Dg3OX8|~U*WKq zE2gFort0p(_Vx8BvhyYoup27JR7j>5D3g(@_Oul~$b$pY5W!a{|08YHNfP}gka9t2 zSh61E0;X4FANRGKRE}gMENCvSZ#_|jNP(OKbp%@379`n2lMzH^B3viA+C{w8!Wvj% zN4)!h-Z6xVkH6NZZ9R+3?y%k^X7M%S1{qIgB{=&yQ+=q|p+$^@D9!-cM|IYTJlDce z35*~YX(YhnOJ**Rt|#n#iEGt4TwX=Ncp}p&Ta^=7DAdqYO!sOga^dVmT#Kw6C$X9}bt%54$#@cnhDri0TH&z3^vk4( z^l0g(aZC#< z?8u2dA*`FFo0Y|$V;?_K934ahxV6slF!=Z}P?w>pR2x77)pmRWxr+DBD_(S7K}~o> z9@FbJX6NQ?)iXR5nnAKj!l!y^kwkE{s!ZXUUUPw31!BF5lnl5p&*@wQar4~jI`jbM z5>^8ir2JO8esEiJEjPfQX$sBbEyHb#p+-Ra@c3 zXnl;%NJM6M(gP{O3F{h#)#$oODcDWWSE}|zoPv&+a7aXq*M(~lq1Ed)&~?<0J*WE= z>Dkmhty`=#3D0CiOgtAMD!mqlgpG`6&UUy5%=to#7LI;_vq@Q$aLLE(&fV9j3kx6t z;I)#Q#7eLL=4i@fbYFB04DuGqn43KHHZ^_kgE0SdP(^2L;m?PjJI|n4WTFDU8*Gpa zA$_TwG&JflDqZLc%O^o8L1&RBTO_O-P~yrexEsNHAr{P}!3A2bH0X<)9cg+px0>}$ zx8M3Iy>&u!HDwM8z{4m;<;neF!MZ1*ys(n)I&2#o>=5I}bb9*QMt8GtCc=v~WKL=5 z@cLO6X_eHP>^DTFsK4DhV+4 zY^GXpHwyeXU54F11V~CYx(r5})bulgSY;bsib?*bz-2vwrUtf=QWNWIWL&{W=0a-Y z+Ks9aC>wT6z)ta68i`4qYMH%i+$GiIf=6g8hfQE?8+Sn0G6<+t{T9=t?Z*FCJFk6o{ac18-w4zSwCVSz|V-q zSq&6*!@CDEt_@XCSC^R`&XE^rJVj49<0TtgwM`?Dzh0Xg6d&Qaz+i@JcK+_y2J+f< z!(1(r13vI>M-R0%MH_%aQo41SMj}r@OXDhzj%CzP+#@bP=$!MEPmyiALu@Zw@Q$)& zR+nDR#{x{Bpr6$cIWW(-+F`M!Jn&J(1IGQVrS+=!u)(HkW7TCayUV9z%8Qxl*m7f# zk&v`^335Su10qUIWdvqWJVKYx;h{p@7Sjw$=P$%H}|YH9v?5x0g!<7ai#m>Nc8FG1r1> zC{tDB9BeHRU8dL;9{I4U9hR3L!I(0S(?|^zbg~V|CL8R6&l4LZt^)R*%#j%AKGFr* zPTlDxNl^BOQ%MNJy~8rkH956CEFUU%iCrSw6bifr$cn6*Gp!^9eQ;ceg*7M4d8W@f zJt}w!Vs|?=r8>+LC<>vrCqRLOknAzb=FdkTwfrUO7Or&sF3W&mw zN)QNi*9p%+;Y-uPT!h;@1V4UnEpAzs#Hav~c_i|Hh{W?Ay^XUGY-BC4X|WdhW?OLm zMvGS07h-XZ;_$a94~O8sgdx~haT0|*k6;CPScqIb3Yh1e3Q>8yR?gh_;PYB_wv9`H z27K@w02hdgo*cO-_$xTSfBCKO|DWc3!c#st0B4=GNgh}H*2RDC?Ka|nhR?s$fBr;2 zNHs#)?mWgqUhqUtjn`N#2Qy%=Bi4gtBArdZ-R9CH_IUX1OEwIKd%>>1%f4a5;PG?) z&xf6#N;P7TSI>D%B&lC(eh7BUk#SJ!Zs(-`z`h-N_Awh=_lKmS_Az&d*$hPb^?W6d zJ)ULu&#!}F@XU)Ue;efRi7?KsG68P+TPgAdy1_^6GFBx|;@_b~JO@>EEKq9puL5N1r4qRow6tul`McS<=KtiA0T=9W5YMNBL{8;dk@Hk# zTp#pjPj`3U2U$Ax9GPLn{=Z@HLa&`S*Jrs&%jYvO=W#M(zw>maVln&oss39JZBw_H zS5@OYDZ`6c{CgOyP}bV!1-`1<&biEkNgj)6&eMO-MKt3@?GxQ#k~<&7xl{?i3}!s~ zeHM$$4v_w%h^LEOx}(I&zZWuBg-EY>Qiwc=swCmH7al${bYT+gd48AF)bm6^EBXI$cW>`mv;M>G@W~hc|4Dx6B}}ZpNGPdn zrZSV%l5V$E7H!6mp^|su2_@GYv0l&f0DwS$zw$AMv`A?Gu!&*&efF~wjHhVV(EQH{ zARp-e=(|uRpvV9?WQ}=PC2fju^nvbfTlc4B%tPu*e?5X->&uC+U82u8@g7&oZ9RWqyxcDiMZqp514U3l$a# zbO^|$5dzaN;7K;)LA`sbAbp%meXYUNaO`0u!wZoQ=mxiwm#IGJ%-;_;@1DMJx*xG- z_t>35an;{g6q`ZY3EoEib%WRH&rMC{>>=^K$hoFlP*RWByI*1Cdhrl^Rxcc4o>83; z_!~NNw$t7FtY;SP*w1hr(*NHIRg65nnHqL?XF~<-0I!B7iAWT9ea33SXCJuDg$nqb z|1ML0g~@>Q4)%GTj+p)rzfz-^&qt%-<0ns_?LB|-_3rNONVgfpeC|VI-~=;~Ye0Ef zNT^swpnFD9JYqxip*Ma~Gr^-osLDChD6%TWr)S#GOwD>a{rn*F_;;BWJUN!pepyHr z@4^h)A1fa?k9P4H%8-alpP}G!LQ9PZ zwkuBvxAjZHth7%sXU6`e$oW+KT*L`k%SUW)_cndc#a!l#&ALZQ5~x=Ti-d{yS;FH~ z^XMp6c?r_?xQwQv2vjD*a~qtYKh4+h!-A^|HOR}6j<4M8X%RB9v*QfHoN1nq*nnhfnmw36GO97jI{|P&1iC zuphii<^T~s)!S2rGZB@E$afgTaP(9m^NU`vIpy$Z9;cvP_esWZh{q@weLIdCc>TWD zuLj1GqDKcr)v)Jzn8_UKn3z=L_DVXK$9XPu5t+NGy`yU0U3W?T9^YVQbXM>wyP8Q= zX;mpYivoP?omOL)PtDDiF*V~L(=nEnNrI_yM(k+vMi$4pP?{g22x4MUk_Gu0F97Lz z&OtBS%orb#=!Ji=d8r^yyNdfG*BP>F+vAhI+iaPf4J{+zS|J8{Dn3BuXL)?d3*npW z@B{3>eNb7kD66NvM@BebkOcg9v8XcKdtM<_@mZl>A9sH0eE9VFWAlKDa^aJS`=@7D;Iuu$hmW4u zCvR_DedA0`G*0fdbDnuf90%rabE@~*p%#Lwh8SHAVo03Yeli&;$l{`Jv}g~nmKnt; z(*cz^t)wVAB}HB+zk&y;pjm#@T%DCl2Gm5AV{AYwt&nS?4fBHa*_LrU+os0Y|Brfp z-2*l%rFsaw@qt6pGr`yZv;M;hh(5TFPnQ!-r)hlHd$_<*o# z^F4nAw!E)YJWUyA$9XJ)7!s}&6o!C!o=CA6yx6=x%bHC_o)^ihR3CehTdyUL{>BrYh9Wm#Z^Kb5bCb9sF0g}+Anh%>(i|OEAA|YX4Ke_KSqf5RrXYiU zDOg<8kM-{3qEsUHr)3-o9AhqU0-6|Ld9Au=_I-SNMIse(ClFE??>by!?Wy|5=Wc-1JXq{ zIX}Tox)IJCt0I;*?KBz7W0?l*@coGOH2Qmb)q6#l^`L^I9SZeE^(K4Szn7?bi-mDi$SE>Y7h z$#WVzCs~}0SZ^3S4xR*0dvtW^mZ_;s(5KQmz4-#h)2ppzYs_TUDTnpJal&zRKm*nb z2>m7(F(cS&n5i$nx^vm+Vca&>2CP6v;7AMXx!pAL2(58}B#bRi*v z@(bw~GvS*tc^3O|>Q^238qz_Z;L|6&ZXW8PeL3-YYCZ)ZnvjiHI*rr!@Q$RSu)$Y4 z*@|vop9J+Bz{pj%?S@67d>+Cj$XFEXObWM|hY~k0W^Xl%kI4&M@AQioc8RF{TOhw< zNGRx0F-%*h0ydgYUJDpLiEIHyM)b!3FHX=u>wk(%kuT68Ds*!!ROu~BB_?Hpi4rv% z)Z3mivlv`aoV7Vs0;8$ZLY7O?G~q`?ykj0-2&^g9cguVc<5r1pqAr^)a8s5__!SUz zL(40~X}o>iJ#-6|t0Af$7(GuXbi_lPzfJQ7s2;MkN+ao0GzCt*wvK+07|FF-?Y^n2 z)wSeaT?*?n@?IldRhX4<37X@dsIeNc;o}!S#`=-*Z`-7~7CYX^NEoSD;~p#wAoqeM zaf6GjBbhk0@nV?{yuR0{riVh?+lI^H&BwHwr5f1}v@lZmJdK>fcOWIzYUSy}MN9X_ zSG5fS+Y*;NDLKV}7#_2I2hgx{xaDh{4En}Z7Qp3c5&Pze2H5=D!C58~IbE>R4E*X3 zD%Z7Js^awN1!k6ly7m=x?K@s^ggktYBRd0EsH_~yHPlXVynDf917`3)8O5RFMll_scvn?QA6e2<|6F$TOU+jx_&DjIG|@E^l>@0Ydl()>W<& zUz(duoVMt;%2Go48BI)>_1OkS0U}_1O=%P$zkUTK#(Vkt7D-q_c=6xX65*aC_-`#* z!QgIhVxAQXKZRFun zC>cOTClu>VPC|}ohP!5*{)hsEx5VQ@A5`FT{63zSb5<(dyG%*$U~>`0<-Cf4yrSf0 z=3DSrj#Ky+eX7TzxDo>IV5jl#;J?SaHEwB4^8MGyP7-&ALj<^H z)Yu1I0+bm9)FN6)f{4$Vyh>kD(J#b3h1*8F;7nfW9%pe@;~2Ssz_fy?g6Bm^ju187 zL@iz&%2Z;YsQ$W4jikz;G^Jh97Yw2*7`kV!n&gZkrA zEynUa73iA6&GyL<)%>nDa;*iX4xJ6purV2W1#!I^P9tkMQ7GqG2}TkUo(E>KO{JzI zP?IEH$;kAeE+xamO03@OBhWjO2@Z1*xY7M^9n(}&n#{wN3s7A4mJ*uy6M41mfhiA- z81PT8kN-dBN?pl38ZlMmGM#4hw;sFc#YEj2Ujg_Yh$+DCM|HkbMTmE^B-3wAeXP6t zyI2sEK9Q<{%6a1e+GnpYnAB7rM&cLmMOaea652g%ym0L-q<7RX3lU)m7-IAM5-gS? zy*$SxE~#chlp%v{)uhfbPk-len8-2$bsMNS;`Co4G%}I$#11}|1qFW{?z7R(zq8nXCl3fbBULw zOPJ6f)Jcm_&954~Z zsLAiW%}Z5K)d-ntZm)VaE2T`tj=mu<4uv3CoLV~rR_HqXt<*a|On9k8m4``>0y)4m zSWH}~3KR|px`<4I+AV0u73J@vjjcxmSb3u4j!9C=w8VaTynk?L@Nyk)0dp%g4SAA?$f33qvRdu~CF%j5239>@WiAC{+Lpn% z>*xg-Yx{X2t|9v>zfu*VF_;&k6Ezv@_S-ZwJqF-hlmFStPp=Qp_D|m2#>dQd^2_1h z&W>IkzIl7}_Mc~OfBO63o3rnZULTHFE1OW;$*n7+kOSV~_eVdy88J&0G>K;cD^tz* zFdeGU+gLMXlRX9QPk59VlQ0~8`rEzcw~pe{>B1pk;|<98SYE=sA&-|TbAee2aJ_Df zL1*<~nRuRho-wHEVR|64ewWL6uaiG&RxNTZtS?J~*)(iwBp(VEFZ*FVf3Po}+-|Uk?3{JiAUH%_PXkl?%&bJ-YFwSUh5bli9_0Dz40I(f9KP@M%zxQVcLg zS!v9Okg1a-uRjWzhnnw;w8$662xhwAxS|(gv4c4%c*eX1la5LJjnVGL0wbD)g7YpxtwKLgKw~xyMfieG8IE zFB`m$7VE$;lGb`PSV;Ghu_L%LxVBkiPSbm=b4v!^DqCwaPds+!Q&k>_ev)abuPKgu zFlrFuPyk~$L;7||4>bnlqwqRTFO)-djuPK>r2(H9+!Sy&d*;ff+&nV)4wu*@`oE3q zWQQow@D8I5gjLYGeZmL;K^=7e+?VOwx@ywXDBc87!r zw|J^xq>8+(Bpxj`fPAuDn;gGVzLq>tuxiilrV`q!xltt-;TTLw@%p2T#E3n8yjxjP zO(0=q9Q7OW7UYSnTTdC|WfX%Nn5m)xX``ihpT%e@vp&-=_MYzUHkzR&*rLZS$ie9m zUXAyE*K*$9#e)3-)6u<5b_Ka6}jD4#tf(sEyB1#dh{l9$$*Q(R~oA zm`anHcAe`&mV*7F$m4NY)Ub{n9@7=^0DTyG{NmEOTdOQI{w9zDW;yB&LQo)!aqy}N3A)php9{1fWGmX9}FIo@#9cqY-|hz&}W57dn3A{x|&s&hZX z_%z|V9YsdX&2B=`mwi+OsWFvR*mdJim-y(=E^tBk`0;GMIMc(NK`#%TFD5dHG?$t$&L(lL ziZguRw5U8(_jZ-@9Kf!NEj}*hd~p^>XTyd7QpAN`O*@eOzR^UBFEPdm?l)f5d22Ob zE+!fY&F1yp^RtHPp+DT+eb~`XAD2nH?;1d=`@M#M+;qsNxuXdK=_}{+rJb0ro~(F} zdum!NZDMr1s|KM&6=}4z4P-O9`%Jpl1An=Q%1~?oWy00+$zl@#wzEyO_R9vs9?s6h z`z${@ThgdpNjM;T=R(m&efhE}bK}o!pdS?lVRbc{^Y@yq9ZPf%d-~+@l7=e?-b`xF zA>^{YeJGgaXWF3lLF<~3iUioo<0dtxQbTxJK$!Tx;b`?X6PY*pg#I;h5Y+)!at_+W z5&jF5FO5dzKivAX5PUvjDi>$^U&L}i1rrun+h#3arPhS_P1tLg)p%PjLZKzSZeEDI zu8f;_rsJAVMIMz!lZdIrOvj8hF-`G|0lV;j{KtQ^5h^Z44la;k9v3mWL)V&+6F20( zqu|WCBiGO{N$C$g|8>1rIOUpp^4D&le+h-9fJ(KCtr-AM7D(V`hW7P>N4Q zj|J@G$8+}aW8=K8qSWqwDpQ;s>Vf=l#?us{uKJ!G#RzWpfh)FEt?E({I{TTob;$_c z)#M*|GBl6+OvJ@ZGf8?N^XXX}S-DWRkh%L3$NZ7UDf=m(29?})7PSR8m%b_~3VjW= zn99>Lfai>!{k!qA4kdX*MV~w+Smaa`u9$CoFGe}Wy6lI|2N#e1M zZmd2Sd}Ck$Y<-K;IlVPkDMv`BHx}?Uvbt^8po51tQ%JR3NA6M@3GCs+I+2ju1NfUO0h|yfix$teX ziUaGLN1cy7INAT@?0=p9^u`<|r7GmS>s!5+BM0I^vuU7vuhfh7YF2RfP@1Nq(6sZ| z(H-FCaGyD{k9$wV*WkX`1){wtYQ#Gmeo#5O4GN(O7gC1$M4;Wy?)@=WV(;nANAbsd zPgjq`*m&>Z`aK_F2-eeA&T2f+^Kn7^d5Yk5@szjwQUiAZ&cT|0P|G!k$Hm&bwP_*edg@Bri}_uEGWp)fzAI2_9KQsm_69;Yz_C*Wf3fk*1DdY+mp0>?CsQ&5vb zG#k300#4fe9Nr~zs?0k%9aOnQ7_(}$)|d3gv*#I|yDKsVK=2?>76oJ_(jP*$>9dK< z&oEK5vb!^pQ*1H^Gf5~@g~{8f+lwrziKz&y8=S5&WDhi_(j)p^n7JBO zau?P~X7qCiX95IGr$UpLu?LFC}2c;5c zN+qUSx!-uF{P3-brEN0mr%>C@pc@i(OH!0!OJ*$`IXv{ zoG)ZP)&Ib32er*{`s3S<&C#IuosEcH&5Qe?bKQ}r|1-Kjg4EruCdl;>7Crs1UNgF) zXZ|9Wz&H{iH?fSNOrUH@%}2|4?AWr)prV9x7bsRyA_=JJ5K$qoVkYKQ!LVHDuZ#Fn zkPbkhP*0U$dBGCFRbe;VGnS8Ihc zAKf6<=_0<0r+FOt7cZ1g-&X-o!(jMz@VF7~v@XM0E0Dh)$={64uOf&b?yS#R!Tk8e zv(^XCb|FluD_xVOEWKIZCJIY0VrC*Nx(H|HMVv9c$zeB}^BJ-*lu#`s2LE`~?#Z5o zxY2%2CZ~G?2cQQ$|CnQtBz8NtOKr)x*?2}$mwf6)W|%F zYMHXS!y8>+lx{~`n~k-)W4hE-CE(J%y^?T$%te+<2zud(#@2$--y&jV8qRn+6;VY_ zS}39lY%CY#2N+P7@idjW*vEd3O*c+1z0(7;hn$Cf+W1uZzs(_>>F`{zInOVM+2TD- ze{CI1YGxVTIm%*ka>G%uK4>)n+;42Gp+hdWp$aqaCK(_v){M-Y)|sgVN6xV`nM4GB zLdBLGz_-d2((|4Z34aHDz2S3Ph-}+)6{dq?p1Eql>h4tSfy5{~pc-7GZhASU22jcl zE9&Eua}53|3l~0|R1P6Zh2XeEEm%h^Et7WgiQ&ahcm23 z1}p6AB_33c;3hUoIliCDtjJFqysNmCjjC2E&WsmBUAPp}+E8I7g)xe{ucP9jLd9(+ z75c|(o=0xW`>$VP8a4n|OMyw!0tnx}sS;>8h`~y|HhGq0ktp;TTPcBX^QwUTqUrmW z{kI1{wDA>5#_65m_&c~EPG27$9@{mCI$avZN(ltmPnk-D$N=GcyZRiEX++?IHwwZz z!ZA#svz@iwg*!G9AP0pOotj1qYajL35kMm41+wlcScmR-G5V`R21X=dqTc_Xy?5_w zBgYoS=Wl-sjgy(cTx???q_6vAeuoBUsI|I-tMu5~lD4+?Q!mwVg5wD|bn8j{r{TTDWtv{a7~~j+t5i&=IFVNg zor)8QorCf1EFo9Gq5)7A`2k+Uf`{oUU1PguK@P2Tus38hobfCdESnIq4tIEedWzAD_PCM9G(LUp9W1+3&jk5N0JSM8H9KW;DZ+u9EEqt3w4Y{ zby$e`ke~>m_Cbu076Lw7$0`BhA-`;-jIb(3Gc>_B zf?_HodPHX&j|uV!EF&r)IQ_j#fgOUkKeN%iF z8NT3|T1uUPg3bcNr;00cYDok%gON5xlW_oMkj6zVbCFiXppdjw$e+R)-8b2)6cVY# zbG2z`C3UMhbGe8mlQr_>xRY#t@vfB>CdmEAoz6t0|H)Yi`SxAsOK|CRl zD~nZ;3ZiXBB29St{z%n?dhdhxS|IIeRbR7|rz6DgHB_D{Rg=xbg-)lP$`IxtoTY5A z(Y87QY^!YGdOz|6TlUrnA`Nn;&W_(zj!r!>#YTypH(ip-ELI1w)Rv*?Dp>fmxo?Yw zZ@{e^SK}t^x-LF9zi#*Q)fo2X3~2_k$(v?8X~67A?%xzg`pA%$F{FZ+OfghKdSvCjf)!BsnTHK9d;peb1k%R7Gfp6dy9qCl`h?8K4cL8r2mF8r_>agDEOn|@ zY`_#}&9`9dNtPZkm(DaB5%@FdAG&mE_Qo~Z)TSm-b_(36S$9QDy(jTA=( zqGRTOwJcelBo&uM+xye<9N2w>D&g9Pj5S7*}m*={jmA*#q%Eu?Yw<=a5&JZ!E}6T zboBb<-Pz%#faLrV$ke%ENm{-->EQQM3ctaUSD$S8fT$4|F zR9;!<#*_Y<;?iA6mM1kdM#7fr9h7aFqS3AcmNd8wO(Zpu%PLJX_Dpbb&l=PPE4ASr zkijfqYUG>Amywx`s;{<+m5i#NF%&f!&D^l8Fu_?d#hE%C!A@M9PTEF0h_r4=#g-S0 zo{laDfzFx4BYP&(D<%o?c$j&gs2dax!y)#r)5HszQ?4RK88j+=*b{UVArCWU?Q#ih zvdnGd7eTQC-Q@_k?mX-4+z#+%DLr$@hXp`WlOV`bXQVJpfsN8*DA#}@=K@&(>@^{A zD|3D;oJUCCv=r^LcLhc#=2zgX;S3IR{3BH6!+FlA~xq$@jMXcUBS zuB)5DZJKns=)F(^4G`@^?OS+2qi0;lL(_5)d4p2cb4u2s5+Yno2vX2Oycf8O$8mVq zMZbV#O&a&A+!U}6Ax&-SV57*gY|ZKqZXY1w(ej-v5WBH~6Rz*;D$h0aD=+tB(uTL{;9J0$rH4ehYk6J@gP#Um1x4(Q!EEL0#$}&RW5@~%@50{QkruPL>;oY z9(b%-Pr2;O8LRD@d=x}J;F-;&-uh_Ij1LN_WztU|BJ-z5CjZO0|xQ-XrdR(r_XAosO4}v- zO`ewooFoC3_iGyIFvPIPz;B6C<#Dh927U>_>UKQN93h@OK|jqn__On1O4tUvB{IO8 znU~W{_N#Bnay$xo5~K{J8**e2=UD8OTVmN5%8m<@q-G{>x(aT!HpjB zT%?l@d`p(Z{RukMuEX%Fd{$KfByoS7#h6jP^%SCyCE%pwTV)T^xIadP4*CR9Y-Hs8 zG#nE;iZ+x8dw@RZ=6?O062Nl!bo!*K|B*E9X~ovEU14E!r|FJxq}s^sVl7qCEo+tp zYez$41u0r(`Jsasw-&#_N-`oZt9xFzRA%_3$$%iM8Au+2$PaD;e}+Su>P#a@3IuGW-FbDl5t>>OCcz=y^tDgkbrAUjq>q_%?HCAqm2hkaj;jjiDW*9qsdA#q zz=C5hRw0?+f*ojL1>3NL`vMI}L%P~&dyo+WNyRNT^AWZF^aUOz!0Ke_t9uYAGxPS= zd@1Xlyt_C&@A8j)&OSCbH#c{8cHrO5&CUG3&!6{S{HedQyY*ssb9;NM|EJBZ{`QL( ze?psIV!-)l#yC#?w7GO&`N4fBPrr*^6E5|I&x~LPX>u&|`SUYykW76d1<_Nob;y|q zw8i5g8WXOi+?`H#Hd{fw(r^v>vkc?qE;O2jp(v=zbSQ+`i7}tNY=7#C&DFKH(C0R~ zM>47m>U4bKiF@dehtY*1~RmK2;P7Gtfdv1SN*}s zxfF3ohBDxjAksQmILaiV8Vs`A1Y2Ep<8}2RDbpLv7#C&XEk1*OOsSiPLFAjq^ADRF zC=Ll`MA}XhBZ7$$lSL-dA?me%%VLe)2nS7Z(hrwaO%}4``t34 z0JmfgcSg8}r97>J+Rr>3Uq*yqf-kh1lb8RC#LsjI8KHR?M7@lL!zBOpueu1Eq=j{A zuxua)gySG&((_oLrZf~rm2)#BXBA5Yj{A2>c7Ibm>Mg!w$V)JrbYax5fRUc@G_{qQ z+6}{m0=+*wUSnX-Flc@H)IkZ3#-t=RiJWevU9-hX7Vktg8L31p7ZNg}N(u_Vjji$i z4D??&0SmaX+ZBJVCdtm50lM{pT82uA%~1P<`jD5rUD!+DFVNADl8>;rPTn0HUY@=? zyLkC@&7>-Glam~&5Qn$q2S057+Km)YpW9hFi3VG9y$I^jX%0})G+P?nfRiae$~fjT zL5x-OJ@HQc_}td;coV8;`l*6y@HE9_I@Q*^98>K7ghL#8B&nVo87SaLTtV*;hk)N` zL~hYEh-TbS&YLZwqf^ad-{2q=G^pFsoQ-xn1N#xVfMAq@OEuQoGwW2j@fJYpZIfAj zX3gsP;n~ke`;LiS2!eSJwP&C7jE4p`_2w3nAH<)Zty`P>X$JD3?8xdEJKkCb`;yQc zowkvC5#;vg>AM5S6qQ2`$in8IpAfZi9CvNurYQBYw7#smQLF|8Ir5-t_l&@CrsG?; zOZS()0zlj8j?zP$dqm*3G@=Py!JcYlA;`0B5#RLB)^-;i({T`GqQsQV0$ze~5~F^a zgI6$O@@i1RtF@r@U<76`x`2_-*klQwtol(cA_SHrM<=fbSwd`?N?ECX{XhTt&p-&2 zTPq`yTLP&7WUmkxTdT$NF%PPkN*gGWOoIqA9HENGY;l#NHsyh#$=(09wq9&K-v1urd4kU53V>BCRS-IPsE!VNnUHb7cyiZPCrQV=Ym7rW z#9`ONoQI?=#h|>vbA1 zw5EjLN`e95wPNC}cUwmzlt?~GHj;9UfHwIq;Ffiz%*X?=DJ8U}sLhyQ2Sw2zQ;g~| zr3D(HT(30o<0*hYc-eeb4LDOw$X0B2GlkW9J2!wCrR@BLwR5e>e&<}6;w}fiXb$LH zr*^TfigTi68lOI;i)&3qdUIX^*UQ?AG=#6hp-T4a{Tz^a#=-S{7V0ah*>=~AGP(R4yq zx1{o}70a)%V@j`?igTTiXUr9JRw}_D#yV%ItPVo=LCOM-bk=OvipXsy%@Y-6wF<5& z4K$lEs;b|?OSBeJdOeF#`=;MUZE11Pb{wdtRC2xUz+Jb`7TukBD4-zn$;b3l+s7QY zQQOwMwyTbnwFF20UKmgC$!v;tHrLTT8t(3p$V*pjlVj$L@L3XR%l?GMNdVWN)Eq>H zxgTP$0>$t-tCBxkt&8p6>5(eNx^FN;ZCP0bkcT_FO4r&Gb7G8GGCMWpD=^lezPMW3 zjVy_{H=+}9wFytDzc=HQc{n6V1uiEsUKo{Fbr3}3-p!T^dHE7Kbj{g?bAIwAR)GZe`z$xNh~7?KQto=>+|C>#xn2n95_5xRsQI>``S7m z^O;Nk7gK*;-QS<;^nd-$?cM%X-u`oEbNf;M_YhCXVda+fDJ4^)uTa(tnP{u5Y}>q$ zy4DAj1ts>ZKJ5Br7!3n zMdErCakS--yI8Fo&fVg>>FYgTtHzWRJibL4ha0$o5oeWR4{IR8s4yVW-;B_@DIR5b zif)z?H(v?FRYzth9A+nI(@hxR?Fx;!%H*Qlt*?tASuuCmz&Tp8^Qz%OTN8R4iC)sL z>+*7;)4Ga69Z!_%2;fo<%QDe5+mYfOHo^LmBBN5kmddt8san39&s0}zJBwO@K7St6 z6qcf?|6~)q^?bLku!Opu4$?3z{U(NCKonhOC7D05R|oFx{JyxloEvM z5qtrnu;L-jO&rRaD-sA}d-sMx)MFF1(`!FSP$zDZdmH`ruShk9 zIv?%qh#ImiO%Us{NgI{y^Ws2Ozu6&J(5KW};psKG+dxlcOtZnuDo{vtlkHx;tHg>C zZKU%C1Vx!eiEpyy9JkqY^bYQ?g3ix@iP}vsM*AwEQz_FokSAVdL{bXi&&X)d%8Xi5 zRf$Q|YP-l&xH`%#U0mFRxxpGNQek(&@8mqt^K1Ly08cCNE0I`Sz=Dowiwwu0#OL&< zStcRae;0*{66s$hOU+(&Ux2B$kGdLL{SNTL%^ssZm z_yvx*C4ip$II|i#Eu3L#v^R6XmyvPz@>-6LXMGz3DqTF*qIJgYECQo~$n71U9qt|c z^YZlU-TvYE`QbqUFpw>l0i;`ddwB8YU9s-*(fLI?SM+%Ar^92{@T*28gUZ;LsX;^* z0kjA%z;OBV-tqgxJS3k!Db?(}ZI?}!Ay3dv=@_Ebedt0(=-<7*JU;xnh(*H^Y(-s` zPWY>L#|MXJg=s8JiEm~x$ghaA!}E9V&-NFa3eBt4ZW3o+(c-|~pB(-7`@_puN5_XJ zdv6cVo2=BbrkB~rxj?H$Kmoeby&~G$>S$k^r}=yI8x11V-e|7_U7iv2jHYitNg-lR&%d234j2<3_ z0jlb7b?i^t%hJqG2Jat<^~px{1sI?5#{YP1kCh5vm$vicubBO@GxBHLmX)V}<$eA; zFn8&?U5vFeXpZW)#KR(|kyf4-Vkf!WE$q_kFwkad`%I^(0>^=tTJs$0os^tEO(+HP z?^Fnxi~Qqi%H~63F$c1p3@RslX3&V55PzapF+QNhOgX{_+P!)?oxJlVXh^UZ`O0Hd%P$SbI$%*b@4xeTloNH`1H=|2LOZ)!cignYBK_3xI? zZ-Pp>fcK(^avA2nWM5qFot(V8*tSZdzA{O`0j!rM;1TAfB34n@; zFblP#$~|hHDs4zK+BWsrqKy#_LKO{QgxdR9;1WC|=Yy+`KADg}$caWp*Bj7Q7}n+a z;n~lJXO|a8Zx7$SpLedDcwTnitX!Pm(7f4jeeHSMKR!ArfB|~`lE@^zsp_y(L~6uTVwaL zs_XirCDEfL(W52Nqb1RTS8ZdoE4aHlTB`#I9J_5b#ZCA1tIPAl{r6`_7yrE6J9v9^ za{2!J@NB)h;K9-P-cQGemq#Z@7e{-?s^b3HA;<;Riz7Qb{BLMitP*>CrkOsHtY<>_ zKgeCqR^PDJUo&maA(T#6hZ@o*`s+rhr4xqGg?(Ke8h)# zJ=JZ9`qRDh^IzVb9n_%TE_?~p;Y-Yz?%{i!F&$jVIcr(urP`S@k?P!i*Fk(nLW&)y z60G_&*ST1IgoxJOpB=j!bhALG?^4sCMJUpu!YenEwY(Ih60#C8h#MBS9K>1n%L=e+ zkiE=#vZx5_fo89f5bFVFZxJz84YD8QSpUv)tS?|aT`nw&!`V2978ew$`j0}QKeUi2 zTO|)EAkw{lO&R zKYC1GK=4!6^s+*qwB4n}D}|Q6Kl#(6u;)?O^C;|Dfv_i~1uGHsWJkrr2ze}4XKn$H zi&0uuJWv6V^J?dU!6g=Ol>3qbol3+mdU7o$$ob;4)IfmqAhR^T;HDC>k3yS&U!jd! z7>fyP(tATDtg+)187aC9=n76uD~nlK6W*~={u6i1$}b$Hay5X@AEnY6b}lzZDh8cy z&&;oH2x3Y~i}j#$#}|MzmyKh8q^@1(Uu%Oj4Ox~21`ix@%8ak><1h4aY|AlCAjH>t z5TFfCr=bIY^jp<&Y&!{N?;Qck_5m$Z99P6B1Lhavl%-ySjxp$hS;VqJX*0;0WT|yX zH7IJb&~59SuMT)tamaqFA>%$(#SJw?qClg4l5C)-Mj10!Vp+|cF!gD%U@$^!G(l?< z%)})_m2LmVXw+UWydDK1X|FpjT3N%LPoFY~OIAq%>tV}q)9nIOMbx(oMT$nM zC4V|1nrEQMZSfYzIkc`_lRH*j{vI)!a7b@di>>a0SnB43T2?JBD^OqgP_kmxBWOxU z%k)7R{H~4EElFBdKETwQyJ~n@p)WAFwDDCB4TmIwY-1H8OUrzLf$6qaJ+5>ypQ_E? zY?o!FzsL$LZm`vi)@5*C#vptlAA(PxY5+RlKP}L0Crp|=FOFTaIo%rT-eD)ZqYJ?I zVyuD%{0|Bv`}&%0^kwgHC7_fJY3j1?`$3=}vqVA_y1&~i+9oSy^dig`zUTF;%d^9igTu3HDLGHKa_ph?~6de-ZR1=ntkku?;WNJwB312 zEx~=(oCd)TRbrK>45nHPi)sxRk48bHqLk{t3uj4pT{TnLlhrP#F%9YX?mQN2Z=Xht zCpb{DlG2ZJDD1|$S)ddVQWgjUaR}X0@}$>l^J|%VE}QDD z>vh%@pot3U=e+J(V=>@lYLH_1DXS|i$^wr4SlE%4=|n{iN%M3lQ$IQsA~g!f#pXzy zCY_QNUFIM)11z3yf(a}bFSzyLFu6`mc)r<&n(FGc@wB*@8jrYW6jCqeFjN%(>OO4O&}zo=TcpOfW4)@DrCI)vfQvb2)l7Ye%y)3@K{jYy2k>$T~VX_3hxv&)))kob**u0vle>P&3P7g0~Q)4lv%2I{aZH} zr&|1cNjEkn-t=7mvz z{^_ec368u8Njg&+1)L^9H12s5oN$(!_vlld^}KM#I7t#35-`td+Q;VR=H~9s4*a{h zxtagBzq|GPPyLKp>h_@{q2g>nffS?+l@d-_W z-$4x3z5ao91KPXkx2A++O;;c$1T)kc`5TukU`xby+|t1`e!n6V#*2X|BFi@JPC<$@ z2iu$Nn^sXJRB;0_^+SLsNnkQep3OqStYTHXat%i*ZT%C`*7()1pGw-au{1^|1D@Erf1J zGK3;y!o@dmsT6;N0psmob2gN=fW936wpcY=Gn=TGZcCka63ggAzDxPn1MdF?>Hkp7 zM>KAB4Ajy8?VZgW{oi@s-+iS25AiHT|6RoWC%G~fqwx#TRJ9hrxg&zjhT>|XlqM2U zDjD9M=Lt5oRUx|K07Kok#lr5YJrVaFeFr^fm3* z^58!Tw_XrZ&rfSL7OdSdK=&QMhl$3%7H&zmD2XsFw|G{q6G&TEvW;g-`a+ZOkzc1! zNGsE195f9XU92r#&)BPkPG1EfVdp%Vd3=@_!b4f4`DoHKgwJfrVcHPvi{~zSZo&WOMm)cs*dFapMjWh3&`mM1Pa?E+_2h$Q?{ebc5prEdj7N24m z-w)8eX~s@MGTo5>{ES01cnOewbldIcluP0?UlTKT#87*R<96E4dak4{6ht>T4E(I1 zX|N?;q0gVu%Ytr5S7%(1lA7A7rIjx9v`jTrs8(#Z%y8MMIm-J4G{>ZkL>*%qN_@yd zll+=DV=GRcUH6ZQ{qo7E4O>#M3oM_$k~R>^W<+i#hbxJ(!`Dq~zcS5zWi*l|Ko%5dW7Y}(6eei76fR=BdFPj91;V*k>#*P4hv#qxjh=iZDZ zyEN123?AGX=lP zG7=!>!2wmH8C*0lRyD4k77M4d;0lp7jtD>U9YL~YNod5#@@7ehI_!Fc{zu;9J06W9 z;_(hw0Xug$(@#_U5ngBStm9fu&v19gqY)R&RCUsMgW7wZM`AueIF7@>1HPpP(Xi5% z(zaAPoL-a2YQmJ94v_qw=1oY;=UcQUJ>SWY;DjU>HC_D#rHsDpYIF5C4tR)%g&t4O zj(*;|IP6@!`^VwQ!05^@U(uClWnUt`a{=;(Jg$Ef_asL6%l8+rIzP0HIm~Z_r+y5! zG=rC@-3F49U!A`AwkEj{*H&ql1<+YH_HQS@lF7eB)5<9#HGa2?Y1m#uQER(*3Y)u= z+ua?Tdr>%sP4M}1mV^V|zzfA!h_4gQV>ale;TJtR?cq4cM~73>2K!ICK1)AT@zv~9 zzZY$1_0qRW7@#`u8w2QR>fJV8%J505k&LSl4A4Dd#7ED1oet{$4Rt!bKO@#_XJy)h z?%y`-7e+o3yRr60Z~QD>FS)&xZTt>&JH(9a?x2ouqF=4LzoWLUt>#mqLH)9aKSxA9 z6Hz6>`AR5%C+{TjEm=K1KKhwQ{$se)hy~DTDi5mD|381em6QKG-`#ofDF1ngr<}J{ z7l%<063J(QR2MB3u5{J1B!_!i&dX~KG1JpI#Y|80VkQJuuzAg>WIs%(Qc`W9+s1HQVTc!_v2)4>H*@B!qi`LAuDk8`Xm8xC`Zw)6rgiE7d4`5gW9ac!O4XkUc!(lzBJ|qOmcH zSx^eN@-ijh&_8XfE%g%R`F&Ad-oD-hjML<<=vn0|Agk)25^_qWn0u3BV~$E@N!L1d z%+{_G#TiLE=LzOytO8~-^(&8No`BnLshNphYzl5;0gETJhOodOesB&Vrv9;O-DQCO zPykds)^gG?c@yGYv)NQ>d#xZ8I_6RLLJhO~6J`W%nC+Z)xGmjYWHt^qo0=k=vR6Y%U5FE?PgpWq^bZ+heS8?0wn4*1uDP0_w0k9@fO$Azm(R*)Nh-OxJ z&7C|xU;3Gc{|E1v&J;(%h%n9?Ujl0Q|IPkpf&YK8{pkPqAkU{yE@#?z(sLoZ(H7!n zDLV8>C%*G%p8Y=|*bfO~9UZ^5Apxk9|8Dj3`+sNi`R3#Pe~9N(!Ov6AvT{|j*+FNh zqEfMd8IYh(IADb|S}&6oE1(v18q-7j1DGFb)=v3X9i0x)35`hM>OA)qQ%SO=js3oE zy=5eB=s1!p#E_L?dvkLWerCBC8l-A?q-4o|7Sr`jjUSCa1HIPPB0h9rpyC*Xoy z(RVKV;e>?KE}O^=Qc?xJ8Fh;NO>uoyO|GYWm;be6h1#u>W|m{Yd{G;;|D2mZnSja{9Vlrj#$kn%m=Q zs?2lMa;x^7C7Uby?CLAL9XK_1c5k-+u2yB2ue@3XUOw&3yy`r={x_)HYd%1?s?VP`B&_OKucf7Kv-&0GuXRuqBDxr$6uwo$<7&wQDi`j% z*!cW0XXYQSGx>jM`hPo3LZc>gJ7y6mCzSEMFu=_8#W!Bg0iD^@3LuM53cMDX2giXDARPJR_yvL zB+w z(WBET&iAWc4s!5St$k-ZVu70JH%f<^U;HSab+@?Ot4E-FWvx{7Bb*UB|7;&V* zYvS&izp~vFS50lYy*$jRgKP;zg}qgJoWm-?F&+kCzyreabLA&A9-t4;a2P)O)%pf8 za!!-rCJ4!x95N4w%6|}S;jOYd+4UKv9J~~O^bEjY-6(it#N^ITr;)|F$ER@)Oo_pq z{DV*9k(oZWg2OFj_tJ2uIli16Z-1-epk5+5*AESk`-+5JIkvn6#;U`=>IQ6N?^3Iw zc@D2Cc$Uhp)f8*x8Mm4NY2@0{tFw0fp|Ysvbt2CtY4(F2!;W|n|%Flg+j^;Ms=DS^&n3}{eM+4y$_(<{aUUeF8t8#oyaA*ZD86 z>~q_I*PQ>)`^EU*TQ45{e;(w?(;+p*`YhM6P@k#~dvS2Y&bj!A1}_o$mm~AJZX-|y z+o-#a+B}(&cA>rVSqzk2$%Qflg}TOBNyR3eonk%_SI5Sk=nBrp1)ripuF3+R)llaG z90-Oc&pLhbu|$`l&o$udc|oxPgbPs4*-SUI&zv&Ywz&Svrvu%D9dN?J(`IN-2zYz`o2hV9E^7 zIFB>QjM%POI!<%1T8q;Bkg1P1oz|a9K%_IVSQI8FXwlu45zt(5Ox(YktLTHG+ zYhs(rI3O|4p@MKo5z>4U<&?&L;F3^yi%>M14s|#`lZ3m4Qw1X_1G19_a2P!_?b5M> zi@ zFVA-#<3B#glVfRYWowB<_si;7ADHt;E_1ZdQd!wDQ)g|9OwJMO0p*+uitRht@q+TR zauT&&N_zfhHW<#}?d7Y%77e8B9hNj#uaKTw;kc{RzjMP$s*FvHtB12CD30zCr~k!h zRKjC5!+(|4ON9owLV|1tU$Hv|&G+F)PLc>4!r;6W%wn42CYhfDFlUhRl9tI-SQ{jV z@{e&Ccmc@mVT&`<05yZU#&)Mp|Iz>ZL7urI@~dUhk&7Wt=*K&?Bp&$SPQem@bOi`$ zf@MKZNOBn`)FX@$CifMdarz^CCrLt+%aD$(jo9`?rs-htsUyf)p#az;`GrS@;GsF9~99z5duSK0sBEF}uVxlpl1 zn+-2VLC8s>Ca;j(%w3p99x=CF1(;!QLwXJ`~L>Nv3rlP``_a6y*S_Q$HC}6j>h+aNAJB^azFn4p52Vaf4|>PBVTXY5?@-Yq)eKM=(f7?XC8cafNlYkTSf9-ke{!Joet^2oi)O*@T!-U{# zV_2VC=D&|Rf$!;==l&m4-w#+agCwEDnLj4Y5<%76|NGm8{7*YCwjS^Q5AqbyBL^{yU(LK791$SY0q^r)R~_{>TFb)b*Abij4`-_8E>JpJG4Kl=YX$de-k zHe(*MxJ(Bu7eMc(KO`LYGunQx7RZts;FZzoWQaX6j=iFRq^XE?xFWi0I{M1%W_83G zKMPb-AIe7D2!e$82MLY$3*mB}paUg1WQ5DO0teK)Cdr1>q#f+-U%vbK@a*j9;1G;7 z_ustxD9=2w=#Wo<>j5E1C=gA7S4QqA}tV zf*|zFSTxj4@jVTq0qUo>qbdG4K)cVMZ$ED#G-A-}!**I09!aA0@wohVe&#y=Z@e#& z|6yxqJAeN7clwXT;?8p>N+gGi>sl-sKp$+X#qBuM{b>Inc2xG zh;SJEP7>Wm_M&kYnun2E#JAKZ%JbmqB^ta;nH1C(Cl*rqO%diOB$#nTBciIJDfI~& z(FA22fM!6@j1eCV@6cxu`m#W*je>-^m$(7Tx@jn9yLnVVkFwaERgO1C zG(ahh1u1dL2{29*2($X>QyCUm@cVLThXyPd^MV4iummXQo|AuT3khngn{xJ=C2tD? zYDEgpO|V;PW!3rj;D_cp{}UQE&iQ9Pb?3i0^K<9_^WDe%pAYgl&VRW0dI=HdrFHJZ zhZ!S>>dZf*Az62Mg%*!ns9GWs7R`JY=lqN5kM zcsS{1hA3GZ&SD&^qo=Z59L~l;l$EhS0S81em1nb%u=VwII4hOwa+rrwKh+djqwloQJgybbe@$2^ZgP z#Q`h+2m{931@u^(#j#3IS$JI2aaI4Ny!qscLLUm9J<2c^dFDO;hhiZ`bD4kF>wgRJ zf4BSnNB^$}d0gUhb@s31VE;+3mw(&yUF;c^hf&kz0g_0Y@d-_W-=R6>NY~Oi8G$EK zbJKtaOHtJft+sb`A)1z}kG#5LCNsnjS{-CXcT;(Ts==wqF*NUd|C?RcR{qRG|0O5T zA<@lzx4;_uzxn)mUjDoFV)Ie|`yfw_{*&lN#(k4Xm-1R)BbKB1ykICt65Tv7WtV{r z-^KV}Gol=aT(Nb`$T|*86k_T?oqIIg-64@@%FVu?UI*-+$Eq zJj|2HA3(Zb@fz5l22rZvk^5lXE+Wo?*RL+m5BJ}n9bNqMa_`{n(aGiM-ud}2@6HY` zFJB!UAI?>!rtrbh`QA^*hnGhuM;AwX$Ex!F+2P*B(YurNbz4Rf`Z#v0u;dqeWb0pq z+Wi4|SO@jAVuCF*W8cDxlV}lLcV*%)_iNveCL0| zfbY4d=KSB@+1e_^f8Bkw|9_Cj4($FQq1?Z&;eM0&?DIrqx6|7dj^E}{+077@MwjbZ zZrd^!f;GXt8HP7$)EF?jNyE3d*U=i-csdKNt8=h6=KEYx-fQBzr-z&EXlW1Nm}37Y z9OB3$N!tX{6(}bSV^=oqMq?Tp`<=60SGB4TK(2b{pkX-!^Uh+&=GQ^hnk;Lv%&@0Z z5ct(%uiQ~2B@imC(9x+Hit(Zfdb3_d#MSC(|8vCmUpk`y95MXUtv*lm{>n;)07ZJk z1J_-;VNR$IF)G`N4=&mDkt(*nzMl1p_G?*iY!Ph>KANE;*|HD#j-<%DslmG5+7UDnd?mXK6JjmmwVM;TjHGws3pw{`A z+~9KjS~I3fL%Cw!>Y-IvUzO%8u>V!jmF@rS=ezm*AKSZ+`F|hgaqs`PYKl#>tX!ha zY{7Irnh?sm9-a2`q(L>b@I|p2ymUu7H(*>~Ka2IcE+O0ew^NIHEFX)ue8l;{w4D-0 zJ6FCsYOS{WCy^i1Kqbeo$wz7O-DUJy0}(`=BsW&}z9H%N- zFDsj$&`5KG1H_XVfwyTS_L5FxBQQJjCd7R)$v_k(noTSZnT)!qi>EJ&L4xaAaO6+T zjebga5_o3>mPZhc-Eo9-9TX}xUxvl1^)T9!sq%Xs<&7xj#OxP%CYgt5!ZU#%M+ zlo)7n#buzQt|lZ7u}5SI-bSH{76jVdISYd=NYEmH=eKQn?C0DJCdFw_T+63=9VE^b z;EtsI8y%`ax5cQ=NJuavT{VLC7l|M7L+$`@#IKh2!l~cw_rvMApSk!yjyW04LN=!n zKn?%5^P=GYx!r%{{~qQsMaX*7&KNmYQ+*El*N5>AHI`R}3YuccX+@ zwOU@YZQ=k3Jj`qxPW93yAzN9bonkhfTHCJ)!G0*y1?aK^&61?ED4sAEX?-zTUUVQd z^N~Bz{6j$;fRH0=gLu3N@vhly+LO9(D>l|zoCFa^PrDatnBAW+Baqpr-Ag?zfR47M zn3GNL*7IF9n<69JV%kOKH`4iy(Wo2u+|%AqHwelDs`vZ`vA@lJ6TwoghpF2Hj`l!?Yym*oK|K54gf7Jgx$deByuGorJKMMvRCzvEY7Cr=t^L^f$$iBp?j+(bG?}B>arF z{{GiKx<~l-8a?a%UxeOVT%7hksl$Ei^Rso;*$32qDmrYVmoHI!Yjd-WepQi~i?m_E z{XbVI$eR7%?-$}f>^|Rmy#GJQuF4qa$|tqfdf6m-T&G`<^gAUh}x0tvs25RUL$2ZWT`ARP=+R%8MEE3XgVvVimP zseBfs|8s=`s^$N-3;zEvHXrH#gFGvw|M_?UrC#ot7<_ z07ibDmlQ9t{~Lz_spJ1PU*zq7o^NeG`hPsg^Y!=tQV~MF3K>`;q{rMtTXmofal`$- zEfyc2RXz*S|G7-T>-fK&=i7Pu|6+IZk^Vo#GdKPJ1}wxMN#5$7#rFS7{T6B16lY|@)hdfgr%c~H$j8CfjQ@kvNH>ByT~dw|+Q zGNK9Tz(*i!0gX1`7c@eA;(lJ5jw@nERxhb+xrTzVf9B1TLRf@0qJC!9X(h7@ zgJ|}#BffXEZniYozcDtmLF9BNw@TE8_;;bwHXR#{f+*m3Y8$9uMMo#?-Bj5lry)s< zBuFl77-j1H>y5&0Y?uWqT!1|qaU4XDzaTf+d0SvjO~eC>MVbs}J4+_Kxm4-bFB38j z7*FoxmNLu`>|Fp&*@`$cl!ogv4q7txgz}DnTrz zLB39^M6hjY<~RltA`W4$5qCmvNb-m0yuOX6p8xBLQ!^MxG$9N2QFs4)UXcH9Zob%g z~+%y=y!stlN5mE+zp3T_pJPvk3`9G!E%d z+Aamrcmv!*cyL3&naX&NBfs?oMPw|K;;+RC83iASFAujruXm9kIbg{aOOQ}Z5)=jz z>9)ED=a=W4CZzQQ?bGR$M(F4LbL0mJYjwu~@4^2i_^s~n_oN5^*DogHp7@{s$!?-v z*3b}p*RvRnf{?J*-@5EJZvCx0#MiCAb@?=I{q6s?o}iy`63`h#M+b+j)xBXJ^+~I% zbfT?p;1k@F)kIWhiB*C1=*_K|7VAL2X7C%Q-9$;D))bX>&1(_{J;NVYx^<& z_k%o7w4Eb*L&7QA7jxfg{S6&OjN>qr=>l%a5T(Z2f+R?W3Vl#@e_d1NQTmvt$+)-P zYCU;^UK5VaNKDaDG@`B6)zy_pBSu5g62}Tk#3$_gNYzB5A!f8R4)|m?>;|+a%6H`2 zH0sV`A9EtwZM81X2|*vyZxSlen7;S<)MM}i{&y(~_13`-R54!2r``;VEI@*P2}4k* z(AgIiiY_P$C2o&vVQ@{*O~1R<-A0|iqnoYnW_NSlN`!&A03{(PUYF7ZmM?=F!?-)rxqXWXcB!JA|wWVcwneKMp=e~8{Y5vSp zc9HT%&DK4-(jHk?_2WO8O@|~A(-?sfeO>vher=;=_(E58nd7greom>6{Io2Z22n7b zO;MKNJr7qDgo{8{4wscs{;CWv{1M=qMNY_;gexPY=RCojjPFXC66;tOMNE~gY8_pD zLT!E*llB0$&nQs&_i(?*j#S^Apr z_5giCZNb_LAWd864b&F5g7yGyZ*Fd)&!`<$&1>A}3=l zfvcU`bG8R)=f&0rl5fE5AbmA5e~YCecTf^g=t8^Fl!11SK$1pB=u}lfPzR16#)5H# zV(K#!uw{0T6zB@>i-|i&+5Qpz9EqYRD_YY!y2^1*O<^?P;&@zF>mI$kAxRSWikHzW zG&F!X9;4d{1)fWuV*xW6p*U8W6ts2)UP(hD4qG_>uhzvN7Nh3oAlRgK=o-0fg3%fU z%8A4Es`4v!A6)O~P6;(NeBIy-M)hsW6#gkd(3DE>DaZXjXac^D`G)&;>UkXy0YXx8?; zz*%q^fk)l#+@n3;mjuiVQaTDq=!*j)58;w@q8MHu#g<*&rKz3jTz zK~O8^(XsAzEjL`t9Hx!{(`EChjB!lOc?AIz+(#wX*$gguC9UMv_GgUKslL3{w!Rv; zygFKQFXNPn^`*+Dvw3vthHDk`Xh)iMGt0WrE@~jqak*ad z+|~h3&g*FNJ=&MHa~SD5l}GnIx&lYC`9TA?@}ZS0=+~a_2a+930h0Qj5lxUvK5OXb z7e<)G7iD}e3@{@#l978cXAB(Lm_B!6>OW(sN4VD$?J&FiQu7!@W3fP*^Ir@d$trcN zFE@`Gt_Jj8!(-9=8!$*fILXueCF!b$>szDu8ZL38ui*<*e0d&;Iq{_ivjVQ(5@I(K zu8S&OI)e+CXF93|OTcvss@1uHv*3dLR4qm-;7im2^S2VXKD6Nq+#w(FG!Dg?;^DBp zicKnr+bQOgGKe!^p(vmbs7RcUm_)$Hk{goT!5EZDi;{MC(R)TlvrwwV^`{^jqYs{3 z$x0a@RUUi4oTI(tpVAKZX~Yv6h9t=pQr9Vm*kQq#8CouU%Q`q5cGSo^URt|1o&`SX zsSmwuQ}>>z{@1(cOpEaI^T?@G1ki{enxH97h#V@3Ak08_RrHcqZhmVjE{@M_OqY~j z3a->0uU;OqfBL@nmP~1Kmpb630cLt(Fb%8&W=Xna8U@zSr4#C>svQVgLw&{dfh4g| zekr&N^Sruwyhw{Sfo{op)OL;y)F*q|Kr$8k*1)w4zJQ(_MFHPW8KUy@IN%1~q!oOM zm(k)I0q3gVs)I@Pb6#NSszAF(SMBzy;QCn_7P^|dN9vHn~{w$QCd#lf5Q+{;)14C(5v_|=aBm$9<7D_L4 zBQ)T0gkCFK_wN}g2HFy9i*a!b`Iuv?{8IdJKCWm5Uv;7RsjLcGyNYkTD{(|#8SzN3 z)&VZ~H2+h8iv_<|!F3+|Zo`%0%nNZaEtJ3NZ@vyzGLL4s)_j9<9!>j<)1UW&g_14E zd34B==3Zw-eC0yIUDd^FJL9AU_Z;7Noj9SK z07KkHZWI>R*KBxNdOf^HU{a4TVzVLggCrHv7na+Si@yL@5J`5_Nnr2MkxjY+26MQ~_)To((oyej5u0aeqQ@qXW!w)d2=*1{`c4pGMEPxEl~f?dYvwJ0?MlFh}6S zFpJSWculMXt_n^0JrYGbo+#S^U{XFv>7i|{`8~QC&e+|M&Mg-xh0CcauZN4{u~q)6 z-`(za=Qm+0h0Ce@u7}H3gfGeFy5Mpuzw6<$72(UOyDhjXF2_Z<1ovOK7o)!-bu(W0 zoS2u*-3vjC&xyg{f}L$i`K9OtiEbJaZ*&NfB@*2P36*!u8=M4EG#q2b2;8T%ADvloo3 z6EY%+Aojg6iAW;OKvUR-uNNZXBO*CT8zXd%J*+Ef|hNb}zC4#~gG`qJvK0owzSbzjmD?Oe8Wy&~mc}uPtlGCQkTVkbB&l0h z_voBN3=GB_ZG0U0^41F`+iLMOA_WRA(W+7{9tY8NmAv~m;&F#90uwzuf$c708_D#ey*5kVa#vHqthvy8Ke>OYm!r)E@V#V@I~CjLK+D)2~TFXX6Rt zak#!1>mr5tAxXq4ZXm=j!lf&(5U5NU!-G*!DJCz6E3ND-zZ933I^CVB1S#e8-ZhV^ zvQB%Du$9AAhp>BekQP-khEfB)mBLlSgWQ|kM#{7*O@rhmScbqQE;kY`?adfOHFDZ} zO$VMa6eqzHCwG-9(k0_ZAHxf!Oz3OFwbT-8pi(Qs)xb7rS-G^8y{PiWbxQpvh*}w3O|+02u7*Zb zD}n2@&V*g{W9lI@)p2Iet~J~#Tyr|>Sa5~GhK;L?3(rTrY4=3`khpC*TbA+Z_=6I$PY=9q6k-o*Hs=aTYl_>tKaSa@9x%O5@j2%3P-9uTy>6AOA>esuG0!XsytkE zj#NuZlr>znhE^F|Y0>J|SF`SG-7_<|8u(8^U(t-NK&DNCI(F9FrArIn^z$pRJJ1KXnWfv{v zWvqD#`poMCnPEhqVl@^L>43 zyK=&%LrjX3Qcs7zs9B3`DQ~?ZTxqV&$UMsLkz9!>FKr#(1+Y-owriA2!9|mCkWS}4 zI`TC*G#TS4U@|bLSkSkVz>_gXjEQ6uIt%?w_}er*LM>b)8fL@1N5^y=csNAzwOAn% z9F4RwWQfRJA{RISYT@z&CRWq|tk)}akM=ws+=va!DZ)22@Fj}GceUQ+HJtc4=;pez z)xrhdKkow>!GHl^;*AX11A)avNfyw^1_U%y0^;{ai`*k?;j*m4?vY-*z`l z$S<)UeM{{vUx_iyWQ-r5aN_aoG*LloSQq^g@Co{5?_&Q=@$m8~!s?sWNpDDr z>iO~E;VBBZECW!u zII;m(Mf#~j5SesTk+#dUbiEw63Pw<1mjVjff~CN1Ky1t4vuM2m+N+e<>x(P^sDYrN zq_rLmLr9ipLb^7Kz{$_mpEAmo&C5G>zy#m6-jy*I!RH4Sa%Kb#i?qk5aW5y2tsYe- z_PA1`x|o3L8OqYZ=)fBa z3c=HCq?EwZZkKE}R0U=c8~_tixfno1FyrQ&%7u^+2;xSt-*lK|gOa>y${MH4M#d}y zA(LEvOuS$e5PzdMV{RT-&Rqv4Pe80Djv1@^r|E}`!7)1v7)oXlh*$OE zoH62ekN7i&aL6c%DU+VG*K@6?gC6kkN3)RcKNH z0R%9W#lZ#aBk{y{JQBABQ^7i>93@1m?Usg5-+!-i@SRICVj?&MNmUR5y5Lf^VQxX1 zpdsZGfJpq&$cn&qiTcPBTu`i9+EjzTP_S8g)P@dZIRH{kqx%}y^cI&XESq2BDsOQa zT+Y+9?l8^ZX>^+E;tM0sd7L&oNE?AV|4Ev`<|JuLQnU={KNv|X#iEmBd?A;+mNi&ndrTU9d4Ekss3_;&7o4q8oahC%ZRApAR6}!hYMJQ@BPr#5#L3PBTX|9a%`FkVKqblgJRPtR_rh3RkJA$9*nGVV$MN{rmt` zz$Il7ZaJa~50yt$S?epsS5#`|VT6%W0Ks;ZV%!ZGBLysN(_>6&AGHO)t`>|+u^~lY zh=fuwe5J%WS2DI>8N3&SxwX6H@JdfZ!uSIK4#%(zNixJ<#mqa_fMDeWxMx4@?N?~e zms|q?SIRt8hP;w+F-U4Gx!x=xcq$81YCA2AQ_(%sD)P6H0-B|cpPDm;lm?MUP@ZW9 zP-l!BD&i(uDLGM;I3A;fO2xLG`;;(>52yoz`Z)ca8r*l#aK;hiQrRHSA~X3djK#=@ zuDwmt?!=0ikc4y{s~~3aVSRuxwF*wDf37SPr0oY<0~Xs_P#Xey5r^&>ib*mJ7z1L4 z8M-B*G?jKQ6AP~QjO@knUNS9(afO`)K$9Gj;R97wrn-b6>vG6YQJp{ zTE{Tl7K!)hETtQCRExpYPy3PE5ceMK&l166c-K}4htk&3l{ZU5D9C!^&$%pj%HS#x z7#CbsCb;+L>P7d(c6a_r+m&z?3~^m>Rhi%}X?|MrSWDYKYueB>P+pd(WR%D4lxeMi6nOCsloR@K>RB@GtP#P}937rZCV8#%Q zNrJ_PrCp11a7B4xu>n^)U@7;{ffz8IxlPiz)d&VOTo8Nw20O*=Z|E%=fw5D>!#i}E zp-G9@nU~7~OkxsQ_Q4P9Phu3Fj3)*>l0gL*(36qh9=W_Ix*G(don{NUp zn1_5AEfcORjT{YqbS5oDrB4Ax2@wpARsO2KM4l5k=w#3gQbq1FJ*EutC8~82RnI(? z{#uvFCf8`~so(&!$|veR58HgXs7k@iWvJ@dyAw1vfLZ4xRSTxzuwC-hv`874 zZjY%FFq}R*@R{YO&^!lE9?UusWCfV66k!Eyni*Lt z!rs+^S;ykmfLXLn%7F>s&6pG=8!KQ=Fn`$mp;A0q0j4W~ST!~^xN}bbVqjL~U7I)M zT2n9^8Qs`mDlX;R6(Ki`Z~>8VwN16YV`&MN!nB9OaEQHYunnvRbAg>+6`kPT@@aIf zQ)gU)8dNQ`g{qkMdlaSodJahC~5Z8i}jWgrrhoBc}jckZwA_JXBCN0F%zR4JW5H zP%e1MsB(;#>ghD9stTp*;O1hVHa;&GV?C-8l00w@5*x6pj3f`7gd|78D^ij?a267K zD>S-&oBrqa7D*Ruy}#&ukUF=I1Ag!vK2+KI3gVUn6@?p&36K{CX}}=But%eacugYp zEC9cQO1bd=QPAsvXJ*IQ#=1v)ruGbEZThQpL6UVtv2es|!<<)=QYH9ORLe`f-Bq&1 zO86aX%mII(iM_KCgKsw15RQvdj+KUiQ z5_|_~CLk9grxN3tw6-+)Uq*CfFiRMPSOY&{@A?)e{yv??mgwtaPs5fG4sXyxDn6(93n8CQaS#oVk~Rm? zxT~|pd2|Xz5=!~yEC~mw{g<^^F~{p{(ChtWO;%rLTDsPT8qmR|*gThPHoO#Y4!}RV zrQjERt2L)SAjpo!Bv2jHcEcl@m#ww49f!wSHD9N|aLjE;chtmc3SBIyDTy$37} zk=kvgbJZ!rE|AgzvNS*{0xF;hjC)T39laZz^ul1MvC=cksdO6yrTk#dLe>}t>CcW9 zn?hH;r5xGTE2!ddS#`LXtnd*YoML)sz6%`qs5~S9kbn+2L29A`;GKP8+dYhU#VsE1Q z8D!I%bt7g|nP|$@E;`7JA>=!Req%Hmp!=4lO`lpBooX94)9pda_|olmjbBfYpmZVO z1eycSa4?O`)i9gp0S`%gfZFF`+^P1xyYPho0Zx+TO80Dhk*hpnm%LGyEmf zHx|eGMEcwZ4B_-zizT#~ua7xXevrTcerJA%T$kA=)wXEGT$#N*5M6;9D4glPJB(vZ zj0AXRWHdlJHl3!X9-SI1v$v&6k0Zj5`~lLx^&K>OlX<`m44oc)1Sh*9f`Iz!{u6ww z5?Ia{Njw?}0tV2C-@Qy>(wkz&Nn!$-8@1j!Kkc0#?(SGS=fHGhS1gZ;q)sE026+Z2 zG;zQInM??uB@z0s^LHm|BEmb^GKPtApo@vw{s}X(yMsvNQJ?s7%GXGD+(oZRk_3Fc zwTB{di(;JMDd8kR6U@+%kjS=Mnk6A@J`XdoV^J_dcXXDzZDBOr-BFlQ{<-8|GaRxs zaER1#>ZVABOt-X;gR!fktSW3Z@FIti1`D;sN>( zz0U6FEEz*1a;|&0_bE>3Z^YvrLGM5Jh6#?m31PiY@~=+df9^s1y-(sF_^r_Xcf~K+ zO^C(OO3R`dlcsC}x@Wy-%9Jfgw8a{P^9P*Vp||hPFT|ouNE~8MN)rFF^-9gY*^9tv zx-~XX3afZ8w+J+>ps?r150ZY*50b5H>$LPLkqaqcl_(P(|7Giy=sLrf+L#hmqKm(5 zy%L>>e{?q~;$VmBka>25iI!Yw#MmE|A={+C#D>_u`DBjbo0J=8CWMO8wPNb4uqat^ z2s1<==e+56m~+&$TxS)MIY}6zGwGH@BO+mtm0f=F2su7*pay15&~%gL}2i=yr(;07iV} z4(xYZtv!?wk4~o~^5tbJV47>7kQQl&MwIoN^^{qlgu^}QX(XE%oV~N4N$w^Zt2FZe_**&dw}jcErdRFyWL;Yv&`Np zsjqqUR&m6|NcTzvnVA;fDPh34+$U+?7@8Q%YD28q*?6K(P_;ma@7h1D(*1*K_fI~? zhI<3G&#yZQKB)VM42(SJK_q0Y zBkIy*EJH8^j7yQM9gE?Ky0vt&<~z!%Ud+V0bv;ROG-#O^>x$2?Kh13qjJ;h)amxR} zxVW@$XnCRkcS{?T)k$-w`|8RsX~pXV$CH4Qss;^giHmfuB?K=AVnx-qWMm2L3d;EL zPB3Vab46?ClEtM7`YEBeEQ>?-{%F0>scdQ>Tgmzi1m{DOF$(;y3ZSRl^Eegnk9^b@ z{0i`gIzob~4xh7IR62sc3+^L-Gq56b{IL1MX0evQ+feKby{b1v&oh>GsNWZ+)2J)< zN4}nTS5-&F1GDPrk4C*ArQzR|T&Sm_Y@#7%fd?57R6%&LH{%mWk6Gb#ALnbmcf)Gb z)L6*ZJS}3!2{BM((?@$U5-@ZXG;EkW~$|C?gh3UW{cjFllU%D)z+@ z$>tBnyyRozd{x#(0;A%JdJ`N)B&_6N09Xx`cH7b#Z+We6T}?RA zxv3QWK$zA4pS^cojvGf72IuecDezJ5Bgvo0s*+^cUD57yG_t#`ZC@0Ud**aF91@B| zRdK{3*#s%O#?}}6f8TPSWCNK12$Eo3B#k`XNBm){NL&&LAd$#xQMG>;3=<@N>KJB& zfpR1RMJfggBAVO_7u5=PwTO9YN$%315`?4i?P6vMu;Q6m@f21}PD;;~DYduN@{oog zv%+H<0D3@$zg)uaG75)!%CX1>K|O;bnS%33ix)RYTD``;-Uz<-T|PlF6Y+B;Y_6y+ zXGwUTvssjcEH~auCzoJ8Bm+Ed<(YOaW+T30=QGARov=#uFF-hBj9i?RnT&CLCiU;& z-l>D{YOOq%1(Cop8cby~mSqR*odg-H54BV^h%!N=hh+1=|NFl;73-H?vSdg$Kc}BZ z>Rp+|DGkA}xOU>ygsLzBpz?C?-hRIBL|P`!oU4T`sL51kzN$FThr%mVFo0M|n}JEv z2_o|>XXEHI*-~qI+i{K)lI9`$bK5J*I?SK=J1E5(_IifCO*Qu8_6|DE*Cd8TO6F;) zgmZ9HFcet}RDFi}0_U@y?nT!0!Dw|z++-3ZpYaHWKb5}RuUE^Np!Uo|L&PEJJsK>n zQ*!3zY{EVdi7S9}9j#&1#Nj`;{Kwl(1?A;4t`(CZyK)xkSStmpJ(Qk0H#U$Qix27y z(IwjpL2x|r+(@RTP8dU;6D5)3z=hP&w%q;T zv}}lR!JFEW$F%OCHIC_oY^|8Xbi1B#!POv+Jko{C=|tL00c ztM!XEFz1nY_2~RujQ2~A6+usS*%LL-Ywh%Oo$IOr3H{oQV2Nv76EEu_yLQBVRfo{2 z@?8r4!Wg;dd3He_&W1&bCjjsyI39rGVUG?TVec~fd&wMR;uRaNM?EAzz~ukf9DVfx zMsu75aTzkjJ+8JwhAs8ES;M5?orCaT-N>Uxym-50&~Q2;xqg#|Y)Jb1`~8;Md+~fW zwArw_v^7I(9Ha!Ml#(OO0iG3UE;ys(q+t1&28?VSogHuMEp6NUlpLKMlQ54UN2@>a zNzRHqVpr7~QeAyG{AZan@(Y`jzrnerayJ;10u+%zD4?(!tp_(sFW`}fU1@^gQBiJt z8O+%%Em+MsJW3|k`CIHL*Biy@NGYyPUmm`A^U|M%8+n;{QR1QZy)mgZ z%gr_qpjUZgR5im$$NB5L5|`O*$wQ7Z0kEv1O>;Cf+yfh9xSn%uK2PE~YR z&WbWal3PojYHbZi^NG$KxTLB;|1*m6b0zF;L0_aK9f91P#xWE*KH;*oBT?a#-_lY! z)=S+kxw!c91*4%{Ute6{fmRriD*;5fL07IXe&;cdMly2x&einc4s5cjgiPsq#>6#% zR4{mIXv6RO-|q{K*4{`Z*!hcmPy?KMm3>^?o{@9pgl{?gywA3WXv3+XR!`u0!B3!49>f9t-QgZqO#U#`D31P{-* ze7g7ll;$BzSok&#*~`x;BIzB8Z$Na{=wsRbf4bM)|GWGB!NdN)kEbE}2xvj$bm9lJ zC}QSEX>S^ZAxpG4!dB$695GL;A~t_~P6qzq!0&r~@&g(8yU)a*o7So+ACfpq%Fo7E zxg^EsK%8RYR2&ld)?QA#NDK+KZLDVxbD!BO<`~<^Lnc>khOFe%EKff-Z=^KK^v}LO z@cT~4FnJ8Tg3uaeMyToVDtV8Sl21?a^t06Jg3O=1plp^VXADZI68b{QeFwp^?L>LW z1;acP-Z@tlXuE#jS#AE;-2VpGwfY)cy#EJJ2K#%>{eQ6ku>bGn*&tF^t7Ku~Cehpn zzGGTM|91H1wKq=l87&HFBP$BLU_nfC))lZ2G6Mi#tm5}^-*Kw$@kVF_1q2rd&@k#9 zgZex0_xeOR0oeuB`jxsc_yybm1#r#=vD_{VYzuOOjF{PC5_35)G!NJopdp77*eQmrRc&`Uf#K4YBv5@0T;&en~Nn?vx z+8Ja#Q#b7flMXj{pj;9|r)(1OBA*Y5>w-Y{wV<0;2H)8rhp355$K;I7q@s-%IgJu= z*~&rlv87a%qqz-3Z{Th{i;|G{I5e^ivtZBwo&Syd&W5u=ev%p#6@Olw43c81;B%m% z)e-_s$TnrcC0_a%iI84OBSF(uPF~11x%O$Aq0OhWL1;2pl$JNg%a!;kB$Q+RV(IWb zqZz{eE)jjBGz%KtsYR|v!&;>tsv!V<@+BHVmQ=|*(8|)d8?sMW6T3V%j3#Hwb~OiE z0?6){vyg0sm_kxCA5#oYEr*wbP3ZQH90%4{eJgYANGe>g$G0hnhc<{O#;cNzP`szA zO}Rr}iJ!Q&;&@Id=jBZ1oH3?Tkb`3#~9ITnlwyKo9 z!8GM8@g3X38Ilhlp|5Ichr|UAW8gX)gvf~*!rw#%c{@q-Vn_!4-6y;9gFIeUIU*Oo z{@L13)rTlSMaS7?lwl6gcA1j1dP)J)x<>8LR@g?^)<9<=D07x34D<#Re6>cfI4fJ< zt;lCSVMP#bJG%6cxcYD17C03(vOTEqchI^L^Nds=wa}?yZujZl_a;4KDxkg$jfSmA z(70|}lVqbY9nsFIM_04yxPOiT4E5LyilT>-f^#1( za=@6x3QDIlm@SEI>x3cpCOI8(6*CWYlU&Wm&_U-XCVh|9?4nTy;5tjQGKNsx71B4k zMlqL$Qoe4nOXgMp6_WI^ia65W9q5cl zNx(=1(6Y}N=3kXoD^>VTMQm_E$i<_%bF{PKk~0_m^~ej&%)_yZco~u}*N&qpiEaJr z-B(2_)cq(fCW5%Iyr{nDy_q^h&gu7i2j4@B$750^U`GWbYMO6?IoIiH?U9U(lbTwk z>{U#i_vMSG7T4Dv6!yP-5riq@G+@`)KEDclB?bk}@9@hPEGGcFzQ(PjiT5lGQ39>b zaEVNDtRVr@mR@ODRDNiDV1kn}jvb=+k>rQ~KfFr5yi?;B z1+KMSDBF<8H%XmoJX>=1TC3PK zf7f6UVcO6R3~0Sv2Uig#uin2uIdk+mg|-Z6%i;G*+ zLxg-NxMM^Q;8iSo!Zslf1b~UbR_l!~o0lphZzEL;+J%Jd_fmYXC9_CEBnO~yRklZo z*|f?m-kEGjvxTF%^j`yw0WFJZnn(X+I*_hHg0t%JPDsb}mc>*QRpvx#ZJKfBjAMf| zD!h?iyvo+|lu>XK7K4FhtQdOfs#@9R)g4>MWfCZVpO9ra3(P?6wy?`IivZ;_I!7P3 zS$c&vC>fWKYhE(t%1W8hoaOHYXg@_UtJH~>)n_SxBAx25Xm09Kdv$A>dLot{Ai`aa z9ckteB~@sRZ;2n&xh}!bjfSIEGAy2@Q-BgB7a^;xkAt>txOgtX7P*FU@u*my>k$BD zs(yC%T1Q;OYa#BrZnj%eFYSMwCP>HeFiDsofi$0EtYwyCwjOait>RRxEhtT##%YO>K&31Rb(gSTRG*h@ zUi%%;qh^r}B@1|$VF-Y>+(r5gN-GTsNEV4pHOmX`t3w=@6fu*&U}O)2@HDBM zKXr1MYAq_(5N#ov+?pqk^uXZJT2n3$=u)S7Q5cdS5e$@nj>Y4&+Wc7rBrH0w*NeW)a@-kSIG9n5d1(XDF zT87#bjf|t5eG-6WIlF?!PtwqPpL!5WSIUGXqo%^{$Py=Mhy!_*eqvYB{|vnF#g#Um z3;w+USVNK{B`xUjpU??1q13o*;++=mc{IGDNgxO+24!M04Yc;|kt^ER%iy71HD<=U zFw{=Ix;0T~OSZ%1&6-i*Ta@747+clwCO4v1$}p8&ChO)qI^`*x^3WMDN8h#(uFAzLkE?!~;eMqPmO{RsVJ0AkrD2S{Aak)RB_>lA+2kvi6uI?xFE*rnr!MCSEOKTkgUpm)Jn0` zXB)wSZ|=tx6_F3mH+k07|7zV`?|jPQjOE-fvSq5^CHmj}gQrdVk0<^9gZD-O_uLybz>=`_ zx3fY19wlM=i4*DdB?i-b?5MX|!YD{K4Z})k3i?oS2!Rm_@+dowD1z}4@wGue1rZi! zdYsZ)B&uM_!ZKFsh?Fz4Mdxu(fN3HazIx)TXS9y^_2lK+G?NimJ&^nlWnJ-tjmIn~ zhQxiFo=E`2YbBK!68GirC5_!WwmQI2lI_|YqqY&^Q|64gKMefAzSnn&e?&918IBnD zf2p)}{?AdN)Owc*RiG=TA3^gUzw1XP6JN@fCHwJ!}f7VVJE z2IbhyhFf?xk*h62w5SQON=Sn!wDg;@7#3Vss!PYAT8@F4I~6i#)~C5(znDJs$jdD?QPp((5i@K48rermsRtp zX5OeLl~>pxnaUyTWASedu1Wg&?hitlGA@6|U-r7#+ul zB@1U8dEapl$VH7J-v|1vxe^gRI}W?YbQ@f*1e~@4F%AzTsHd;t4C$hgaNhV8P9BvU zHMl0vV;Ak}DowR-chy!&-9}rt)7^@ttO#>BaG zZ%zub6|;nFfhn=2>+NmQCtnFDGV(duqFE*y-D)?tO}6aytz~Zjm0o~1g2}AC2qGCT za0}80Sm3kFxli0yG1uBjds@Ug?>WY zgQ*%-<5{s~+xSeXRzJIK#!-I5kZ9rQ)+p=Vsw0a9)18h6&4gAR50-YjS(jUbRx>VY zf>mFMwW5nwW3!-J%(FG8r4AM(Hd(zp!`4YJT|9C+@#tQ%`E4LHy7VvEoVdz@>aH5C zvMVp!wsmx<@!!Q1*be}Ee}C#ZSVF*!Ka2D?I*Tm~ygIz@M*nkT{a?c_g?ssk26o5L zCH~(}o<4cf(EmL>c+mgd&jU+M%O)1+PW%PZUV`$`1_C@_1?9UdfJTsn|1N8{aqM!4$%=nOe z*u>yJn;+}u9^!qdOkJT!#5feXV)=*1#1gwtccsBYHy<9W8~8ZTgH@ zfi-Y$H9_kFb=xTwgX-80t8KnT(G$_M-|9S3SLUpZvjsBFZ-U09!CV`|_Sak(4JxwE zxnNEZ9typy!H5-9P1Z>op2-tI2Aumttze!puy4=u=qifY#5BJ&tYK>jHjht^>V-~@ zUsNyhGCAZ`f&_6E56R}Kczc-4H=9uqTHkMZ+$o2K#_Y;ImsD=}WJpri@=W-$TJ^8$ z;Hz+c&{&QH!Y#%i@1(KhKTN#5DR}|7!9dg)K|s0mcw`nOL(cSg3t2Q#pppA4!FQI6Hfdj!5V`Bb^jqy?=jVv5c_7-3SA^ z>^5nW*6U&YbC#uX4^GjYu+Lqu-0srR|VHJqh1KDS#SxkHpWssVZxDz z*0+4}>v4T!2cq(d& z%W9WmuR0m78kL_t8iu)O_uc9H?L{DqI>3>7;z)SjW<6E^W)PAh)!7M5H{TIc%&o=g zmloi<=Gq!;VAV2eGt@~F{E{=TN+7aetPABJ7JM!&Z`WU&aH$d=zl<{P3d-CVyk%k< zvttmsc;ZI!{05OOZVY1MKB1wB@zZn662W8O@p1JVfBG2&Q?1v#Wb-A}yh~=U8qgIw z^#baNVzm}J#@%?uk}GqoBNdeU+@K%eMe$Rf&deYe1mjE9@*x=SF-i$9Xi;(>#TmDv z3J9AZ1x>()#70QL+JEurb)$-IwOd|f5sXHnW`jy+ag-(n`&^iK0ddXSG!fVOX0k;? zmO}>pe!q#QJG#>ooe`r-+)+n_qPwD^7e=`OVk=QfZ8hFtyJ>xoS=RLV?bc`LniXp` z=r5e>yGEqoZ;v|Jh#<}8fJ1(#97H{^!E#ojQGOXu;SoMH2~c4C_-d>K7EDv(o-vvS z(^b-7W}De~-TW z^76NZ#gJY?i-)Z?jndV(vvA6tkb5!W0to0@v`08%)lFe6Yu#K7pP*N+LtwnZ}W)SH?m&RBykD5J+hK-l+ z4^Mx7`QBW!L1wAcszJVfZI3XR&eD+V@9!@x-wC=xpM<=Db^u(zv%}r2(k#CW>h9v* z_@*t4w`Lb{ez+9t?TVIKOEJGn4J(UP-*;BFqbWSEqI4kw(i|@*W;{e?4~6Mtm&(n5 z5)E1Sp&9{N9k^;y-4Zq#*UJW4lVq(9-O=G)VoygJvjW*z6>T8bxXUdgpWTFE@E@L3|e$ zuNE}?8yLJ=pt@^o!6noA?_@mr?Xj`4>7@DcCbpB>XiwfvmfKL?iCJMKD!3D~!j9%{ zV^-K=i2sN1#&^X3EzAhGjQ@N3w3+|!VE=*tyPxNq@PEquuVvA9cO?4`EcKSqY|>$< zhhd&HNnhJ)uDjbiFq-Rbylyi`oFyWz1-f3lruU9b;N${WY5->`ci-l32*Oh1w|225 zfUaS=X5~WvsD^8GZTxMt&WM&JR%>RV^=;NN9qH?K7HifuwZ45pv)&5U1B!z7an%Ays|d0Kad+-EqC`iJbdJ*DbK<(B5sXYMo?}kjD7m6> zWW)oNxZNh}*6>w_GK!03rRyRK+#njElJps~rQ)6*feW+Ft=-wj^Y0+JO*`+H?BTg! z%U!Znr2jWhlkU-M8`v|W-|4YD7j4w|hz^V_iEDE=Y{b8Xn^q?yp%U<}!bhxNsvk$Z zV6{wHqt5Z|@=g)4e|*J3lMZoH-lh2-2pIza5>CFv_{(Pokm+|04K~Kl*3h zgMZbF>7*zAQ9t=r(yJO8(Lhi%cu8SP>5JpG5`U-2LfS?`-~?iHRj_y6h3!xwK}`m=DIJ{Ir) z-GjkiGymt{>E6Tne=pBQE4jDh9IBk4>U^Udv-zvnA5=2rk6YlPFzodvQ86t?aD7ub zgjJz!%ze#S!AY6b^HM`TYMojcO-X1Hnn|ifnO5@UG$FK>A{>u*-*JwS73JhZb6^XB zm5aW}8S71^UM2`$Pf@;Jv#D(m*X4{BtPLBCHlT_22AQ%b2LruLP7;|CTnrNR(vX+5 zn6~iXlSA^sa;>ut18>j0Oc(c564332{)8Nzl86%;^Hj+_tK{Tb0%-;Y5loU2K7vmo zqrp_nN|kS%>txwF1GeutzlxhGp$T3-lYn6*%{ReO5*1V>0vC11X&k4YFhx2`1i9ry z$0NU`B}teyUI~CiRmyI$(58@;RWNxYCMN-sji+V7qmU`k!7*ox`4hkLle=I=5MJoE zBeBdBAykWOdoet)G@)&PMqJtjDC~2V!Zxaa80rj zdXJdUJ93AVoaHDXW1lk?6u4?KmXFhXhBU~7bWjq4CM^OIe9?JTy#Vo`SfL=UyAW5_ zNL(U0q9e=MmEa8Hxh`1bCA;vQ|L_0#f1Mx6#t&e49qJ)g;l|ZLBV+BEo8NmT!^TS2s^_?cvV=_-m zjlFqV=K4}?Tczq&4B&|QhKn=Oh6{0vR83uwEq28cfl<-Hj3y=s|9a zbrcnt#ye)}0{==hCeersWt69vEP=p0%#75b;N@j7MdXgl9J1&_8+c>_ZGtWY*@nY< zO9Gt8<#@5v7{U%=Mc{Ay0#h1Sl6~T2!V1ERGPu-o1AuIYWC(MRCLFSnPno7RVq2p3 z=qI3V3K`IDZNxHQlsIp*G!e@WNW-GFT%;&<-a)L$4G_n2?O2uxe& zhBb?)TE#vATg5SvB+QU1EY3y9HpU7;GUbWZ#G`;{bq>fPF*!QrZgV zD>+dekgwzg<3S#Q!#Vj%UVf&tEN0}Z^ObnwA@YB({tF6RXpZB8d?j#f>s+sjXB{yJ zaDdz`3&~f>J^?2u+Ax&dsufI=om6Zf^6Y4P%4;02ph=%7D&r*+f~?GiOiBHNj3xHt z@CT!(A1h8qK(xE}^!p1OQY+`V0!n`XNsK)Bstx*;T)6IqMD;tFP-r2lU&tw24b^sA ztqW*JY<{s<@8CYt8^!6UH=}}F^j@Sv364bq;vUT+e->`k4GL}dmxb3AgAZ5JE7?`^ zzCvLyS42%5Ox4|m9E$5vsK+fqz-DARrXnR`g0eK^)t=VOhqT~;Fho!b`F~tn^(BdR z6S#7LY3P^_X#yr@xVZI%AtD804%88-d0XIQU#f^8JY&H!&Q(w1 zjV4yX3OnM%7qpI%sJQ-EAGWqEQoBQ2m#E1Xgd2FgG%LZ_#~A7+CoeT(Qt;yRk$hBV z9m{jU9F;%_a+ZV)G`>>91>E(RTy&ULUBKm)6@Q6)9Y-*PNkLL6TlBg$%eaFXXuh z?B<1*b%+ksC8`EREl8m#^)f*grWM3U&}vi8A?fy2Bxs%rA;lHh1wtuD0cZWf{+oGa z1t34Azysea9E-xp3Yd<^>Y^#O-A-#qN#SH$4g<@hA%>ABXC-|x;zW(@0w76FjUG#m z`?}d7#Hy|EqNF}bX2e3(y_5rSMG5N?g~jMvCl%}}=qq`9AVxw*3^>F>LhH;m3sLKJ z6X-1YV=v@=O3K;z9nD+B#S)&Tp(Nrt3sC4aS4&v&c zih0H=@a<&tGC!_eXNv(g|*j=o}d0-6Yd09LP(uE3cX8KoG>PWJwHFGBEXMrh;)-0{kfH!tTEWNEvLj3q~E(^c#X$MH}tJB>lI*Wj%nV0JafQ z6Z30iltWMILaP1Rm8cO`HtZOmoZ+=JWaB6iJbP8x#?|DEhNvrtO`voeGD@Uow2f=~ zEf68fcM$@yvt@6-6=RKL2Lf4tpE6Lm7-UIFK~|OlbdV`q4q%{G`c)H?68eUmOwwgC z0=s=Rf0Bd%J(DC(OQ2*k)ZURYZK#~OY-)BmN1mbb6fNPDm29NNHjP03Vs6$*K0Q!%Hg-zAQs!3q_ z-NRAo#nf3Tc-*UL(BW!I1wJFVn-HEor$AF8fnE4Ge$iu6-uDg+0V+IO1xD4Eo= z{#YS#d$T4u5HajY``hkVGz05^$ zX$hB5*U_A@nP60dG~q=~LDvG_Wt?o`kq@(K(Y(9?Wy&l{Lf+%h$TlEb>R{(|7HcnY z?vtO=92bMUkJupFsT;k-G4lR!Dlw+0cTncJSWayV%S}$Mut_AFjDxj+)FSidNGlFO z92}QyuGI-Mnu&8x^a@sj*xXJ{s1DC=ayC zKtrE1!V)+v8F=A`B`^e{>6m69@Rh^DScKa<5`L_&HEvlJM<@VE^N{7fTqKV3_~v z{J;122QB;mrw0%A|M&4gs1X_M&LJeA1&z~*vKouh-W15|khl;`#99Pa%9RT2vH9b3 zGVline&6epAIQMpeJcLkv{p^|kU(5LmT@iC#+0whD16ckak1jXu%s(!Qd=MzD^ylUd*mo ztb>S5QvsHRgjWg)+dg$J+5A(QhhRF7FU21kC5~r^^Bs>=tvNGsYO4Tz9$_;Y#Y0li zJo?>_Mme2Ev;UfjOGr8{1SC3G3lteV`@Ub*n$e_W@qgtdoA|8MC1>JARVt)cQ5ewp zisi*P|F4O7TWdK?i!gBTZ_CW6Xm^fx14I?`j1BSZMj^4I*YUnTc;@&2?8n$z^xt5> z*6Cx3|JUx`UNiph?(W`$|L46tXeBI3e-X=|vKh}*P)m8ct)gfXf&^7s7oN)CnnU8c zjG*;v{FBC57X(!kdy0ADs;y+{uOSo~ZAkrU%B{*11ITVj#s3 zmU95)XlLTpxQx%3Hnk-oEvU0m-7Cr>ozWr+K*Ywu87U+*$WzYA56TEg$-jO?nNDSo zGAg6E@S?=ofR@6D2jXTkCu2s7GQYv;vLPX2p$E{2^JJq^&?jKPF`*GIcl6kG7yM=EpOO{l<473v5c`+$@#eWi+4Em0v^*_Xel8X-A26 z?Q!JkA^Ee=VV{ef9_ABVo{m&_t8ipP@~r<%Ecj|Of(HRG7{@0t;t>rlKfyIJon^F$ za4N)>fdlbK6?QOiGX@S5ZN@1L|4L(;1md&YPr7aS+hi)qmuoOqH>B_wGz?@fVW{kA zwUo&TN?xQh9V74>Mnbd9disl|jlHKEomS=t&>i7vWx%DS6{evvN(_|eow9D_1r092 zaG#^CC{`V%pB>r048159s3lv2laDCGiUUG{6hfM3{Re2$Pyc#&WQ?E4uaYD}#WNft zC3zY%G%Ci-VW1}#V-g>TNBCugG7I<(4T)bK8I=X$1<<4vWg4at3(D})n2j*c3z(o6 zJ5_fA$6`JE08Yu-OWE;hDmKnMEpt!7;)z<08nCHGCmuO16Yafs6qg0dX*i1#NPWS< z-H(ODBk$h6{!OAi8XKnZ%5_z=x)qI^kSLinP+$ODSqf8b=7J$C{C1S{=NVgp# zN*aaC2{;VHmQv>k7V4>*xXmi{4~Y_JzGt+i)qF~G6eWT-{i93*Fn56EhfNaICLR#v zjXX1gn!|?xOL|k_>Z<1+vIp04KSI%ZIwH4P@QmvcG8fSYH;#DW?hyAAErKb01_A^M zwi*SJN+*QoOxm16j7re?(WZv2^1_k!5UwE&Z} z(5g(i?;ct;VDz8Ri-y zY6!Lg?8*fb!J&4F}t!qDXtPNgyHqw3x-V#CU4Ybd$9?sJ+ZX|CX?V8r9>l)+gQB@;DQ z1j4B_-*GTa)}0kGM^`X?*ecCO1mP80?!fa7ww%mLUKrca0jI9TD^xq(Nhe{Ol|o(5k@^D^_YRFOwpg zv8qAG+_Zm_S)1fR)5=Gpcf33vteHvRrA!)k@Z`zfel0_mel@O_F}ISom6Hx0NC_#? zMr*+23Y7CoCx?!l73Z5u8zJDjrQ6!pNg8f96RoVW^A?{CdL0GWqF9`TS4^4L{Njz% z+@n=hNB?!!z_ubUZ_|(|E?!^(_9^GlB!MhGE5I%e01r<9Bk@a=COxN-C`R5MlmSEy z`|J0scU+B?>R}^*!2}{VDT50Q!25O@N0V&*0lp^+^It4lAlH*`S(7Vr5Ly<5C2| zS>d$rjmsL!(omc%hqenRphBE~d6dE`jw$DF<=qLyBoW+BoDFgLs25Svtkw2lUM8L) zLpJwhIO+)iUqn1FVe|jB3@1!&B?v)AJm1Ck;JP73R9$(o6&SY;*frf@%V*q!cD%t(^>?!4`~JrrvGtyRpjq}& z{NCOaBRf<9N;V&FDj}M-9LZQ-4JAr=0S2%N7@EvA3Pz=L<)2b96wa<#&Quf_h(Hz1 z^fu)+@DZxVA2+wvVnHswzUc9qy~0ozE-zzE86g3R#ajybFE;{1{uDru58pYt}5u`ElQr)_k4PycA zfH0Qo>Qx9BG?W4|z5|)hAbwv76ka-&_)gWE%r}Dy{6Jj$qe&7a6J;nU)6s~7az-=x zGf$I$q$B(hD!+I6Q?m^CTl=TOFYxFQ>pjlW5R?c3jaiN#p;yis{?Z+Q^&;AI!S7jF zfUxuv8%@*nvc{%yo#6u)ApS%-uS(#J@6u5Im9d;d@{R&sQi7O9Gy5m`HVs3~CX9eh z&>!#*rTtJJ&FMrytg$A;eJ?~0 z4u%#54W??M#KOH2iy&xD4cP$xfNIjEK;8jML>lw-2FfW|p;0pN?463e@lqCny@11z zQ3xnTYb2g$d{UYh()8qTszMf3l^FVna(1u}=7oZ`0JySMas7pSD`HND8Z1^hJGDXx zp&2r6Fp%)8tywz&#P&6!4na^mtb^>=@mRH7g&3$E3s`5A7E_R#0z9o6<^kn9&Cgo% z(R~Eg10z)&;#~}~`l8tQ6`*MIsvtWyI;kPat8ttPTRYB1b=s=o=};S=WwgyGu{Ko4 zX;q?PT4697Is!%GDpfV=Y-~8KFhf|~%npf~wZDrVkzdXk()CbbLrP~`TNJTTq&f;) zBo*4?mKa27iCaIIrcqGE_7sI4~F{+h`I>l;%N1^pNN<EE0fm?Qw5>?g_gLqTv9W1n8xBn^?TelCzErlymOSe$-x%tob^i{OF8 z?-N=~hor}fKuR*iH)V~7Z#=ACLAM0321pAaGJwIZ-q37_0B8_qAhMJobi($nO{@M8 z&sz3>RrZ&+@7}*W^NY{LDt#=o|K8hc+JEouKiGfY&tp*nLYNgE!OrAn;5*Dk*cG2g z6lE9_MCsCbm9%gKH^nKFL0P5Z9z(NnA7WQ34>x@_v}~!I+uKs<1O^+jyULR2y%F_M z1`FgvBctau}v<>CNg z>JXD5!g}U_G3qr;T}W7jW*VGiF^xdwL(&|xNsdYgyvEQvNRIh12v}B-DWf6F{i?kX zF8g>CjAT^RS=GU4Zl)`X3g8IR9Q}S#!JZp29??K8#z?-E5f?QZk?xObc6{f7`mgrf zkp7=fS)8$)`$e|g6nqK&KiJ)E(*OOvr@IgI|301#73Ubzw94Cpix%GJ^4fQ zU9+L~<%nO8T(bF)xTcqs>kz#23}s#Mf{n*4D2Bv+o1RGk#A_9vYe?Lezn3(2>)7f5 zLs=}<<`^AckI-6yxIc*7p4WF9i8qae@&3<|y8b~s788)lPK zQVVFlk|HmYfE?_>pV92B9FL>V#PwYJfSe;0w^8*WID)JKh6Ri?ARO4wKH4|3MRNQTc>pI`${jTCIfHzT=L zu-4wL4PZq-z3;f&Xbt;aXUQ$xTWTpbcTGFKr&1L&v+K#^=GMgI zi&^UJuBKsJO++_3apfQGN{w`4^H7JGXbbj{O*VoVK#aRGE+VfgWGKMUbbK~~&=JV4 ztxUz_wE+j(KzvJPY~aINTFQdN#)8Lz7^Ro3J%heL+zWZKP_E;)aN#b{eqzLMX|w5Vny4!I;{vp@#>c^3n#{y!*&ZQ1>Dj>ed`O+|FGwV_kVe%GCal>c!e*M z|L*P{H17Wg2YV0q|9g2PDH5lvCPyP0D5k6SuI?QNN17QHV^PL%=Q2vdq3MD5S6nvE zj1@Gb1sys>lC7a;9u`rXSd;&+BuYY-6l9+OXQ7GJ8WLAWtmIQR2B#FMD_t-mhmh8Q zT2x)4tmd9r2XI+Y%~9QaTn1(3B}&_ejwhk7`+=?8g^i8(D;T8-$eJ?j8mIKFcg6pl z=Z5s(&@L~J2C|s`5B7Hto;K+Jlfmvo{^xsnOae}`j0;-bAY^7t1|56$`dX57BZNhb zUg6g&+^9*3Oqzcqs&*?vb64j~6HSJ-G<>gx)@t7m%<{IT71hod1P@SrIzkQ62VWb* zSU!;Uy-D|afW211HO#X~zO;|IV`&g2C3<_a*+mPEqnJjs_lo97OK1Z%M7KIrPPBSi zi6^e%=SEZgj7V8VYv^^aqpH}FS#MP9>PU}}5VZi%d)SCBVN8-4;a9iQKBjMVv|Dh{ za9XUd00g1wK={QY=`1R`njo%h10*Ymy00WzilpY`v|PS)MT@IfEtytF#YTUmG6U8N z%v+I`B~7fZv8_~0rmJdIt8P!;y*Ph+_~xblAT>rm<>_o_yd z@m0&1pY@;hSLwATfvOxkn!Rev)91B}bGrOj@{cr1h`ZyiJ65Y_RI__5gq~2`(~f#3 zF!px3;c6@ZD^xc3nGVnlBE1McYm2u8atYEFL32lVQugE>5GZ@}HUPTfP5W=T5i}q$ z9K6I@(R3u;qq@Fg8)G1;cvO(IkCCn>YG=w|HNeumT59b%*{(jCHsjl7Lp5Xan@)EX;k7Zjq-K_0 zUk65^oYC-I63=zek5wn6W?ygB<TxYMG@+*LNdbq6d0WBm=Ufs$wDLX^bO6-yZBCwV$zCZjG!^fN2!ynBXQ zamr|1O#fN`bW;pQNmN8Me!*foKNBP%NGob!O^R&7ZKVc0Fr7>%Q*a_4r71{}pki?ui5=ve1 zM|holC-Qz%d?I`V`tsRTo3p9rvzuTR6E1%EAu7gEsBQm?*3gMA=_7biUAV1dQp^sW zJw7dwlBv%GdS(mZnw}gf8tK*5FZl;nU!FoPV^aLI!LWRGBZ_JI6q}Ym@yDg5`DQ<> zbL}5o8tj(@^sayF1#XPrnI*D046~{HXbWJrvDS^JbH!?WW*o74ORgMfYra|9#azwi zifOCoYmYMKWD$j2yaW=ih?O|IK>W?jpZgy9Iv154z$F)tKCydnGph74G(W7)Atv$oFnvvu)751td46+7-W;>k6!KD#VN#>M5{E7(kvFl}v29V%bDvieY4ga`SjQM;0v|2uA1w;OUJ_OG}0o&zHjNC@^ zarDdLIe@+!j{X6Nea?B}4iIK>Pn2nZDP*nR5iwh0Co5ljD-!+FEJ`L4;Hhs!~=t3YeuPA=XY@IbEThW-YK z$LG0v58U)#^^X90GQ}4GrL_|ngze&ZoILUS)E#&bZ7U}QOO)i+3&#qHjd)kdhN>p` zWI%Ps%y6pBrnes4eVv2y9fN4#7-5DF2e-MbrlgXEFcI>aYZlX`@O5;~-Ou_|_bJvW zb&O4;->X`Fren5tB~=Fb@ zmKNjh%S>v}qFhAlKMmS?Rp{ulI5=Mmv^TwdkeLW*5L(|oTNBE@GE!-0D=VyD%8}bI zmhORkoTZHw4XcGGR*0pe5v7A+Q~WFV)AiGAIPq>P_Hm#HCrlwM7(}Qcb5pgA8nKBj@d5 z!JaF)A`$A6^Y6(7eG8|lId1!!@Pd?O3ne-0+lHM`8R~KM2ABC-hu#sd$yR$8-pPl$ z0RQ(?D>Ak1)r53U2#rwjC2bG47FNs}m=(Z%7?8Kb#8ZFhr{tVmzek#7vzdo3mVk}^ zFy1;cCYbf~XDedkPhs#7cpiqmlAn_RqQ6&U99Tr#sEicA!=1kEhYZ5i{`55kBjJQZRehb@;AnOV25H3`-lJI z)7#e_M?VK~S|EKxFqBGuAbD*!FYAQe*&@MP0Y^&pYG^0e^W>B)gi#CHb&Nva-$Z&l$@7MdJK zPB>P^D!C5p$gVx~BdlW@;U-Z8L}OA0vnL*?wgyt#s9U76ECToh43j)0~n_*&Wp*U+fb5>xHH#ZV{& z2YP-9T+5KzxO`9J$xE&KCOZZEoG+cc1CGv9DH2Tq-#pHBMgU$mD1~{N9}X$wI|gY9 zd3YRWUNH=1n(Ro_Z=~qk zV$e-6y9G_S?YjIJ@vjKMQF)Pt5b_1M88sd?_41yKEZswVs zI5mx?pZFfE)I$l1lAqP&P>VPo>`wCwAsC$c$Fe>XSpVwqYJ#cftLLYUJ+W|?)>03x z`+QERRY6I|>AHuNSfCvQ>VTTdm|>~pXc66UM)8`u+FZn3yAN1nrY6Ne;5ue@_vA%;#UL@HhDAoli+7cPECG(X;4!g31`z1}xH9&UF!OAV{S`U6$Yr!w! z%Mm0rEnC^^(0Y2GRJVKG4xJrBHkyu2eA~S`SQw~kaw`>|h!>jvQbrA`qZ2Bs&5kws z@oRYzRn6za(RLJlN%_uq1^+VIQ}uoAFX5uyp>aE6__Z->&}o z28~zw%h6hB#xr=2$Pn*I$P73!tv3r49}XBl25wuiecsKNDiZDbuaDm$9QRB*Urjr~ zulKKCG5)wl{=UDX^`sU3`KE$7T_6oOw9OVsJptyz=dd4t^*DY#(9XUx!}}qa5aVbH zQOVou07aG51SOmyJ*wYC%s9%m)nqc&gn zJik-?b@!KOR`%$(!3697A^56_toQ4x3}Np5L#;SWEvJKg{As#y+*3~daJYZ)`C%I*RR9k+eOF4a_hPKLojQ(vhI0oz6eb zrUi&uDK_r!RB&oDG$dfk-5+xwS;QnFz3On_-i&>1Tc$(e`}GPQ75s3(BgKeB{AvLy z5r*lD3>^9%-Y30Iqsj?|H9AeCV9@R7>G6nKU*^|{xELgKYlJ@!1GJ3GJLG$IvMGG@ z&iwc747CntFGYx+pq{>^zsz60U}%pK?QO(~+;Q+j^(Yj!b=EbJGf zVTC*^LO@zT9+n3BR}&tC_>$F}@X0ld)~=xiC;|GVD-mWy|Q-D6+pO36NpiTATo={JRG8My?TV`PWKLJrPjJ z-*fdu00q?05V9{!(CF~<4|g0+podh+kkiakSSk(fQp$?+LN#?*zIuw&C+PloIXH6{ zd}~0avXTIjl)&=teEs88GOVzC<0bPo*RD$CcDRy7<>wTCcx4Scy-LHYaT$v>|IIelh!3%E4a#H1%J=6a{4BT#w{#s{1`+u%gLa2^f(WEiyTgluj(=*MvH%L*NXm z;IB<*)fL}_UcDpT*lbqZEM*=Ig16G+0*HCe^?E$*pYy)?A}5D^<{U%S$kZmuT+%{gwf)TK<>` zI(z-EGozpv4s(#*Vn6CIaXYa!1}aNwN=vGsNGefrSzGL%>C@*VZq|zxvq_!8)o%T# z_XF_}(lHS&wGQ9j-;uw(p^a&78*l*1Z4s=Q?o5YP@<_~j5HssIm_vw+DW;3Iq;ILCW zB^OC>mWIzZmdbxYo`khN0pR}3MsSAK_7T>YB5sT5t(vfu_ec!hSy3MuR_YgF%~)f; z54#OIXrPAiL}E*}9iYd)4=9?8$o;~6z+mE`3;WQqKd1vURmd)2wpcvWZfQ`nIK=!J z6BgqV^dZf1qCGj8J}u)rMk9;7sCCWPSBTkv(aDTw?m>jT(W1JO!n$MlhGxvd@A#!f zbPv_%|2;&NjD-6f)@jaGbDNZP6EGm1% z#&31t==Ss$BrCA8l-Hyg@m8rYAP3jX&4>Rb`wYN*^NuU>9C1FsvD1#Q*@CE&@w%Jq35nCgRy#xM zUp!NcrRZd1p)-kNw?+`eY}`Y)+)H1FIku(ii+*)-OWl@g>H699$95-=>d1VPJYC$A z405^eXNCq|zrsm^7~8EtI0|-W65rm7s}wzkK`r0lL(m2&phPnw?xu=x*k{^Z8=dR@lb5;&E4;|)%uh%RXb zNnM4e`_ye~fiRFW@4N!)^;x>Q{fIOANtqvycv4&YxI(`+55ynD zuO=6ij3~OdzCROc^ARQ~gHXi(D65{L<^dQe8Vm@lu_#zu8&xBD^A9a}8gyG)JNaG+ z8v)0$gwt6Sz}$Q*BhT*z{D?e2RaUyZLf?yQJxZULbfNp=mKgMI=w6}nZg=g`= zN~a9}0#E4mAlNYjl#257@c$3~V?{b8+Gl7DZH9;XLRpSh1X;J!8+omrr+EM4#vK6@ z<-l3Ems_f;sfedwfAQ^!$|6pn6q-EECvi+gEAGdb`R{}P=rz`FZ;E>*spVVorR_ z>nsIOfWyCN%{&Ts0pHRs=I(lJeO5Z9yjd%r%3|mOC(sa9u8N;?VC>QRZ3>3JjePr; zTYgL2*XPtL0=)rHRki38rO}uF?Vr`HsvxCc^BRp{T{6xu07!s_FUGlp|6%PiW5E!= zc|_#LosVI&z{e|6+CiWzvC=%i;x}X8cV@1O!4PGtAq{#bIXC=$Af5x9kzqvZKW92ryOc_CW=U}c1J8j&i|ePlOgw|MdXwyK!ee&Jdr zAp7!ff;V`fH`|$(CaoB@d9kRMd!{7i6#(td<5tJ=t@E(Io5GJ-c>ooD!;+h$VW3!r zctD)Qw8oDBdwlIWzti-q4MMcd<42~@-UGA{w$qQwtn4mI+7EJlIH&2K4I2P2ah|L( z7h%wVxIH5PvoNTY0M(U2uCO_#s#-mdE|-)0*`OFZiVK&*I5@w7_zRX2V?37BG2T#K zn+OYxCOnE0^Vs67sh*+eTRo1c`1b2S1N%2~8KEI95>29-j*yRn*G|*B)vEwPj%ZtI zt={kD$=2g~*Si!}fuQ8kgT)yK5KoIFak5cA3YilOlWv2Y;|rJ}S%{xPaZ4#H0r=4` z*91)wN`cVxAV)OUDYNOi3yGA=$D!tLfI2O)M<{XC+`ozh|7`{V1ur;(S7KEwK$W<+Qjl%k4VugJRrQe!uM08)g3OH z=`3t!?xI!R)&C}Aw{d-mNg7BpS;6>wV%6*;SH%X!rVgw>-Up{tdb3am5oI=Oefi$% z_oF%nM#6*Nn?d0YlI8JR{aId5s1M360c*bXLKwXMjJeY3#QqHo&0PAZ<2nd5i7Q_m zpe((TQP_A~i`7|VIkCU=0FNvh>^bO2^I_bcz@@kG2jFo1At|Q>z4s;OW z_1+NwG^x{`aXyx0Vb-CGgX--Iv@1MnNMH}wHB?-8UD=sfGP<$(V71FS0Lo@cV0XBJ zg>~=@LoAr@ptHZH9kY44!WNt_e@LWB=e@|2!$>#4+|3IR7KA-IZm2YNE#ray19i{} zr-GlP%za7#rnf#zPk4{Y>bOg>OJnyvfz@cVEOrc3d4KeO8bb72N?bY0x>kE_+go@6 z9OOL8bENtcmLgOmAbvLVUAiBk2He-LQAvib#k{eLQjed>bV~!1iPkUNw{kJ5vm&kn$0YgiN zmnFkY@49IHUByl(zONc0g#e zyV9(HhLgSe>T>#Z`W2Cc4uwf;LZsX9u6I-iHcnX=+2p4@=g(L=~{< zaJcR^BBFDd&#(K|eNW2r)=Tmb&Ul-F=~-D&+Ohu=QqCpkhhPpo;d#$L4Y72t4wSE9 zAEH-$a<6S4a>|T1sjYIa%suSt_bL6)Ci-}QvE2HV#}@$w)~tCs1^MEg9CAXXFf{{E zf-m_u9Bv9xO0dNZOPN8x&)-D`B}3m=a>G{ww=F)f3HE~}rJuUz#%Q?_iWKlm8zE=1>S)FW|2)>^4;s!*)C%=qLNVRGrYdWqi@&*6aCB4k)I|9*=+*I zz(>B_J^_a933LbysYl)H;*{1rlBZbvay$MPMTj?Qw=Cj+bA-qY^~~-H=aC`+aBwRY z9~KMBOW|S-^f^G|St4~X@1n&e)90)Zr625b>c^nF9oi+F*`{1Iu4vu)p8gB5TxeDx zdu_+w!?!SH(1Hg_eR92(kHtH~1q5>|dH9(tj&ReI&Q0PGJ4+`kL3B;`nY?m&)U8#j zc1FiCrE?L$>Sn4(YZRkY5yYckL;-7QQRvjLGIY%FX(o>%HMQ~&6)5th%Kxh*ggsFE zUy{(O^5fE=Z>^8*0iA*-_*bz~N!-GCgsie2y%K~OXij|@d%`wo0G4e1V#S}zs0;h% z7Q1la$^yAERTRILj;ddYMM{dawcUH4?(0Vyeb*f`6gsQ29f7h2Ug8d1QS51qnAEm~ zHe%y;KL{7l+-jHyc|m9SUfKQM37*(bD5f?9*ui--f}H{2Qs}lJUtIPf#47K99blK?5IB*_8q~NZaeNc0Vp<%7YL&;~@y`GZ zz121$@jqKMtcN4$&rn;U03Eo<5))ePW~Q2s^y)M6zpFI0zt@?MpBc8RUvak9aG3t# zL|t6e)&$$5G?vTKc{(g2)?twy!AwWhY2VzW^u0|9ASKI9V#qKnn5}kmSu-8i-_m}B z{BZ)|L6$zT8ciHEoJX)?+)$OS^A8WIRi?!m(zT(wutGWOeB)#35Gs_xszqA2%c-ki z@x8@iYAHzu_8$s?_leCnJGW2*n{R%pR{OcF#T34Wk<9kbS1T=bsYznOm3${a z$bhNlC8WP_YhY}FtR|eV`ljwnu@uFq9XlCh&oaG~mCCEhI=fM^fzzzubu#8!uIJ&( zwDQi0jH~yWW~-oUCj3()w&F2$K984o;rYrK_8cmpt4HtwpPpoUIG47^Bl-*Wgm1Pv z0pd1OuLp03wM+s(_D_<`-&?4>kzRY=JPpgORt7;gB71~ZWq$~?_j;OqNaagd-6n<={+D1ZMos8V#SMk1aIpe)(;GNwOI}wtx{O79&am zKLS78NW>{SWzEsCE5wxQ#ToC(wpyw8zq3H4%B8r$Z`%r*JAUWNgxj}eYap~d+Wbkq zx44|IUct$C!_Mb%2HyUax8{lFll)Db6I}6(9oT4-mi5sXoXsaX6a)vWhShKjcBQKD z6(--SnqPRCNo&5bI)*HI7Sv2=;aR|$W^IATrn+vygC+^cR_plA<4{6vAN)az=Bg@ho|XkQYQ zw<5mCF#dGMd0! zLzp>ZTK#}hDma5tjj_oL%y~3uJ@^I#(y#N8(zT*3Qe0{JVc{jH z8HZm8c-*l()YeJn@%YFeE9!*Jt(P__7<4faI~{a_)hf!KR*2PmaYo4=n7Y0^?Q}TR zt}M2xYpuny_w!li3Gt3Ag)IjrP_2b;aSLdY{db&a<*!`93FzgBI=1aE zap}hP3<^DsAj=%fONL;TsY;po3K`pJS)RXNTP^?Z4WK(hx*)R7H32;F-#YJUJ z4}^uu*Fl&RPfA%%bQJ-0Fv>IFwoY!l(qe6(vHJ;(7itzCKPps8K_S&v-kiFoc|pHiV~Vj;CpqxzrFu>QT$WOvi0X@2Ql{t|Lg7e z_!cIns22M+`J z*jK#p29@A-mgkzHUY-xVDAn-;x(#Eo>4AigCoeW{O!bCko9K3yk&--UjA#4Z69O zZ4n7WS&)~n33)9YRxAoE1={s1;Sbx+#iNq;`?G3gKC?`Mf+Lj&p= zR;su)wAWUi{S*oSSsQE4Z4)HS=)Xa~frH5$ig&jkJuzOTyj!;)5e0k$N;PJkMfg}TD7HGotj=eOY8d?aXn_<>)=>!qk}RI}x7YNA zPYd=PxqTWWN1o;TYwY?H33@6|1A&eH9`D;)d(R_In9|5Lr^7EHTt(R{u+KG* z!S1neOZnUUA9=ZeA&8gwQ70C5zp7ok#AIAKPH>jeYuYDCl zc4JQ`f}3*i8RornqTU)ZX;;2B>9C*A1lT+U=xvvv4hv48uK4oaOUf!Cjf!$IT7lwvifNKV^4tC3f1=*PE zz0?skK{s)?qIKWM;I8LDBvGjbdt|jo9-xJw-YQn;Cv{;fw|YBc{XUKugDOD4uAaTVsD5mVZG#+yWHcoNZEnF3S9$mTWeJyBW{TQ`)fAgINYp+c)aD#NK`B5 zni82G)^wvb224tTI|08jLYW`x9YBp$ye4qkKxeWJ#_|-0R<<92T$&+BVAEDr?ZfBO z{c1t>fiy_Uprvu>b`>Vx7>3~(`pI=nwD{0d`ay+slY4p4K!!v^AizFaKkljUZgwf0 z`t;^vE4s@h-fB_h_R`#f&3^6RRYIx2+OvKG^?r(0oHP}mxo z57nXAgZYmtlkZ zJOWqFk~9Rn5zS}q-ZZ;`Nb!*IHZOuue-v~0vcBdtD^}!BkDX{KXsX!`$tIls4-eic z@Gk>|d`j4lURX{%+3`ECCC<>5A_V_nq0v#k+QZ^7p9S6~RuO4S+?9W@ki_i)Jw+qn z3l@6nz1z&w9vgjo?M+3c!}8T)Ug}79b0N9*lDU02mH*kRtHG>YPie=zno#n}F_Jt4 z0dnugBW))0RY-d;E2=-|yt*waq%h+Br)i1@zLWpSS+BDa5a4G0iFSJ%Gnh3KVlj4U z1xT`iDSabs7)XPkU#`OiJe2O97EqEk4JDvxC*r3I%YPJEIb>cyKa#;c-@NRVx~(WkB?W1`4v6=7;w@bLe`h4TL4LNws%GM`+Hv5Sf-}!EoJDx1`?1w2b1+Usbj1*s{;;q)$xz>tqh#gwvYCGd0)^QrSH5T=UDGuB&@WmjZXy5p;Lb5>B6|XEqyWxt zI}xTHrXATB5Alp&+L1PG?>LV$1gw9I`4~J%B|}5mcXvN1B_!?cT6qm2pdRe2RmdpS zc|=0MOnWuLF0xj#R=AzUT&N=jWjsJ>w{x;Vm0l1TVSLH?SN`G)NE<1a5MZQfPg5V8 z1XbKvH%eTvcy#dGuh@SFf(5cz@(l??+GE&euK^QP=VCbdR;t9X;G@S z&^3ewEr@JRB4_Cra>y4g6sIY@SH#M(YGZS#s~G3@{cP~&kj8PWCek~(4D`sez6_I_ z$qILtbk{?!Gns!NV@PGPOfgU0Nsb(btq3;S1*Jarw{nD>XTc(eE)SwG^l7a%E6W-; zvLS`x7K%TOL#Vr`f*RC*Sf?=`y|IU*g8j&uqH8Ku%(@3x5uJe8XOGSY9BfSEFbgRc zc${7Knc7K}3FJ4#A01{WZHcF7md*tTq3SPLXyaO^vh*s3l8l@Ch+tqP*GTgS8q}Ah6mmnPh9} zQVDBHG%6||eM9^8IYqu;p~FiNg~12vxxG|!LpD|dm0u_3E)X2McVPG^y%rm%zfWvoBWn)61x)lq~! zrux+~Dsu*PvakO|1NZeB!x{>9ZQatoLE=L!Ykxpva$g8lu2d&)>v%VEOm zF@%%%%xpnG2b+&SDU6IxAB*kbQ``dAcm{*hBvdKk=xIigFP*C28;YpF`J)Okq#1)X zpLlqEPSOcML|j+l7G30Xd~{ACAkDeBfaJ>+YG9z^GQD537;S)bdd#WDR;J~Io%!bq z74hK1tqiUUPxjYy0(`=$CC#sLPgk43&|$4(X1a3fgS5V*9FgY6C#5T%H7D0-Cf$%D zF4Dq;Sch%|&E!&<(IcXnU;=dajkk)mid6AmI?QlELbae1$Pta2XN!DAC8mjp_-o@u zhegm$X4C&h#TCj}no(sQqlpaAUzE|m;=jFX36usegSgeW@WBiCBR;E@u`WU)3pKq6 z2#XATBqoZJi8k$%jbU%q#!khp&`Nkrg#?f%h3H3=4^ad)n30A?KWH;I(n`&a7bkvu zI*`hV&a;netQK|6V^qSRr4@!1FJGpqV2=F)e$?085NTR!jY;lQp=oLc8KGr(V7brI z9u21H=%8N&zzCl8Pc$(l)MxI^3m7wu*9h3%lGznFFPQ82tj)1KopH~iF{rF}IYFeI ztRJ|KT5Hb;M5?q0u*@EmILR=}FN+Bx=%h89sOc@1Da#F)jkdD5po}S82LnXuNgFG4 z8=$-NTk@w81>MN6ROY6JTi6{~uw*?l#FZnk=*0?`MHq~V_sSaxF2Aw0BUgl)8RVI= zG95XN{yCf4I2sv(m;t%gmR2*QNC)}JexTJ@{SyK77PBw?jP-$&$G7D-Gmu3+l&$4@ z*2a{NW0!O){ zBZr4GdsGE2k2d39atyn( zRX-?+h+DfLp>x|6o$Y;1hLu3WC4*9Ue8yBwW%Yz6I}irru@o@uqLj~yDQRinNd?LS zRva*UgUMWU65{Eih!nY{e z%svy2q7qkv3+c-fc;-5Z^tZxpI&NiMaQELdWo43cN>S)g>*oH370%OF;9MqR<+)B- z$}s(~UDI=2rCd*JmRp-UYG46druu6EA&xnzGKM5or9o(jeiQJ!lA}t05p(IUs(j5V zXvvlUeu};vf@CRPa74(QS6UwW0vB7U%0&Gvg;iIGtZeb?m-+jK5{EZKH*_Zs4KNI{i5_vrw+} z&?a9BvE;)?P#!A#rzR%*iA7VQf!bR6?<2KNHCRE@8WWr5IKQa#RiV)B0OdCYI3lwo zuOnE^iNAH#B z?Ku?2{L?}n3%2~SFs-riprBd^{WPcga7lrQih;XZGHw-fC)^vta0%XC+mm3u8La|? z$h!caT#Q!iecwUx6=95f0D0NlK>1)gU~{LFdeGULl-`}Et)EiKmKD+R+llz$Ln1E1 ze@vnLjW1Iu`kyI8Eo^(4XRI3T)E^aOv@#$O5QFbCsC@B~ag^9MO3n3s0sJV?4af2n zPC^qloFZ7`c@`fXbkOh9 zzYeAe^T1ri=|jCtoh!}ODfsl-y$mTF#Gs$*@3IqWXk&ySq?KlU&=u6+-4qU6ga+8* zCg|ZJ`%+)I5R?Eh6bPu%Qc^J%gg^N8+kgenCy+0z-ZH1I;0YD^`x&H9m9KgikVk;8 zst3q(DY*UB%LbM&VcPgTJ|Q8MeUuU1ntsYl1-VvqSCkpW0wrKgx4f6IW2=ie*w7LE zzQgyH0S7#MdQi|kTa8VC99d)fRgF>lrXeAG`%4rG0CZ$we1fetFS%e9_O;Q;p#8)$ zac9pFSpZF);am293>KbwA_Z&3DcC~S=N3!BLd0nK9OFU^`r%3I0id^`@;V^35aqL* zsXEca0U!o{hB>=WemD?xW!C;ac=Nw@y)GgI{$%!8_5Zrq-K@+~vt$`_@1DI0^}K^2 zt#2v#`K)D|4#lL3%%aM~QeQ`fM*b+Nu|z4{>$ZXc3n}tbB=pERAq1tbcnkW`KCETM zPo*R!<~y68m3NTe$ixB(e5QqC(jtktbJJ~qQ05ab{)JsQn@&JPK){cK&+jk5?eqCw zM-+x8L*u(%I-L_H`3LRMc81OOqLaXN3o*w_A1pi4QJ2PKTuj1Y(zB*Oa?GX(MnfK2 zhH*5X4QI$%%BP30sY88vtqr&q-m-V$Q;ZkU;u8KHdd9cs3HkU||B7^GPf=_1d1j47 zk7d(1r-a*3sJ>F(Gho}0)6r+WkuL#!7L(fkgL%#aKideg_ zrA0V86SP~}tnYk&@9U`+2z)e+q|$rxh0O#ea-OSQw5=Fqq%x`FnseWEz}wWBYi1sK zu|xT{_I z6TGn?2wXmz(F#l)Ifz8_e$qMWY<%gxYt4?4;~*_FD``+0>R|i6k9-W$x@XJ%owTw( zlPwf|wKzjUx_(0h7jIqubl6M+*?nAYn)|yQ1 zC)Ku+pYTZ&>%d-;m7`o#4GPHB?`B}4Si%h(>ahg$@L()*Xk%oiEcP)49U6-pVf?m? z=0#|G*cO`s(J9O6E3o8>x&@!*@AlVMGTVX!i4~!F_X2t8eZ6 zPhLstsrWe16yh<&Zpo-&arg>Sy%^=~C!=3Wc7=#*5I0DA{W+DaP23U(OZi@?rdnlU z`DXi{?|{AZO<3a5xKo-E9E&MlPG~2R^o3BP@GKTwCmw9gVVQiW1hFjKr|25c<9o0O0x>67ZSy*vP0KVq<+L%7b%IOE}_>iR0F((>`tdP~FbCZq?lZSf(23(RO z1|k|5_Pi!NEn6j*5jLkI!fJ{TmS<;#-{2fL<$%4!du%`H&?NW7;(ImfvkH355e}&= zxY}(W^s-*RpOUZ4aq8#Vb`gF3CkQmTBEv3IjHpP5G(X<0#q|7>@K3)@f=F^rrCjHH zY#w=5Z~#WdR7-^PEq&iZXrOd`B~z3_ze#P>Zct`V_*p6F*ZSt-pp|;O3CekFIw)-I zOXUnQkM%etlc@`XJj%XKJsM~ia2o~TkCL7-BJPAn>HLVLd(doaH4NR!P=fA5u<&&& z-8ezUmYOln?PjgoW8hcjfGGrj+adLdFM238`fDkZ5UWws^^Z$8c5u8ffEk()DJZa% ziT;!qrL;I5$LTDKn=o8EF);X^JHRd@iuHaGF9L29yEFL)d?EHpfvV`N(C@^_ipLPS|&Y5N_6a_q)kyB9@}`$di3S z8yVD%#YB92Uu|78I>|d5xH=GG=FBdzAyhT*ClK0RIL-75p2zUXWKN$(Qj{b?RSC?) z%=E79B0j(9lUwO_qpePk6=M3i*Rvr1tEh_~>C%C&=7P2eReuZ5sQdb`AT2WWFKsOH z=ElFZUw)A#RQ>vNQ6G|1sjSYs1#p)>*|G~8U6mj^;(9q{^3p+N!47atbJ5pp5bzg% z|D}6i2KEe5DgCHcUofk|GDc@V@NNINH4f zz_lDXX;@fPjmmWJnC?*nqeblcaBv8J=TIm-&q#8Cm#2fAUwc|;S?#A57{3E~|Kq>75s0_cs`+JThA?A?uKRv{;aEspbQ%ta8Yi0Olisd}A zcd#1vFHfWSFHiFvBu`0CA3g}AQPENJsjfm5-R=4OseslqR*lIjcPVcZH@o)Cnt321 z_|!uH9!#~-Z7|+xXa*1WiJvxBO|^>C0Dx4hjO}{|(W3o5Feh@UwY(F4PQoy1RFdIQ zw6T=(j;;<1mF;cyEW)IcrdXe?(H_}*+bL*$lsURo+b_)Ii{OBn6n^OAWd0?61X&0*AcKW{A#f9u0)uQ|^L1f=er zZ#}T#TVma8ENT>^mZj=NiLmgn$u+Zyr^Xf1g!gF8HO${dwLXsLUvIcNE43Ub+uM5O zK%nU;vOn7z9_u9wDzt+8KtAR(Tq_!L^{gDP3Lc&W$`z~hykBD6t#TX^sA{SW#fia- zJZFU3iaXq6M^jgfu7Y(IUF*)6D-$!HmO3kn&s>R!ruMVsDSpH47LX=h1^Y6r@O32g z&kPM{8n#ZQ1MD+c{ZP*Nldopl8^<>rq&P8UH4bFSdYJ$Ju5LXF`4{lEHpX;)rQB z6RL=^-Fx#8AykX+&=ycu#^I7dCJrKD zG={4VBVEdpeAUJ+Wme5X2V5;3LR(v`$j>fJ&h6Go?98L7u5uN#Z^?gWIZXT8Nw!C* zF-_wS+bmscS9HVA+s?j^ZwAqJfjEdK>3QRb)B0m&9vt)w=bB7`?Q%uEOeti{lP-C_ zc^bP1=E)F!#bs)9{}Y$dGPoSAN)&+x6`OBzeqt<2SEYE5uaIrK|GlShV7-Ul_!o2~ z7-3^+B8q)nt$sX&4Actez>TEGv-5T8&U4XFuF#s#ruiH|IyABo5(c^#2qsgS${ah;7x~00^k{Ri40yIE~ z@9#i%vpN}=!MN(lB^0ba^5?_ldj?(}ef4UsQKmjC1W_#M|627Q-9xC$N^e;h2aJ2A zIpHEirWh_|{su@i7Kz3q`=E9;kH#Z6ZVIJ5R(ci0GINo~s{D2uMN=XIWkg8p;IpBz z>j*i#$&I$nxg+uHi5^XI6I9(H+=BM*{p;(9Ux!qDz3hwg3D z+Qp_3&nJySA$QkvMJ223U!2T{oJDONR@nH2!<^N0bAj;iuWBu$;KfCdoc35A;RuYb z{HeIGfng3`FM##_x>XP-#c7QSE#zW7!P1%!13tJiYq;cd8PEhCKVjOrPX@ya9}EXG+MpJzF=eDoD824Rmk1N^OsnqEYR` z;~`s1m41bY0t0u2tIp&JY2UwjQ^^;78r>cI;b98o>Ud@_bX`o!Vf;08l5qc^l7sWX zPA;jKE8usE!n)%zY+K$IG<$r+u9anm!)GQulJ?6zW``c{44~(;)7bRYtu;&0Atw>L zsb)}Cx3mlAaWYy!w6Agci-V!GG@%{eeA6!3fxt5^6O^}GogVas9 z`;`L02sG73l(#ckb-TQ_0q;HV`xbC10;ZAnQ_+8>b_1A$7)QA4zcGGvwhpw+amm+d zS+IsA!$jC7vFaLbiPgH5e5mi+gw^2_BhUw4(0xT}_=g1!w3Ov8=1R{csu~Og zx#RF9iOrOSmv(Ln2r)dUXT}|0ADcz_`zbL5&61YHBB!fyUgU^c09vpj$A;;M)%2CB z`tD7eTt68kOF5eR5K3Pu2XN!j0@ko);l4s=<5z!D09-|zZlQn2Y!WfG>Cz8=jy4%B z$d)(3)etWgYep$1J1^uvR6o77ZXX!zIlqFbCqh(WrRQ??baASus-o?LPJX>AUn7=e zF=`e+78Y|^!8A#`AT%F8b;PFX#iMYDH|4iXr4ZpDRnfnM?i4mF55?_MbGc{drfRoY+UXyQ~TanDjrsysrrHK%ew26GZ9{7 z=X|{9`oBGoudFE)`~I=6r4X#^z=|Eq?f5zxXD_EMu6W0#FnT||-Xh^suPeyj;g z(Vj$3vTd=bj8*T=GMRES$ZRbd##VksUh@@B_osQzC{O>Gndn(VIbFM0dA0yoOJq}y zZ^yZ>^o4)vB(;pmnYhDKrq#46b+(EZMe8JKb|_j}qrRVr$EO&@-+{aO>V7N<2IB{n ztA8bHhN7%DWDjeiJoVHU5v?@PD;sB6FIP3_yXsqbWrW+N8D75CQg<`e|038AGW4Ob zvbOVySCa9N9{+#XyT|9u8a3YYNq219?l|e#=-9S8ww;b`+qTV)ZQHgw&VBBF_MX~P z=gmyj)Tx>?v-2+`FILw2UEdEU;!LJJWtndyw1PaPmAJOQGR22BJ)2R1qz$WLL;>u%q+qY_&5W#CJ#N*^#S1nh^m53ZbMr;-qZ-pMzyIt4e!qofUV z^bQ4Wbuu9j>P)3!i_Srp?=SaN^?x$9U3`M8&6R=Mnp17tjJTvoZFp9ai97I{SX-5V zQV2HnD#?Yhnz?W7fpXTMXqrA7>NI{q^O4WTkr=xyvJ91G40icPuu1MP{!+UIHrg)U z1?atn#O@F4uaz|*U(R7759Ach={wPw+z?J+;V z;m_S;gqUXnN7TU_M7n1oEJzx(AIQppt)het6sBS^O*OodDCwNcXs+R7!RcmU#u|~@ z)6W>^f?F21g{725jZrs=c7RG6%f-X@mt~)%`y1Y;qVz85 z${VdJaIET0ppzCPObz3@CLBT^^cAZGX;Z}>vw!q2rCiHSeZIeDb(!>UD3mD#Y;3TI zVab|m{K)2UuW$6CIpwOrSJXA+Myl>{oY$#QWatS$`?GiXUJM`2Zi4CVT_1lw86K@k z!`UMXUlOje=_=b)V8c^mERCkv66dr|X4&;~(4ynzDW5omPZ?R@9Mu$K5#Kz)zDv7s zQDo_Lp^Img;>oFyi88K9WtPmLKd!0gtm6{hG{-)z&$o!wY}^|ECcqeiFEO;p7B9+@s+J{=~{ns`N-{z z)=Wn=ITBZpOmvv*QJ1}BxJj!PggJP zX~5ILvmj8}npT?ypuN}lovde%Y<*F*4Vs{!6X1orFBEwlDsB5?(*lxiY1QrbX1|)i}`x~To|v$EamJuk3XwXs2H^Zej!tjakkZ+)HEMYVDg<}b13 z_?Os2+;w$*QVzycwn`&w7Xjd-&#(ETqhCW1 zU$=jGUH$nizzM8=fh$BVoGL>|Z^8`J=w%jo{&VIa3ZAGfyio}?{0u%ID>X2p`eNwd z8o%wWwV926Fq;?h$%`8x$;pzYC@>&$Ee*%|?c(CgX7Q^~75w=S9 zvbzH*C%9T%ED0I9{SMeBI}PhYFoo!zMs?FZAgAa4%sSCttpVX}wiI|wNKL7X|K+~dhkn~$#g8~t@V*Jr^E;Q4Uy7ufQ0IUp|T524U-;hYZZnczJ^ z-M*`M*?YeE70O8MG=SgA%jeINp7f6M{B^)`knn?TvPNz3#fxQED{4f_g*;$LxylJ2rPcJ`1*EfPF~7beC6m;QRJKO^>18Z_PvZo3Doh zHLo9@@G<!!yYwOtU*-#UH@@@Z)n%K=01UUt!BMBy+Cb z2;8K(A6vAb&y9dGHi@-lm%K}#=f>zrH(KHk$F5Q`*^dj`#`A&GLC|mNLgUZjpyF$< z*~M`!!ip|_-<+Qupz;Su21Ef;l)pRYG+aPhh7TK4{>pKywS>&H2VU;?Tp~ zI^tN`v#aU==GLc=C-(tEV`u{>rLAt)k!~#J>b8cuhaeu)TK^t!7sJrHXMj<+rMA6c z>9w>0Fs+f({dnd|6=^v&eEH-DK9yG=11<>p+&O@6xEtohuP)swsVPgjwyB1#KF`jd zsIl)tc-{UQTl9MlT!j6PGKWa2L0qYC=?(D-?z)pw@Gqs#7RG_`Gc5H9$`Q;_2jd!& zX5=Cq1lNAYU4lu82~UrHbmXe^uRw6qmFeIm zl#?wImAOUf#f0Qlb zefUnUMb!F{qjb**E8P>RCd;I9Ev8t~BV76^DqPV$wz71bI9``Q#ooWlRWBw3Q$KOOQ51G=DIHj{u5id#Tgl7G?TWF0yD=3fC6+4A8 zqyQ5ujN+;KP7IY1c8FIOKO|vn(YU-c0aUgw|0-LPN;`=zDcX$n+6|bZ4B#zj;yZbkkL)}cY#}j zDK1qsAvv<0j@PhMm21{abk5k5jnrko|~Z#s1p`x1ENDkxBkKw>M8Uv5kt zk79_HtnDwc)gWHRapyYD0s_;3L?nUNXH_8a795$)&*z~_7!nyoHI&a(hmJ3nG&3g4 zHcJ^8tg)t`@xb}#rNK)QvIg|xFR|tIiFZ>WZ(I&ft?HK_&7g6B<;40=f%HDMAX&bo>Vl=6RF8nM!PaiWKSBtMND=tNXh-Z4m|}PeMUM)PI<%w3#4#blC`Jk@{SSZXsYbxg-az~+MP zG{u+ymShAyGaG9#(>liiJD~GtS~fnjc6hZhH25^EgfX?*tr^bdj>*+Nn25cmz-1AJ z(o2`Iz<9)pqOT85PS8(`HH}jTo;0$lhef8#L7oMLct z^J>l`u2{f+CM&K)j0EyXeF&2oR_AP?>5o^bW279$Mu!s>G?#YBgsS5>op9vi3Tpiyo<}6@DNgtB>DdfB1vB?!_)|Q$+5F(Kd*TMW&R_j}S zkJf3{f9=okS&2+ zLMLC3eRd@-h&xe0pOS$;6=!s17(`(NO-rLCPR-nmL=Oo4eB7andN$N@|_f zTr_QJ2vIFpI}H;~n_iMg7^_~x^P6f9)KJY*y{nwAvaT#$y9r#n=NAWcS1L}lEIR}; zbkQTd;I~p+0f(Yu4KN#d9-;N&+VME{L7uLAd7NbNOU}eUWS5d;5f8d4LX z{A%+qw3t>p?^+0I?@(H}qg7ad8Oc?%>G0MslYXrlD+c<$e2d{-?$mb%1-Xq$>OIG+FXsbu0hM~hhc1p`VC^OK z*oh_9Br?KO4we+hnh}=Lhq^pW7DmI2!+Ykc*QEM?tZNVSC)#T=(hgcRT# z5>vpz^c_l*!^yi!#{jt|A`iYNA1_IMGZoP1a3AUkp!e)OOmgTC<5*fEW~g+|l| zP@j-X$UlWBg)xzL^}8#C9xnV)zjAO0lF45&!khUr7cp^8%8KL9!9l+&)PZWkf_a>I zf-Vo_wFC$M&1=<%M1f#)K8F^q-Lp=U1|%rhxRgRqeZ6F!S-^~}MMIE+sJSWNqyq9< zfk*oP;@p;M@X=^4dD3|3UK5ngkmKr*mMsJ zY%G*;$&)v{QQnP59g#NkB&G_oqwZ;Aa{ zJ(g-5B*Jroc))rNzDfJ51pK;I9U~=N_g#{GLT#v;%V?D6PcHv(5oIh_` zMbT{ofWcSF$LE@W{b9tHZQ%bm1qweoy6gs)pZvUEAPHJ~fbZ0rxktmX>4Ngf6LA!F z&|y*Yq;+OVM2COPqd-7Od?fSS@XYgqsVhH$=Q%|+%z4P>$A-S<3v&vN3!Cfxg!{Zz z!O*Q2LOQ-|bviF{^BQ1g5-nij=Hue>VCL}n^s;+?e=*>PqfAtMZI#R6g^!7*yjaaO zeO+_sKIz8idF+JXLfvgu97+z0+)BUK7Kup&sz24~NQv6*ISESrG?`z_j!>%on+~J)1_}jb|;r5@MgFG_!zkQ9=GVVZn4~I&{ zICG{CR5sFOiD9ZCdJXtbS{h4{DmQJEY1;>RlDbPPK}KRXlLZY!<|7+eZnR-d#kYKX zyC)V9!VqzWem}V=Q!`-YM@Ci(Bnva25c2UR@{jDRB#}DYp7Z0rjlR!3w6TuUe(yWO zUlRcBZ-(IPKp4^JcA)Mr-9McZ-aGd>Z~pCO6sCf^4p&uysX9b)pu*wm_Ws;PH;v<_ zXeJUrlp$)(J(POa=&52yBl;teHl?o6wHeBxN?$wg(veXgUJVa!-LFF^#7C3L+(EzA z|K?95hSN|$4Bn23hgI^AS>e&&j{XsyD8eWKoNIQP$AQF8^e|jUuhx`5hX3}31kP1= zn%j$Hh+C|8QBpu{@){Tr+p8c(-Y(oV<6D<%g4|V-pVMun{Pn);rt3u}Eh&M5vbDNv|H30&K#xGvzM52j~8f0+YnPEUDF8r zm1;D|N=Gt-5!u`R6U4`ny4CCxST1O@Xir61XsB=R z@)&(=S;_P_`;m)SHS7wGqXk24DzISdk9Td>KiBle_U@J*p?hwzS5mWIM`13Y*DL!? zDx$XGDtt(e0RqAq%aU~mO>OvQY$6r*E z{yvrIY=yKZ#8?K~kHBfjviG;dlO%*l6vn}Ng{cExZJ*$dldKrl5bKR1vGe2g@RqtU zM|sRxjur2C#!{hbVHsiLICcXjv@M+*r2xI;XmoM;0MTDS&F)0Czf53LRTfk;2`zkv zjI%bmC8-Ofe(?=GIU=TS#+m{o*>|q<(4r+#_vVU-27_Xk8jk{kvA9lM*+kX0kI`xe zr;)~w{>}~gBeV!wI(Rek88=8IJi|V<;zf)6ri2P>`umS$sPcmVT6!fk zj6Oh-#IrRA^sqt#T@pOl4yLn9t^>~AJGoILYV-LHg_s=DjiAVAfQLZH2?rYf=dfQB zjMVhdvI2*7(FeiVeGr%Bz&+$;D#GR~35H2laGAi-bnn&jh#eM^;dMujG}B2&T8t>c zJOI6`5&m_?TnCYHZ7W=}xeXFLjny;u-99sI@+G!v{&lpf&M&Z=N0#qTzh!CH{pLe<~F z+6)sqp?`!1C8zx|58L>_Z1qoVhDvPjy+2l32uy?6Akskn#3|^RS%b=6Z2)tAZZ-=m z$i9^me$T;2;FkE*lVD?8ReN1UoV>G}z0YfRQiK7Ou1*DqxU+-*od4W2u#&vfh$-Y* zXd#Zn{kRPJ$UjAXeYv$5Y+b-8tx#MxtjOC1nsy@iVHXtY&hyi12f)gd%P*@D-{22v z+#OBh76f=2oweuv+)wEJ`nm7T$4xyI1?kRp0G`VmkWC7$%sLZ0Z(3GShQ_{qlkHOh zp^WG8_V@10GE;PwmGT%XT@|jl<+{waDM%sI3P)tf4CYhb9tso++&S^992#~5Sx2(o zI*20n5asBAHIH(!9KCDU!sPd<3yiLhY@&G+J1yo>`h(cv@h12LWHsK}cSN8$0-hnQ zu8t4}P%3UNd)Cw-4<7LOIM1Z;PSTrN>v^1Iab*n~)a?hii$3$hN(yO)Mb zUFV_e+B?_s4sfAnJEj`pI0vKWrqQ+UX#!hiiK<)}AQ8P@yHH=_(ODY;uRxq7X*9#! zS9gR@27@_;(XCwOPg@ZJL`7%~p}l*$D%1H{?^StEJ70q#w{Lv)g8FSqw$oC}7ReA_p|#Xfv+^pvAIv|VakmyL*-?5elsqsayRBSJBEzLFKVB9 zt1P%yoKqJ^COs{67w4SYV-QUpri$ZN!0i>{MxJ`RldZCK#0^gNcEbY;S`H312QX zV_U%_UEh}c(dw>9lBAx9$vcPL+-as`&|BO=FH_3NzulM*p1KYJ`J$L*$g5>J^~F@a zmr8~wqa|@-Fwp!{%{EU=MPN1NQ*5zF`*p>>#G%O^M$^jgqBq#a(nKWvkb2d)581CV z(xD@En@9K8@k^%#W3d8jHmjCP)O4~%3*zf>VT0;YkCGSy*u1KkQWScN1&{igt(wl2 zbozJE3{P^^aC)t%M}P0O5%$bvlbt5`_J&$uDP}JYXaEA@>8^wf=+1C%(fuv7fSDN8 z@my-|h36I~@T;%CH@Ep8@G4U%_1={zHf*VH)+Yx%X@EnR1mFZVxE+%K` zo}U1p|Gz_+|7i%r_}?MS|IHyx)doOy08A4dP{Mz*eB}fDlj1u7Hq)n<+W@mX$3iVO zWs7j4v^c=Fj7O9$Zo@&$nQ=fDdb3+7SLf%tnm*L*Tg0!ihJ2&J*k_c#ZJ8)_(;ja- zJkDr=u|ogK{(TW%>Kpmol--9*>kyAX8EWq_imEV_9IfwnX?!;14QPQgy^Q#p8meWT zR~B8s!c4TRrBwi~?4wLDD*+`)3sW-2+NDX#b_@-XMIUU&ynFl(zBH#! zt>mSclmX3+LncF4pEquvRS9n@ANHC4)hasa;a-o1--fuY0rE6B^59?+_y?SDoUzJZs#% zZ!i3SKes6jEm@Lp(TloL-ukW#I7$4j=4j~on>o{4&dG7KXTG9q=wf{l#b;%&EdjC0 zbEp}n-G;@9TEhi;BX*jJ-4(q!YWM}s!dVNWIgU#*3o+wE?F$HdJI8Z(Hh?CX6zbW- z6#K0%f94hn+oZIiD=cYx9s2?|`v`H&7K%EToS6l(=lKW%ntp;~=#!V$*D4)&lQt}xJK>Nci4-~n4;$RlipbxT z*StU-C*?G#s~?m4)d5h=c+4kHZ_a$yr?QCs{uxg(KMQV-CTQAds9g$LYsCW}lsQVG z7u_N*P414hj4#=HE2hLjpyZB_ZF0jucqtS+num9T9>Q2-iqrM z>n_7C;AvHOtSCFyxS=moYPW%1o=)af)a-w11rU!_ed`Cro;e%nEbsIm=j=GL0~p4J z4o|j|p11#o70UK~a^MJnJ&WsoM{YpPuV4c-=GI9qirAs~xi?<`8$n&o9kJ^6DB!lY zCtB@Y5B8z`vF&*;b{S%DU7(X|<%S!ildE)%P1A1B=`;Ixf?f`;r7-0sgKa?8LN2d6 zyGB&r22ABlDopbi8Y}ZF`r5u=p?A4_Tw6haR@#4tFb}5zpRW8lHTi&V&w5{HTX^HT zzzerv(quG!J+O`>A|9d?94|L*Kv|Zz+|a*^b)r*LlXvU3l(>!~wVJ@VvlmV%Ty^i- zMr}NdQ*DC0HiJ4$mIZw(_*tshzUy>OjU=O&vUbcIr8a4cy|&~2oNOy*8^&BXukxc; zcYb*mhPJLUGYx8A9$ie*I)@Q%X%W7tRL?*U%!<4@rYfF_AfahslnOrBlnR6~NG(uW zG|*DHO3buG?ASc;WdD#0C(>w>`OgrBkPwmJq37*+;NRRcNt7B)01IW(^7ZusDetmmV-otI{$Kp64ElMN!t!Hl`cH#A}^ z4a-7tWfi<#=5f(v(p|}Z5kkBKx<1HW%ZDvrS}evXdTtkX@2wCcQkpiKJ@eV#LnL7( z9Fp37oj79&N;!l%Ji@T(oJ1;Ha6yRt^t*}NAnz||xm0JH%OyTtudk!l>F&cuhzx<6 zR|#FrZ;07Nmj$Ql8w~~9r*)l3*2)i|7}1rXmJRr8{sjCR4m{BnM?wJCrOPRqb5B|5m6Vg06DOUFbUw6YQA;o(3-zJt_Guu9 z%+Ed3MYj+nR>6xGHphuWQ6i9VKNzw=?j=pXEufLbWeWb}!*J`ZxQ{tDR%Nse=?rZA zcEG?eErEu)0ZYGE4+#*t>(YPM6G8l}=uXvr81Pxhd8fWi3m#%YLqP;t-SYlj<7mr| zH7{k$Pi4Bdt%0oP_6E4TjP02rnL5hsbbn0u6x22HF*P^^w;wP=i~z1;x^}`tGzL8M zemU0MuIdAFy7=55_s{h)tk;I`K78C*+QxgXLf7d2V%KulJFa`f@!u+=h^WP@Sf15D>ZNmINYr;T(_Ou#>)QmHp zb%8Zbl@rBC{WF9?j4po$^xod?hdl8=-TBFrGOuNciHl368)3q*VwiS)3tc9@B~A@( zfckA+t*jrjd952O$jAZny22RgO)F5(+#;rA{e)))*1K(Lz8M7}c4A z0r**Kowr5I?rfkJg|$c2apTAlTYyTLWuNoE^%9+U#({3Y&Dz2+;N_3OhJ zmhzzUNF;BB>9&omBFW*fn0u%NXZGj%z@~K!eYQkzu2=o13e&{T;q3-+WXQpfLg_aT zChhJH4y`TcQuFx;{UTw$AiL!--E2>nu{o(g4r7LLKhb(N9KC?J03df<1Qc18R< zznuZ9$TBl&$q^;!#{wq)7_=sVu47PSon{?Cfkp z@9h0?K>zVH;+p|SUgF&Fuh$E6Hzyv0X$ao=<*9YSr^}tPW7e58sNp2$W@)2`rOab$ zkHSvTbi2jC+=F;Lt^@^O9pLF)p~KR-E9@gi7#F*Ud(ALHbhi{Jy7wX(8ySJu6^dk>p(s zBC(d>`!Loy0ic$xd-^K~8|Yl`^Fk5{UjxtsTTB~szY};JfUwUSHjsrANJ3WnxO;pqmEPH98yrGfl)BWiYk1q)=yV zDhv}{oa^h$h3QRn#Hq*FC}WZn{{0*OuoLe)^Ee4u+E4<-5&?K>UHO<=?Vk$TukfOM zutQSbz7?NbN78&R;ZpHa31Ia_JACmtvs026{bV%N=E#1{`fy9=U!OCS6m$wy>NGU) z-$~5qRnb(DBbrO+W3>=v!Qg9IUv*?eet^MstGf6}^Zt?z6E+Fy98(`;Dk0bxB4)HY zKRzz6^KtzaF|IW)j5ZwLq!&&guBtD$VuH`6W&LiWuYyyjcf9AiUPk?cGW4tOw{h?K z#v?Nw)C4ZwP?aW}w6iMDPEyY3(<+eoi_+|$>GnsJh5<;J{t`M^E+g0l$SDDS#+7Iu zAQA&*_B|)mc2OA<^RVU11~#9a*pULaJNA6*~dZFn6c16?DL1` z7oqa;@n)R3KloFP^R5)H4AV znlhOtj~p1iTcO`_Z_;~U;NuFKt!{+amgmp}yUDqUXWkGAX6^76$&Ye-y_~tCB3&r4 zqSn)E<8|RiRrs;MVmj(K@?LbvlZmfCG>r+dkWF5MMnFGJ+@~AVH(J#fWcKU@BSc(z zjw6_!vB;nUc5aEV++O?gt5NMW!_$ykro#5^TNf@2aTRR)Bb{pRYDYNO-_@4;jx>{^ zLcL+@&nAN9LXdyXZsGt+92+DYg3tfAwcfoCYt8;JXY0tXH z+xbK9h1UX$^4YRee8-Z>WgDO%?}S|@tJUpqz~yb-N`gEIqN>esVEn=vI?}wbLYU;V z^-CZm8D!nR1+oo%nZ<)!s0tI(dhPxG7hmcU!QQH>QxPzN_+BF5wl|ien_HA(Fb;(K zrUW=qjtPT}P)J5#pbg>rg-U1!QbvWsI!L-xlcTQ&s3Q9(&5=^xzJ?=8Y~SyEHt(%f z3sLVrb|D|Z+R}TZnT1u&LG5mSXaF>}NsZ`s$s;%iF!Q4aM!Dr=7%@8o*Sd#T$yXpq z#>qq+BxxTTr1TBg3K0|N%)|oYF?+AD6-geP9T-X)>JnZy|ny2bGI=~a2 z5AS{W!~0m(%m@=FQ!s)h(@RK=Kn#+X8||IqY z@yVJWsBx$Wf=lq6Pd9vE>z*)ao0ClO_varjaySoj8LBK4T(mG` zJvQH#p#|IUpP##I!>}7spYwjS$i@0 z4sZ5O^_pF`_eDE!Swx#T!*0GC%!YQIxC(x(ZWW&W+3Ndpl3W>Vv~Rk+*)zK_6K!j` zvM~@k}*`JvQ!`5-?q#Yty zK$2Vs*0ZgvI+>I1UXksj`>Q{|*6nvq-!6OHwY0Q~1oK)Mx_RN3fGVV%e~vk7e}T z?#!lU11!&ZZ04)06UsiF9tsS&zrD$nsYgSk?8*l}g;QFYAt@)~3 zi7B082dF@_PVB<|;44Fc_%wnqu*QxpCs}{y%Z?RW+^BxIF0o|mux&n5vKxOLND(yrXtKzQlDTZ)-KhfHl1AnG+kf5`Y-?1c z%^A%D=Q2J3EU|NBgP^v@Ws~55;h*4}$>Wkm&Zw3xozNvTk5_%?lEOAN1@byILGTRWOp8GdWc8ZGOJ#*J+c znz*;w%^auR`#wkg>~2UnY7BMW^3Gn-Au634KL7f3PFZOoo#VBX;>m1Th~y74>r}PT z1^bOc1sYKBkD=76)V9U}4kVQR=nJ>7Hjqp@BwaA?h6-qkw&{%664Z3xd+D-b&6na# zr_*%|nMyUSLw-2UY0#W+mytsHn^zQ+4TWp50_~)h3VK(HN$i5jE)*J(l&T>Q!5-}c zTv@5_H){}^8tRl?PPZfGhXp>P7_ND>7|v18FoI20qAH#h#_~z{7Vqn+U!FjBuC56^ z5SGRh7(TzDUa_M;Uz+S~jsVp9^Ka+_wKZQDod(~oONQmnUkGy$2;P03?){32tx=`C zD0@*XYosrG4sIdTWhb=Gj7N>T#yO&b2P7HKC)FiLHRtlgp!mXc*AK*?G;MPw-&A;K zMN@I(hssu355h_jK~sXqb+Vfhn{|ZlpQlCGre9(^w+9a-Ita_{5O04K=2o{&%z_0~ zt1}%5gUoktXE~~tL02PsT{rkGHh53WvuGOZwguIJiDsLW3AF+HcdC)|YWAwXEbXp8 zU{zlU*D@ii#cjr-gMZq8;LyLUUAkP?;sMY~6f$+a7FE*7FCt}rOj}gZVF!tcJIYy0 zf6wXfr*C_$?LD&qk=q=~BpL|ck~bCWH`q_-&2tsd}KAzK+ZN*(O_C6=nZ&6zGd^LGGZjm?7UNeC?$iutaW2Y*fyT@Z7T<=NI%`1V~@kj*~ zwSDcjQ3eU$w3_s*x>VaUfmlRm-?-2AT%|_=GH3Ti!I~3gWIW5aD}h20Q2TPXpPBMI zL+2fXvHeU!E@fiRN&NNo+jOKiPJ987cjteEO0(lJY+lgrHG6POr=6;Y%dRIn7@y!C zvP+I~9;NTLTU+xQYdeY8p(qf&E@x0yx4Z@Kd@!0tyrX%XNKc+u64ya!Fzt-vXTs2U z5tnhZ^ol9$6UfZzVHM14^C*1o?{F^({np}F(2ino3Zo?F#xf;74y$LZp-z~LeeSTBd>pyP?&nuUPPIe4wDqV>7Sr#rW`UnXNx($}cWudstEf0Tq8A>J-HF zbC>(U=R?qpL30|Ex7kIiKkl!Vl-SDtItys%JB_v>kdW##%O3E68oaOoUgC7 zwkThgou*NJL34hLk5)gE)Qz~GoC1GJPBDH7kX(k+>^dtwDBrJ3h*KZ_ZOtGn_5E$l zDF1EEuVo*Sp!Q6P8V`<4>#+gU-HUkc3O^Vj!q&c0=M0)BCtsw@4Z)^bdXFgPEveBP zevqy>i1#d?FWxqiXAvZWr$0UI<=z}k(R9;rx#YLj3IyIMx0WDSZ0xkAy}qbwy-nJ7 z@C!e&_*{xRI%OM463ap)=grI8;Rd)}Z@Hf0`?S0JSS>LT z1`ccvmH}LE<9#M03486o^6 zdfL_cyRQNHyRVr!NNFYebsWp;)f7`YR!@gp8z{I_IF4U53c@L3EzOaeWLB-DL&9eV zPJ)Fy9%7P23?3{w!5Zn@(9o&0?rHuYW(7ApAD$uTjz$cedjzB#TCT#KOJ1SZ?49}2 zq~vyWaY{cLZ}`v<{b*w~ekD{?!BlmI=kxxA=F52TiKDE>_~>Byd^%ZeFm(o8-!@>z zd~Sz4TO9$Kt{EqUAl5pKznJsT{R+L=9s&xId;@3PFWWvhHT&p*6c}JleBXk`^Kw{M=WtT^aY#{@z-1Sxca4Lb3p_ z^nIbm>l$!iGjBs|R{iV7Vgm0rIkgqz*<&+%i!S6htJFw`@EG1D0oQw#LWVbTp=Drq z6>{5AqL5F`KMR{*ol~L7DeLsy8`)AeU6a2%cgr^D#NT*q_&VI&QjnV_ zQJ1oersV`isih@kMWc<42HAzVKc^tA{om{o(DuNh4m6LRS`4(-X-jad?)=LxncU?V z@z&2X3)-c~iiUN}j0g zw@E88{EWRKD>XEt`e|t48n5T0!8->Dh@f2U?B+eJ@zSb?b-DN%~hV>(Lj%&i%+<~u*hjjK3kPcRFiw825a)Amt+ zQ=}$kiPz{=6?ZwgUFe4Ds!gup^P1dNnTD&>%-trW@n7{?CVjN^TA1_?$)ftBA;b=<}J|4ltH&t4nom|{z=U!esc(Veoq>N!}TrB%qu6O6}KEJ2j1 zzwk&#^@;PGq3k|XyB~bqeTy|v9tq{n<>m9^N>6$Pre+TQNzIJbXeIrVnh6jm?@*N4 zKX|X|)M;I0CO}OKm6vVTGD#s{z)u;iaUJPd*ePd0PAO&0mh3rtl{(V?bQ8#~)%zI@ zopG3o9_q5X?S!hX36A2Zb>fg#f@xd(g9Oc1y5@@m0_SWPPuRf*TG59BdQp%pDccmx zX0@g+jQDq&~$ zdWPcv}_FZv325hrafaEnD1{=*8rGf{?u9{9K9lqUbYOw)fXRc4p!DjC4-CDJa>?T8;vxc< zGQ-|xg!+%WWFWD*i5CxamkbL4&J4fXfvp){aB(tI`F~n7M|bAE3KL{nuSe z`j@+e3Urr-M%UR*H(>8b-e_Q@7(z)!J61YJKF`3}c_v7fKK4R9gQj7Q0ecq*lTW6- zENf2DgU5iIRv+aGz^acuFg*+4A2BUa*jT@pRhE%=?oo(dd|zB9RAxHJaor8;%U$mZ zA0M>>JIg31uEe=CxU`&-lr?d0qGw$wSSm_lN(?NH(<#(!G)L2$QWT;g@4NLt`cs?!1=hj@ z17iy#V{RzoX6;B^1{ZLT+tr(^eevD4?;oqfLi7FmVsC^+)P^s5KS_A&NVTf%94k!fbZmm>-tr?iVt(kHEiodOy zdgjPrj|=tK)xdw)rBEwdCLV!wMm7P`L>Ew)H}dHnENr6J5C(tU8Q52Ns%leGAiD%V z9dpB-9cE6_ULx)tj1F}os@v)P| zSm$vg3~3~e1Dt1!pr0MKbVkrI|E!QICjX0FGNJ~uOP{u{y}S27c4_=S>=NR!#*aiv z;i<#>FJF~e8>AC~{V@~sk-KHO%|0#fuEUgdPr$f831`HNr?ags0O7^Y#N zjk9o2jcoWC&QtpJFqaI>Sr?4fZKb>%p_hk>3T?t!IIn(4DC{5o)6@ViG0atj3p2bl?9u*0 zIy0WB3;6`iwNf7nnzU5_&%4&F%TF!DL;7e{+R!$85~fPIqg2^^5F~oVRz5S19WF?; zEEp$V$E9D>UOr z`^GigsxBcwRzXLRj~Jyml}U3LC#?tS29^t2es7{N>u-#O8~0J!Zy#w;k07e|lW5X^ zZ{#8*Qau0_N0H!0xa)(98$sDuMEVG%Fm!xPdJls62`BV7G!uIh4Ghgpk~pQYnGv59 zPW?5PEH^?T6j;Swo^Y|R^7uCXDh>h=8|02SbBi@wfdJm^e{C7c&R?5vY|cwAZwFK3?2rm_btMcFywf^r79p0 zN|ezNVU#eyMrYYmjqNNo9IEXmO(85X+;4l8_QJ#`F$uN{*~-04za8qoLm@{(4CrMH1HU=T zps;`6r3RlyP=;1qB=411)hKe0hm?g!j{adf5G$34*KdIU4UIoc8;c4<9BT;6{8AN+ zjN1!p6_$rR29VTIOBiT{Ez(87(|!;Q@ee2D>2=PImLo~Wz=FWW)>gnERVFzW!VJ)E zhLn1cHG|mYflrmH_C)7e(}bkY3PbKTc8L-jEXl1HUyA>B>hk9gTGMl&nB+UR!+x;o1l#h(5_gA&@U)Mm;Y zDm!lMzM_&8i5PtrR@w&xt%+kgPD9)P8TK*LBpK!2$Ts6V2+Txa0EHt=dox5qQ2;&21FkiMgqLm*}RYt%B?wmX>)bKg5XkUz-+Jd2|Nhn9~j8OtWU_>zX9VT6L&YTJx3z^VNjw z^_ij(elUgJ=tDRgQaSXD$lJL8674NDLF=LQL`b9RZn^1M5K&~vOX=G0oB z0g}L@zA(>pI_Necr({T67d!qk!znACo+n4UFCNXRf74$ zL%#RlnL+4S`pl%x=^;pq3j4wdXq~9;A{lTYAWAg}H9grlIxW8jM|#)3g(Ta~8_#IM zWNQ(9Azh?)wqk|HJ#I`JKlg|mQ2w(^%acfZxlo%&7<1Hog=Yzl$-;z#nE?xI%)ljU z=g3%BAANF5b*i*IstxIIz5lo+#P(cB$#mOBMDgr}JNQ7$`silkgb%9kT5@@`{#C^b z11~^si6v24^1j{8%o<`W1RXt2MpcHKGe(89A#;eS?3W?Chyf1)*x`Dd7FkoCUD?H> zP>|*J5X``6N&}>P)M5l7jOn{aU(c>_9iTg2HUGa{Q5s`Ng{;U zx0tADL2mBJ)m}%DlHfAKH8*_^P7q@i#$t|H`leoxRN!P^Lt*e|j*|iqhel})) zk^k4231dj3{%p*s5Dn1mL7KRKHfDfsdTEAc6RvEH4^MeVnsy8m7ZmQl#tc&_2Z>af z6;=Q~M$OCv2mfJMrg6%a7w!LxTN+|JFCd%KqA*{{wXx(&g^vMpOQ}F^sW5uyKW-_? z!b~?{*MD>3f}I5<7jN(@e=>!{{cYg?Y0OCYs@atc=VLLD)@RI<(M9Zf@ zxt&-uXn33|&_?2SPI=`84j^-3Y*xR)^a`k&S_G6{L-$Sr6l5niDU1t}; zy^r8~9;s33VH)TYoxy1uUsj0hA$hMzoj^oL5CpWGl&-vBeEr@S!T=gZHG$9+mFGDk zGgIX}#23@`kAlodXP16262qOtPr$!MLglFG7gOXEMU-;&^q9bH%_eN^GUzpFdRPN& z(E3K{0Fgqshun)WfcuZD=Yh=}GLk`r-&a?+$tQmyeStF#sf0a?UgTZ}9z{^;zi#R; z$OP*J$4KTso!yhDPLYoOTJi^%QZnRwR0GZ--}8>YQSF%f`>OodMN!k|R^51!+2)23 zI;`3u)oe=qlYE$AYj~Bu7tafX0y4&@&TFNAH+7I9X&=LUH09pZQfL-CQ ze7w>KkJ!~uFFZ5MfFAJ2)Wpke*C8#|;$oD>AVqS#4kIJ7pfY^zfj_Md+I=J<%H7n+ z{L`|?uvp1mBtg|m>e|y%suTt4E;Q7Z9yE58Lj!_cqkT5&FgqQDskWUgM8qba;!;>A z_t&q!OM<@l<$h16i$=@{!%51X3x&+Np~K8W=VskDM{6F-mmOzF7yJM~KG7OhOa?b!LLTkP!QZv`s7_g(0JfZv#6sw9J|X6H%3c4A4c} zeBQn^esRkc))Ew-otfkRIy37GY<;KbYhhp^CWzi{$s`ufV`T`10tzieD8O+|EO1;C zS+!O-$@1IN$jnwmAJRFLIYxi=trArMu26F3$9$>XF^)=(TMgbu1THG&m>JVaHX~G< zy0Na&xePEglcI~T4(lEa3oxl}9SsQoZ(XATT-Tu3|5`S8#OO>y;HQ>6NG3=~o0vr^ zaka-WzAVopuikbOaYLwnFafExr`%AFZL}Y;)A>E{4YY-2^%z*1p&EHT@W@DP{j(h5 z@n2~Ms@uGSy&H$Vsss(Di5Qh1KF_ zR6{qxC51wf5pp^{Dac0@$0bjdV!#?f+6UaDCPT1(T;PBUR8` zfk?F+H+phn128z=Rp)SQ_jdx6OzDI-R3@0iS)Nhw(LJ?Qxw#d+ADfg6Ohj_bO1j8% zKD6c}S^-~n)9!0mPj?>SKh>pm+0WJtwnmf`;UuIQW324Gdgm4@ z!SI7RSi*T_j&Y{raiz^q6b*<(W#GUDXbYxBg0iWD2|)Zeuwlv@ing%QlqQ{uxJ!(O zn3C+~*T=e`4tC;EVT^I!X$k|fOA;H${->F(+J0H2QP{qT<_qX>KZ7C(b^vwRWCUt|2MHQG7a~a{))5fDb3An zEsy)V7{KYMjHTKA69_M%UR%U`!b_caH`LSfFL}QOFNwn`6qUA}NdLi05t!wpEjYm3 zOw`#IXsXe&uQo1d`d*7POfzo+MY+;{gqtd<>>E4@B73)RRD^ap{5aUfs@Lj!;kR_@ zRUqzD&I5&reILS*hLu!|5%DTDO2$9srTEq=MEa9O|Bbsq-|q|Y7f6$r<+TXUJIlPU z{$VWR$AQRKEs+*ELn8aeO4Q*Jkc>y!vr*F$J0@`?cRve#XC52-3Kqh-83(6es+`Y% z5HL}`uEdnK05>-E|28&H8rDXM>W3u8TOugsd?tQ7?R%+zkiRmuH? zpyW_GADAOih@TM>n7=n}PenXrG=;8Ft3v*7W0S^t05*$*khCOb#PGg@%V8rfY_AmJ zZey2B!iN5(B=9N-xUtDEPTd{ZO9>qn;zoNU{$+~BFe3eLWAp2CW20LK+}LC!#heV# z{kO3>2G2m2oW2$E(|=*UzvK?bJ{u4pG!))!{7@_rr31OZyd(IQR~{ac^;Kq5F1htN!u>rOO3~xb#b?-g z+7ZvJB}>=XTDvaoZ&yjzB9?e>y%h#-A?=gsASHw_4NW1zkAl%K%GIj3H6)T`EAIqZ z%l&*dpPTP}1zH6gsJIpnVOzc${#N9zRcWTs)+2>ZcA=v%YRC)#@Kp`(-XRmL6!1Mj zoSrnu!15R!E6K;|wmJKC#fmj(|H4+|15Rgj$tpYcO&h>f@Q>ix-JA9yWQDQj)o`n- zy#VV}STcL5ww>$*vVSogAA{||mTR`}i2ht4p1E}x^_rbtnSxRB_nD?ts2Yy+-`)uq zt#hIfDaY?pd|S3fpi#tw8!GC$0g+eDVKV06_Cu)w_>uCGXZ}uh4?T$d>m#mG=KD7X@*=2gkp0EK<4>lOx4MaUnyBTpl*Z75hc|ZCeS>=l4z@JDUZy4B9~#~ zA0aH(@=R{!oUvKD3~*xYKd!HrzpL;$+$T;|)cm-@M1GRvexkd{4lbknCjMwGhgaN_ zCw!t!Scaw&eVdnCo7{H}XLetqFg|Q>9V6bk`iCkY9RfvNiue6?Ys@rONUZ|uC+JHk z=B}!lAm7~1mhjo7s7S76$-@ifix{k#I)9G46F9JuuXEl zF_{2!Gc&;43_O%7lwvs7K zrR+t|HY3#;+B-%7Ic(eVh>zOJJ~Hq2;dLMf>?fU%Dsihpx6z{E%TYtuMzR_?Bzh2v z9QNT~eJRc|?I+^Jq=Y-iz0L^hebYv6u+?DjP0on~wsE!E@gNh3IkJ%pao=0d`|QWt z`oT2eMNf5t)uykR6?Qpy)36p*Rem+$1gNq)dU6~Z8};{wHr;wIZCsX91a|h)mF#+A z(f1*~Jk!e?Ddu}k9KRZjv)NDflIJ5N02z~);6{vRc-Lw`b_q&tpYd2&&876-!32H- z+uy6p?~j0Hl{}DLVrbpQ*lE!TcSi>pFj54&liVu5_RmlP*`@9u`hEbPx3^iqBMUJJ zhh3)T4j(L*rk9x&05X=V$H{n0nVF91OmEnVokTrA%VA6!Z#DsmU*t;aZj4G$WX2O* zZ%yr94BzOlBwoqhPJ%^&AcZa-gPX^a^qL#Y z?viDoI9DakH7LZq?+@Pn%DMmCB}CxZJp<@24fHhx)Iq2N2R0E-&nJ7?nk6+iKzC{H z>yt+1ou9okQ7c;;EgxR};G$RSlTFEd8#K>{M3%Ut=l!=bpt}T#ADNC>#i$7!*xbri zA%4m2?Prqm;WkXh>rrXpq=Kod=USMZ{l3!Ltl*%GnB`?R^IK@sU1qRAJEq?{v<8w~ z1ThYmJS&#Ed@8HSvAJjH`P+=7Bx510R>!%aTwkeUyz5^`ub<_`6Nly;bn$>LnzUnt zz4cqNu5ffWvrq%=*;?KOW$(v_x5l+UZx{Ht-&kc7o)oe$*%Td9YC%0bNr3~KV}&!V zPj~6|_outmPJgUB_vtRxf4WNs8fU&GOkaQA?&I;GmHyAb=A-ud<~=IaKeObCao|rm zwE!Vo^gJkV55ceUGVV_x=;wR4L2f#69kV+39kOsX){3orgzd*zAgO`*Hfv}DL8WNU z9(+mFMaf1alE;|o(w~x3_~a#RZ%}3dt-I_uX@Q1chX(}Gsr`R(9S)GAzNrf+CvM?w zlb5lhq^4fVDqB_)(fU4qytk3F^})9E?g4zLLxo+L$Ck5q$wU)m!Yq92mzp2>8N6RP@@B&v;1`4yl1rnt+VvR)M?AKZwy zss3EqV{@_G;S^{u*jR%9gO{{pzksyVrC_aG7-63cC#Sd15BSc>@YONXh?kc2@slVA#eNt$f0aR7_#D`@ z0|zz-RFnBjxYYrl0~^-QflWwh^S^=3_qU<0UYvge8y~zzPWqGNfPVv%)sKyQCDkxTzm6Z>Eik6D&=hkqt!VmO-xN5yvxAD1b5{6o30WyO zc!}Ocg69kF^uU*DPc@sePm7Q~q8j82e03X_g56M{zhXDLzt-5bszxGp$J|aKHm>4y zn3>~w8MCyx1c_gJ{*=YW|EW1o&CU+{arb+CtajW3Ebyyx_x&_UsA>74@3L)Ik;^=h(mkEYcJuvH%$*o%<$3rqBf&ZR?bvHCFrkishZmz!2=+S78$yi9EX#Fh zYTABle*NO2K(2l71^?)|9#*M(U>yc@MW;bSy(`efIO=+xlnR!7(`9E#y^X)*&AjYS z6+mFgv+WtMX$K{8ch=W;_C#k)s~yrxG6l!}jz51Jq5+cvzd2Z*Z-{&(?)G{8d%llM zy`Af0%^4)&r{Mcu)6mUe8}f!G&TkcE)x_DY_wh78c(Qs%-M-HqO<@H-p*I%*lFk|c z0c4j%r2fKS>;58N2!Sm`Jt->cwwNs3)w9T0BTJn&u5qopXaByynac6WF7f)+*y72* zgum?~I}vvk*Lh#U8d_%NAj|oN%gnhWY4vNohLtm{*Mnx30ym|ueXAt(^9-S~$iEKo z4gx)tbEOb$eV#uK_F^qA?J^LZB}&ARo!pmojkwdzQ9T~cSNk5S^3KZLU_1~nKw3Yh z5ZyavI^Wq!*E3u8GdWqdR`dp*N0lqN_;@`T2jA!fvdGl$H+TkPeqVQ}vnqYx6rZ(I zsk{%t9KSf)jtC&QQ49vHLdqg7_v)j%^=uQWXsBGw=Mz|^dT{y0N*%{hZVA?wf~B;c zG&$r_iX#_g&)p~k(A2^Z?@kdFS`U;^3aIDkHU`XCS?Xw1|J_OMknE`DD&=WTnXIio z(7&duMrw6}U0cuWS2Y-VZvhiY(oy*ZAXRfUQd{2H|7^`L1DNZFC}+E6_WfdW1iaK^ z0RrBRPk#VDVbkUx2Cqo{45oSi)0zo|0od^y8ytyNb|nJdp#i-5*M4Pr%T_%Ym;mH zQg;8MtFNGh0i;a#U&sB%>A{#10s0-K%fK(kI~st%#Bt<^AKhCGDzG+_(itq-Ny(qt z2ytqZK|MLQ9Ft*_>jp3EeDPDm!yNe{z zBfqnY9^oj#c=1Bit?=Bx+;g~DI!9rKL!n`VM*XBw2+#* zcUuRFi4#-yGgc8boYqX4ffbxWymE1jVe2mqsAjyL1vW`Y`smx#O{S)SKen9qhEEXX zjOmF>2QG-TS|swQlC*RQlBzQqen2V7Bo|$#=_NnjVdSu!t*uo#cYMG_th1lRjNFHsiHf5J-w?7trN+ae(N>K4Fg6%b`>alv381>d48m0t%ed%+({$~vX{kew8cmu2! z0*-~h=KciK>@RQEVb-eq5_wDTTvzx4?ECj))6I+qSfkPRx^r1*pHI;Ym4>)wIrF(R z@w-?NczFCOs!B6v%h48(Q;+NS>Z;XfwQTiGNloH+rhJnfnVud~w*ZUNek?|#>|N3g zfmJ-i^|yex(i}ho;O%L3|Jg5W8i}XGY!t52Hgr8wJb0EtK99~uzTd`k@W;;RSSLZ` zxnmzSgH#yzw&im8dB1^bAk$f z*4%+~U=q|%(dz9Wbb>Qe8OfDr{+FAn-m8ZPe7ov2yn8Hw2dEjyR{!1;JXP>bbej|+ z$AgQ32K~&z>rrSAvxl>C;rAC_!k)3K)a2XgGI#QjTdC-(dBD**VyIC^g@$GyNh zu>3XJsq(0%>mx6M6wm>$fC07nS4~&moLq5R713zX64}PUS>1_kyu!6CpQwqcDk`6X zJ^Gw%u|Tfy$teX*n-(=@*=xtcL5b+_K-9+wJ|J1l04vJ1hALZ^rK5V!d-Fxn%dnar zhfxciQ`1gTaFza+3(bfJNSmnUon&QK{PtHF4X zQ*nVtL|Zf@45O;#_4jA#Vd+91WpBhK8?i5#37Up-Q)d1ivR( ze2+pvdGE&-X8r^D4u!GYfGXW5}XARaCPMc518$6Ok=@iaG-xEkmeh_xH>ljd2z_aHq6VZ1QoBzmwex#+f+*RmeMo9B}0 zAOha&9FwV|yx!YsQOe#|Oji4kt7kMP~mVr#VnDqF@BLK8K zLyKxmWDBu@-PB{Woo=fl(?Ut&eKtN~nTSq~d`MZj*2=A)i~=a?xr*@x`Smsf|sHaXQ@I6ujkDY(YCIs+%H6iE&qdf0ACq=YCUS=>Xep@xQP(vaN44}D`uc&2`FXkjBp*l#0(njhaGZ7`Z62&%uBTewXbd88&P6jdf*&H1tSX8) z%G7RD)>E2hC_RrWm%syyF0Mk#^5@%MSl$R3hJE0nSC1cX+-)f)f{VGzAw$ZF-0;Pf z=EpNb7Kzhv16cW=KAazBU=D$_5bGi1o*KY)ffE7ILdtg`OH;wU?L%p`u5T&xz+OWuy4ZLGnmc7C9=f;$1XxcvW*rv}e2S{ax0#HtRLLhI1@ zJBWUn6h!!0&5lYP9~z_*?3{0p`LOEh??_vQ|3*AlROmMZ41~PYLZ!jtOC?lS>MH)>iJ`Q7&v2z@+ zF?zHb;FUX!>}zEhVI^}K8D&34=Ww4dPiq+`_~fH7VMww19e8^QcpTH1iZ9T$10#{s z1T%bWcklUP_p$YPGa{ zY7I=LFZG2+?fA2c&A=b}6;;tm8E-%+Gr=G-zLJXe48&yZv`r0B#mU0K2S&nIQAeQA z#yRIj2r;e)Qh5Dh`vjN9DKt96@$D)DG35tB4On@_3-;!x7tPK_$Xt0{T*BA9ds~Hv z#&2*TUNvQW#zxb{Wua=g!A*&es!hI;1{LD61O^tPg2&dlI0v{0Q1z$Gl#|lbR8p{y z&e1FW*)a$2Jr$DFAw>r>o?!IAutMP_;~CX~xn|~)!3UM6EG{4wn^BpXl`X;Hv70zqHse<*a1~*lf7?t zb)wSaAfi^r=?_sp`3TFVxgTWoGQyA$tM3Nc}1H7b%QaH zR(Hmxccz>~i#b&sia%A&aDzd&hZ=xk6Id!I@1k@BsK$A zd@F0{#rQBBuN_e+AzS;)6nDWpxavrY1}D{3=b z`;)<%kc6zeD3LH#aLWuLHOo{pJg37*#a}5eY`tY(v7Ckb5*ZR?I}AzjI*%fPlA}ug zy3K(9a$yqt$1VL(pAs6Ca6SI;I_LwF1SwTlPS%9w6w;IPz_N?14iL7Fi)x!v;VL_6 zN9=nW&V#eC+hq>T@v6zoI-c;qgubSNr0XdI0&HE#&A3)O18-cmG9s?aj*Xd4tkP{mpC6q-XJ33GyO`b2&^KXIya zlhVbU+}vdQZp!t+!q=q6waJQxo|asbNQG~O@sY|wiu#vojB4!2M^)*}8y^^6ko8M9 zTd`U)1Whr-rpKKowl$^&M>%Uxm_6`I8q?lgC1s5!xm#<_AW1mu{P@I{aImA`CWCU8 z$%3UBrztr!oTIGA6Ew>)49iz#gLkSCo*dUgHmAXgWI5NM3gxX|Acfgi#dTPv{Lot~ zsb(;GmXxiHeWJ6R(xhmMUi8}8IKa&M!(m4#%ZpL*hs)wte7TAofhhSHY-Oc66cD~oGB-961+JajesZ`_j^i?B2Na$C56r*eGhb9NC|8}$~dNA->mp|`XRwaWzL}nw(6emJ@!X# zcsTwF(Z<9%(g}qx9=&;J>>rv~>?amY{|8%AeT!2XRwwwxma1Se1L38!`6j)Sr;UzVfS5*(NeW-@X}oZxgxR zC$=>DOGNSi#+I;ayjMQ4C9m(lWcS(k^pJj@gVq-#yUCY?`%+GU-`J^>uuA$2K&}dN zd}2!vp*rELgPr|aN-$2=;+kI%51>NDnn$_C;3b*aYHAiFVfOY~&7Cjg)8=$HoQ>7P z&<1vq=)_$$sTDC8#`QU&Mn(oD7#4GCw20EVaSzQyb9VS=n7|QzdSPUC>rJoV1TzAJ zNJ8W-k)RCP7$&~8x|8e_{q8U*H9|%PTX_-|U;x3GkTpQ;G##5(^Fir}EIs z(Id9*M5LsQR>1#?rFGnA^8TH`?Zl{9mK@2(kvQeUkC#c{`sHORI|@9{e4(pPX`+Lp{zocS{z( zE3*8~wl>3tuyO$(w;v6jAIn<}ACbei0F+!pLduYK1P$+Zf(fb`KV;RxYJ2{Tq#)xPaB@=<((t$3TC^lQx$}#d;2}_dJd7@|X+Au{qa>|(4m%kWh0Qe9te~!6YDf$3 z2U8zz9I{})uDK2L05ojjr59g$5;lX9xwyJ;3i#Y5;a5SRw{)h~M4s?;pBsH^>TCJ3 zkzt30Yjh85CkEPI2lcgGI66M z-=#3>U?mF64y%P`=O|i*b`3=P!Kg5_L$=hrDXgEm^sLn^50p5yP%=y*7eVcPK>${b zjf0*6xW4r-+B3AsU!7pM=4~3dBlJHjGZZ({NK&##mZ9^!TnKgUYBQ*xl^H%LjaOC> z%@#EK3JGnlgJwp(y#jxowDVi&O?C7GpBw&3Dcw7p!45hZ@5vU*pZ{B2f`u?3BEhQJm* z9IfePYB*F*XVAM=Q!i`D0;bQRe*<3hvYz(k`GB5DidP1$G$rs@Qd*2p#)NoC%TXp0 zrU=rcwi>5rJ16oluQq0A@@6k{;E^$;5r^@txhwC%(ocM0=oP%@YAQ10f1Me~bkCO@ zwA2YCkHoqN6M2E5U~$8hrRnKKL%?4&EbQ1{sWjElCtVm{;wpR6zfX9s*t%Z5LRQR$ zz1vawwq%zFx`OffvzYPxGneRxieioe%TpwNc~w>A{bt+*V`nO(5m7nlA#MhJ1 z1Vz8q&=WppBnHy_nEvYDR!E3!9?^VROzelbaU4a$6W8p(O0- z%P?vFW|2PPbKmmVF4!R4;(<<-)D6SXeL8sJfod)XFKkVx zH3qij2hp4Y=K$TRa(3%)Oh;XA&AC5c8ScD3gVGzh4GVJLPYS* z`Ih~tbq@zn{uLrS?8tb~E=5~7@F@Bd<#j>G9! z%2123OFA%Q@tp!J5ul`P3RqHnuc9(I3Q0cMrKV>4nim**S(ezHW9GWS# z5eEmFrxw`#eb$nKf>42PDS{PGesV|0vFIrWl{>=eHM?;7djKD|w}KqN_0_Y<+i<_P zn`#a|@~Mj?WL{@rP9cmc$LuG&)Lw?hWpG{HJqtpI-1+X{5c@rDwWcw3D zv+I{|EOrnrn2F~?h#sod7c98>rvZn4B(D+7D?cXsoP^$p>t$g5KjkH;0m0YxC@Nn} z9Y({nwag_VpuEJMc_i}nvHLIdyG3@6K}Cn58Up+WPU=_{^$>1d0+d=sboeu9t*TLI zaP&rd^*r*bjJ9IGEaJ5oP+q#Htwclm{V=!FM5U6dP*14g8P;>tA!s$68O`p0U6|%I z7D4|VbqmXJ)V62xiS4_|EUXowV)Z0ZYk&oQTeYV&Ay8gQ`;?b5jrz^1=z{Y3IGjy1 ztMo9GL*I0<_b2Au>z(2)X$pI4*%JidpYKN9YItwmHShV7?O91e(lmhAQh})%^NS2= zl1nuy`t1KwGe!TUW)%F{ij)6K&D?*cX7D&mR@a^NprT_|hK%2Qy>y!VtdFaF=REE) zk-PRlyJ4XXzI8RQs1=X=(XWy!#K2*v(W@X=GcI=|x-b8RtH)6eTw27n9S_|qNlcGlQp*BdL^7dE>t^za$) zapbn;Uuj~RIiD_;4wq1|A5PPx)8v1xhHRTva*|-J&8m9#`j3$?IbY$C=c>Y^f0!~B zMk7e7#o2ii$BrhRx@uHNL==R^p@Hq;OS6g{mSWIhR<^|JhG10?CaO0HY&P)tIIHFc z>MT)r$>N>WEWq2@o@QNu*}umU196e4+gRfF&g2kUIS#$tr<=!E$3{F9^$Z<9GgI#G z3^bN14D|6!h?N32ylK^U?a0;i;B9r$vaU_2OFga8IQYj?4P`QB!ZdXz)}=wN5fN%wl{_TVzQ|peDyLI~OKVA3fE=_tLQ%u^Sx@`I8 zuBaFS-KA4tX@;v3Pj(wvnrW}ME;0;6DL5(E*rXq&QiBH+W+*3omG96{vdfCXk|1gS zp0^&NrEJ~?=a&9GN=O@`ZF9syA6T04)o;f6QmQ>2LEgS(;oLNt_`$PV*>r!&5l;>* z%~X^fMd&mW*u}P(2A&~3 z`;b(c>5+Fuw!1|&f~W`e!B(O*{|11ca3 zuaFLOwI5hqMqe+FnRAxU1iuP;-T{y2pe5zjsj!~p10k36wbuN4_YrHe(pnZGBgQ?K z9q|yu)C`w0>l4Hqi@;-1ywN)AM`KYMw}evN{*`9qo1llbl0O+Kkz5{ zkrf_e-Ic8&LqKwgR(XN0@9!VfOD7fl6`);nZu{>=tneHV3(U&W?>!s4n{Mwu8%vl!}Xp(+|yr1Bim)rFma zv-)B_Qx*JJAHR`F@o`hRt~6!8c5TPpqJ!%g2tt3qM2F{*Ljax>R?L##L5dv+XV~ z{@yN&oZfkB6_0^9b)q)BEDBE&wC(T~+^5_*qV=ec>+^2Ngr$^!_&FM z0*^4b&K)Mr?4PL_lZ*-G&(w_JNb0Kl#4ZE$SzO6epOwuk?PGyx4qH0MOWBJ^aTAt7 zkBkDA4|TWlbfpy!Pw3oXt=AC#Gq_2-TI@nm%h01F9pV7+nouG}2Jl(l?)3Bf8y5Aw zud0?Bd5+C)?q!Y?u&n^lVWgg^e-rHVt@#BI@N#o`D+LG)9FPC;W1nt|tJ3l0NJNbU zy-H}*-(MeMcrNr@H6#vZ)Gt7)0X#tBh}^LPZ=h-^)f)?W8K7-eaUy(P+8RsL#pB05QPnW6|GK6O{P{y|VbsGXU7H`ZgLk8;8+3rx*)+gd+< zre