From a213efa81fc212a903ef58b5dca1e1b0f11c9c21 Mon Sep 17 00:00:00 2001 From: Patrik Lundin Date: Thu, 8 Feb 2024 11:44:28 +0100 Subject: [PATCH 001/161] Fix ubuntu 18.04 issue Error before fix: ``` AttributeError: type object 'datetime.datetime' has no attribute 'fromisoformat' ``` --- files/baas2/sunet-baas2-status | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/files/baas2/sunet-baas2-status b/files/baas2/sunet-baas2-status index 21255a6be..dc60296d3 100755 --- a/files/baas2/sunet-baas2-status +++ b/files/baas2/sunet-baas2-status @@ -78,7 +78,11 @@ def last_scheduled_start_time() -> Union[datetime.datetime, None]: # 2023-04-03 22:00:16 --- SCHEDULEREC OBJECT BEGIN FILE_0000 2023-04-04 00:00:00 if " SCHEDULEREC OBJECT BEGIN " in line: timestamp = line.split(" -")[0] - returned_timestamp = datetime.datetime.fromisoformat(timestamp) + # Can be used again when there are no ubuntu 18.04 machines using this tool + # returned_timestamp = datetime.datetime.fromisoformat(timestamp) + returned_timestamp = datetime.datetime.strptime( + timestamp, "%Y-%m-%d %H:%M:%S" + ) return returned_timestamp @@ -111,7 +115,11 @@ def last_error_time() -> Union[datetime.datetime, None]: ): timestamp_parts = line.split(maxsplit=2) timestamp = timestamp_parts[0] + " " + timestamp_parts[1] - returned_timestamp = datetime.datetime.fromisoformat(timestamp) + # Can be used again when there are no ubuntu 18.04 machines using this tool + # returned_timestamp = datetime.datetime.fromisoformat(timestamp) + returned_timestamp = datetime.datetime.strptime( + timestamp, "%Y-%m-%d %H:%M:%S" + ) return returned_timestamp From 24bbc5441c685ac483e8854cfb69efed03c0c4d9 Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Fri, 9 Feb 2024 11:16:04 +0100 Subject: [PATCH 002/161] Allow traefik ofn 30443 --- manifests/microk8s/node.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/microk8s/node.pp b/manifests/microk8s/node.pp index 2432f0e91..1ed8c363e 100644 --- a/manifests/microk8s/node.pp +++ b/manifests/microk8s/node.pp @@ -39,7 +39,7 @@ } -> sunet::misc::ufw_allow { 'microk8s_ports': from => 'any', - port => [8080, 8443, 16443, 10250, 10255, 25000, 12379, 10257, 10259, 19001], + port => [8080, 8443, 16443, 10250, 10255, 25000, 12379, 10257, 10259, 19001, 30443], } # This is how ufw::allow does it, but that lacks support for "on" -> exec { 'allow-outgoing-on-calico': From 8a3d7d42fa2341cd4dfba99dac8587c56dd06e87 Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Fri, 9 Feb 2024 11:53:15 +0100 Subject: [PATCH 003/161] MICROK8s: Add support for setting ingress ports --- manifests/microk8s/node.pp | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/manifests/microk8s/node.pp b/manifests/microk8s/node.pp index 1ed8c363e..dff014151 100644 --- a/manifests/microk8s/node.pp +++ b/manifests/microk8s/node.pp @@ -1,8 +1,10 @@ # microk8s cluster node class sunet::microk8s::node( - String $channel = '1.27/stable', - Boolean $mayastor = false, - Integer $failure_domain = 42, + String $channel = '1.27/stable', + Boolean $mayastor = false, + Integer $failure_domain = 42, + Integer $web_nodeport = 30080, + Integer $websecure_nodeport = 30443, ) { # Loop through peers and do things that require their ip:s include stdlib @@ -77,13 +79,24 @@ command => '/snap/bin/microk8s enable community', provider => 'shell', } + $line1 ="/snap/bin/microk8s enable traefik --set ports.websecure.nodePort=${websecure_nodeport}" + $line2 = "--set ports.web.nodePort=${web_nodeport} --set deployment.kind=DaemonSet" + $traefik_command = "${line1} ${line2}" unless any2bool($facts['microk8s_traefik']) { exec { 'enable_plugin_traefik': - command => '/snap/bin/microk8s enable traefik', + command => $traefik_command, provider => 'shell', } } } + exec { 'alias_kubectl': + command => '/usr/bin/snap alias microk8s.kubectl kubectl', + provider => 'shell', + } + exec { 'alias_helm': + command => '/usr/bin/snap alias microk8s.helm helm', + provider => 'shell', + } if $mayastor { package { "linux-modules-extra-${facts['kernelrelease']}": ensure => installed, From 8e332d96e6457b5f2b2f39afa5e9d9bd35201616 Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Wed, 20 Mar 2024 11:17:18 +0100 Subject: [PATCH 004/161] Support multiple certs --- manifests/frontend/load_balancer/website2.pp | 27 ++++++++++++++++---- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/manifests/frontend/load_balancer/website2.pp b/manifests/frontend/load_balancer/website2.pp index 2caea4774..5b40459a0 100644 --- a/manifests/frontend/load_balancer/website2.pp +++ b/manifests/frontend/load_balancer/website2.pp @@ -66,12 +66,29 @@ # copy $tls_certificate_bundle to the instance 'certs' directory to detect when it is updated # so the service can be restarted - file { - "${confdir}/${instance}/certs/tls_certificate_bundle.pem": - source => $tls_certificate_bundle, - notify => Sunet::Docker_compose["frontend-${instance}"], - } + $temp_certs = shell_split($tls_certificate_bundle) + $numcerts = length($temp_certs) + if $numcerts > 1 { + $certnum = 0 + $temp_certs.each do |$cert| { + if $cert != 'cer' { + file { "${confdir}/${instance}/certs/tls_certificate_bundle.pem.${certnum}": + source => $tls_certificate_bundle, + notify => Sunet::Docker_compose["frontend-${instance}"], + } + $certnum += 1 + } + file { "${confdir}/${instance}/certs/tls_certificate_bundle.pem": + file => absent, + } + } + } else { + file { "${confdir}/${instance}/certs/tls_certificate_bundle.pem": + source => $tls_certificate_bundle, + notify => Sunet::Docker_compose["frontend-${instance}"], + } + } # 'export' config to one YAML file per instance file { "${confdir}/${instance}/config.yml": From f4cabd0a47bb425b70cd9e601343d2021262010d Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Wed, 20 Mar 2024 11:18:27 +0100 Subject: [PATCH 005/161] Lint --- manifests/frontend/load_balancer/website2.pp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/manifests/frontend/load_balancer/website2.pp b/manifests/frontend/load_balancer/website2.pp index 5b40459a0..7d15c2127 100644 --- a/manifests/frontend/load_balancer/website2.pp +++ b/manifests/frontend/load_balancer/website2.pp @@ -100,9 +100,10 @@ ; } + # Parameters used in frontend/docker-compose_template.erb $dns = pick_default($config['dns'], []) - $exposed_ports = pick_default($config['exposed_ports'], ["443"]) + $exposed_ports = pick_default($config['exposed_ports'], ['443']) $frontendtools_imagetag = pick($config['frontendtools_imagetag'], 'stable') $frontendtools_volumes = pick($config['frontendtools_volumes'], false) $haproxy_image = pick($config['haproxy_image'], 'docker.sunet.se/library/haproxy') From 0a6131e0dc244f7c00c7c03299447cb85545fd4d Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Wed, 20 Mar 2024 11:50:13 +0100 Subject: [PATCH 006/161] Syntax error --- manifests/frontend/load_balancer/website2.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/frontend/load_balancer/website2.pp b/manifests/frontend/load_balancer/website2.pp index 7d15c2127..cd663c47c 100644 --- a/manifests/frontend/load_balancer/website2.pp +++ b/manifests/frontend/load_balancer/website2.pp @@ -70,7 +70,7 @@ $numcerts = length($temp_certs) if $numcerts > 1 { $certnum = 0 - $temp_certs.each do |$cert| { + $temp_certs.each |$cert| { if $cert != 'cer' { file { "${confdir}/${instance}/certs/tls_certificate_bundle.pem.${certnum}": source => $tls_certificate_bundle, From 25028b745121aed3868bc00f8e8dc8945d0e6ba4 Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Wed, 20 Mar 2024 11:57:43 +0100 Subject: [PATCH 007/161] += has been removed --- manifests/frontend/load_balancer/website2.pp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/manifests/frontend/load_balancer/website2.pp b/manifests/frontend/load_balancer/website2.pp index cd663c47c..88c55c0fb 100644 --- a/manifests/frontend/load_balancer/website2.pp +++ b/manifests/frontend/load_balancer/website2.pp @@ -69,14 +69,12 @@ $temp_certs = shell_split($tls_certificate_bundle) $numcerts = length($temp_certs) if $numcerts > 1 { - $certnum = 0 - $temp_certs.each |$cert| { + $temp_certs.each |Integer $index, String $cert| { if $cert != 'cer' { - file { "${confdir}/${instance}/certs/tls_certificate_bundle.pem.${certnum}": + file { "${confdir}/${instance}/certs/tls_certificate_bundle.pem.${index}": source => $tls_certificate_bundle, notify => Sunet::Docker_compose["frontend-${instance}"], } - $certnum += 1 } file { "${confdir}/${instance}/certs/tls_certificate_bundle.pem": file => absent, From c1bbd071dd652e8284cac4a3a0fea3d2486bff10 Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Wed, 20 Mar 2024 11:17:18 +0100 Subject: [PATCH 008/161] Support multiple certs --- manifests/frontend/load_balancer/website2.pp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/manifests/frontend/load_balancer/website2.pp b/manifests/frontend/load_balancer/website2.pp index 88c55c0fb..7d15c2127 100644 --- a/manifests/frontend/load_balancer/website2.pp +++ b/manifests/frontend/load_balancer/website2.pp @@ -69,12 +69,14 @@ $temp_certs = shell_split($tls_certificate_bundle) $numcerts = length($temp_certs) if $numcerts > 1 { - $temp_certs.each |Integer $index, String $cert| { + $certnum = 0 + $temp_certs.each do |$cert| { if $cert != 'cer' { - file { "${confdir}/${instance}/certs/tls_certificate_bundle.pem.${index}": + file { "${confdir}/${instance}/certs/tls_certificate_bundle.pem.${certnum}": source => $tls_certificate_bundle, notify => Sunet::Docker_compose["frontend-${instance}"], } + $certnum += 1 } file { "${confdir}/${instance}/certs/tls_certificate_bundle.pem": file => absent, From 22873d3e6f6c5a62718002bba472ce47824c30a0 Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Wed, 20 Mar 2024 11:50:13 +0100 Subject: [PATCH 009/161] Syntax error --- manifests/frontend/load_balancer/website2.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/frontend/load_balancer/website2.pp b/manifests/frontend/load_balancer/website2.pp index 7d15c2127..cd663c47c 100644 --- a/manifests/frontend/load_balancer/website2.pp +++ b/manifests/frontend/load_balancer/website2.pp @@ -70,7 +70,7 @@ $numcerts = length($temp_certs) if $numcerts > 1 { $certnum = 0 - $temp_certs.each do |$cert| { + $temp_certs.each |$cert| { if $cert != 'cer' { file { "${confdir}/${instance}/certs/tls_certificate_bundle.pem.${certnum}": source => $tls_certificate_bundle, From 0a545ce2ab412525cffa4bda66b11fbef96f10b9 Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Wed, 20 Mar 2024 11:57:43 +0100 Subject: [PATCH 010/161] += has been removed --- manifests/frontend/load_balancer/website2.pp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/manifests/frontend/load_balancer/website2.pp b/manifests/frontend/load_balancer/website2.pp index cd663c47c..88c55c0fb 100644 --- a/manifests/frontend/load_balancer/website2.pp +++ b/manifests/frontend/load_balancer/website2.pp @@ -69,14 +69,12 @@ $temp_certs = shell_split($tls_certificate_bundle) $numcerts = length($temp_certs) if $numcerts > 1 { - $certnum = 0 - $temp_certs.each |$cert| { + $temp_certs.each |Integer $index, String $cert| { if $cert != 'cer' { - file { "${confdir}/${instance}/certs/tls_certificate_bundle.pem.${certnum}": + file { "${confdir}/${instance}/certs/tls_certificate_bundle.pem.${index}": source => $tls_certificate_bundle, notify => Sunet::Docker_compose["frontend-${instance}"], } - $certnum += 1 } file { "${confdir}/${instance}/certs/tls_certificate_bundle.pem": file => absent, From fc3d07b7e7b98fe00d1c0b7fdfd9691674bcd498 Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Wed, 20 Mar 2024 12:50:57 +0100 Subject: [PATCH 011/161] Slicker solution --- manifests/frontend/load_balancer/website2.pp | 21 ++++++++----------- .../frontend/docker-compose_template.erb | 8 ++++++- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/manifests/frontend/load_balancer/website2.pp b/manifests/frontend/load_balancer/website2.pp index 88c55c0fb..7d270e770 100644 --- a/manifests/frontend/load_balancer/website2.pp +++ b/manifests/frontend/load_balancer/website2.pp @@ -66,20 +66,17 @@ # copy $tls_certificate_bundle to the instance 'certs' directory to detect when it is updated # so the service can be restarted - $temp_certs = shell_split($tls_certificate_bundle) - $numcerts = length($temp_certs) - if $numcerts > 1 { - $temp_certs.each |Integer $index, String $cert| { - if $cert != 'cer' { - file { "${confdir}/${instance}/certs/tls_certificate_bundle.pem.${index}": - source => $tls_certificate_bundle, - notify => Sunet::Docker_compose["frontend-${instance}"], - } - } - file { "${confdir}/${instance}/certs/tls_certificate_bundle.pem": - file => absent, + $multi_certs = shell_split($tls_certificate_bundle).filter |String $cert| { $cert != 'crt' } + if length($multi_certs) > 1 { + $multi_certs.each |Integer $index, String $cert| { + file { "${confdir}/${instance}/certs/tls_certificate_bundle.${index}.pem": + source => $cert, + notify => Sunet::Docker_compose["frontend-${instance}"], } } + file { "${confdir}/${instance}/certs/tls_certificate_bundle.pem": + ensure => absent, + } } else { file { "${confdir}/${instance}/certs/tls_certificate_bundle.pem": source => $tls_certificate_bundle, diff --git a/templates/frontend/docker-compose_template.erb b/templates/frontend/docker-compose_template.erb index 2f23965ff..45480ec68 100644 --- a/templates/frontend/docker-compose_template.erb +++ b/templates/frontend/docker-compose_template.erb @@ -16,7 +16,13 @@ services: - '/opt/frontend/scripts/haproxy-start.sh:/haproxy-start.sh:ro' - 'haproxy_data:/etc/haproxy' - 'haproxy_control:/var/run/haproxy-control' -<% if @tls_certificate_bundle -%> +<% if @multi_cert.is_a? Array and @multi_cert.size > 1 -%> + <% index = 0 %> + <% @multi_cert.each do |cert| -%> + - /opt/frontend/config/<%= @instance %>/certs/tls_certificate_bundle.<%= index %>.pem:<%= cert %>:ro + <%- index += 1 -%> + <% end -%> +<% else if @tls_certificate_bundle -%> - /opt/frontend/config/<%= @instance %>/certs/tls_certificate_bundle.pem:<%= @tls_certificate_bundle %>:ro <% else -%> # tls_certificate_bundle not set in Puppet From 29d2aa08d2cf903c1a0ae899e9a928545a5cb85f Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Wed, 20 Mar 2024 13:00:33 +0100 Subject: [PATCH 012/161] Syntax error --- templates/frontend/docker-compose_template.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/frontend/docker-compose_template.erb b/templates/frontend/docker-compose_template.erb index 45480ec68..79c365687 100644 --- a/templates/frontend/docker-compose_template.erb +++ b/templates/frontend/docker-compose_template.erb @@ -22,7 +22,7 @@ services: - /opt/frontend/config/<%= @instance %>/certs/tls_certificate_bundle.<%= index %>.pem:<%= cert %>:ro <%- index += 1 -%> <% end -%> -<% else if @tls_certificate_bundle -%> +<% elsif @tls_certificate_bundle -%> - /opt/frontend/config/<%= @instance %>/certs/tls_certificate_bundle.pem:<%= @tls_certificate_bundle %>:ro <% else -%> # tls_certificate_bundle not set in Puppet From 66c67cde3a414b9f80b82b79714c062691ae6ea1 Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Wed, 20 Mar 2024 14:33:41 +0100 Subject: [PATCH 013/161] Fix variable name --- templates/frontend/docker-compose_template.erb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/frontend/docker-compose_template.erb b/templates/frontend/docker-compose_template.erb index 79c365687..fb9b902f0 100644 --- a/templates/frontend/docker-compose_template.erb +++ b/templates/frontend/docker-compose_template.erb @@ -16,9 +16,9 @@ services: - '/opt/frontend/scripts/haproxy-start.sh:/haproxy-start.sh:ro' - 'haproxy_data:/etc/haproxy' - 'haproxy_control:/var/run/haproxy-control' -<% if @multi_cert.is_a? Array and @multi_cert.size > 1 -%> +<% if @multi_certs.is_a? Array and @multi_certs.length > 1 -%> <% index = 0 %> - <% @multi_cert.each do |cert| -%> + <% @multi_certs.each do |cert| -%> - /opt/frontend/config/<%= @instance %>/certs/tls_certificate_bundle.<%= index %>.pem:<%= cert %>:ro <%- index += 1 -%> <% end -%> From ad5f7b98bec67d7a66c9f0268ea1ddadbfd51207 Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Wed, 20 Mar 2024 14:40:38 +0100 Subject: [PATCH 014/161] Try to fix whitespace --- templates/frontend/docker-compose_template.erb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/frontend/docker-compose_template.erb b/templates/frontend/docker-compose_template.erb index fb9b902f0..caae0fb56 100644 --- a/templates/frontend/docker-compose_template.erb +++ b/templates/frontend/docker-compose_template.erb @@ -16,8 +16,8 @@ services: - '/opt/frontend/scripts/haproxy-start.sh:/haproxy-start.sh:ro' - 'haproxy_data:/etc/haproxy' - 'haproxy_control:/var/run/haproxy-control' -<% if @multi_certs.is_a? Array and @multi_certs.length > 1 -%> - <% index = 0 %> +<% if @multi_certs.is_a? Array and @multi_certs.size > 1 -%> + <%- index = 0 -%> <% @multi_certs.each do |cert| -%> - /opt/frontend/config/<%= @instance %>/certs/tls_certificate_bundle.<%= index %>.pem:<%= cert %>:ro <%- index += 1 -%> From ee439bc3947e4aa015cc63ddfbd0978f052ad2f9 Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Thu, 16 May 2024 11:12:47 +0200 Subject: [PATCH 015/161] Move up postfix stop --- manifests/mail/postfix.pp | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/manifests/mail/postfix.pp b/manifests/mail/postfix.pp index 0ca32ed50..b4d59b5a8 100644 --- a/manifests/mail/postfix.pp +++ b/manifests/mail/postfix.pp @@ -25,10 +25,12 @@ $smtpd_tls_key_file="/certs/${smtp_domain}/privkey.pem" package { 'exim4-base': - ensure => absent, + ensure => absent, provider => 'apt', } - + -> service { 'postfix': + ensure => 'stopped', + } # Composefile sunet::docker_compose { 'postfix': @@ -62,8 +64,4 @@ } } - service { 'postfix': - ensure => 'stopped', - } - } From 8d414d3bba9f7a6715ee93600a8ba9839db62919 Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Fri, 17 May 2024 14:59:37 +0200 Subject: [PATCH 016/161] Force settings that are neccessary --- manifests/mail/dovecot.pp | 6 +++--- manifests/mail/postfix.pp | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifests/mail/dovecot.pp b/manifests/mail/dovecot.pp index cf7f004a9..750d05d91 100644 --- a/manifests/mail/dovecot.pp +++ b/manifests/mail/dovecot.pp @@ -2,9 +2,9 @@ class sunet::mail::dovecot( String $replication_partner, Array[String] $allow_nets, - String $domain = 'sunet.dev', - String $imap_domain = 'sunet-imap.drive.test.sunet.se', - String $environment = 'test', + String $domain, + String $imap_domain, + String $environment, String $account_domain = 'sunet.se', String $interface = 'ens3', String $dovecot_image = 'docker.sunet.se/mail/dovecot', diff --git a/manifests/mail/postfix.pp b/manifests/mail/postfix.pp index b4d59b5a8..3b7a6d318 100644 --- a/manifests/mail/postfix.pp +++ b/manifests/mail/postfix.pp @@ -1,14 +1,14 @@ # Postfix for SUNET class sunet::mail::postfix( - String $domain = 'sunet.dev', - String $smtp_domain = 'sunet-smtp.drive.test.sunet.se', - String $imap_domain = 'sunet-imap.drive.test.sunet.se', - String $environment = 'test', + String $domain, + String $smtp_domain, + String $imap_domain, + String $environment, + Array[String] $imap_servers, String $interface = 'ens3', String $postfix_image = 'docker.sunet.se/mail/postfix', String $postfix_tag = 'SUNET-1', Array[String] $relay_servers = ['mf-tst-ng-1.sunet.se:587', 'mf-tst-ng-2.sunet.se:587'], - Array[String] $imap_servers = ['89.45.237.128', '89.46.21.203'], ) { From 64b72db63175f3b24e47b1f11e9c5a6e7cc516eb Mon Sep 17 00:00:00 2001 From: Maria Haider Date: Thu, 23 May 2024 22:05:57 +0200 Subject: [PATCH 017/161] updated scriptherder check for sunet infra cert --- manifests/ici_ca.pp | 23 +++++---- templates/ici_ca/check_infra_cert_expire.erb | 54 ++++---------------- 2 files changed, 23 insertions(+), 54 deletions(-) diff --git a/manifests/ici_ca.pp b/manifests/ici_ca.pp index c846bfb02..5c5287270 100644 --- a/manifests/ici_ca.pp +++ b/manifests/ici_ca.pp @@ -42,10 +42,13 @@ } # fetch certificate from ici-ca automatically and run a scriptherder job to see if it is valid -define sunet::ici_ca::rp() -{ +define sunet::ici_ca::rp( + Boolean $monitor_infra_cert = true, +) { + $host = $::fqdn $ca = $name + file { '/usr/bin/dl_ici_cert': content => template('sunet/ici_ca/dl_ici_cert.erb'), mode => '0755' @@ -63,11 +66,13 @@ content => template('sunet/ici_ca/check_infra_cert_expire.erb'), mode => '0755' } - sunet::scriptherder::cronjob { 'check_infra_cert': - cmd => "/usr/bin/check_infra_cert_expire /etc/ssl/certs/${host}_infra.crt", - minute => '30', - hour => '8', - ok_criteria => ['exit_status=0', 'max_age=26h'], - warn_criteria => ['exit_status=2', 'max_age=2d'], + + if ($monitor_infra_cert) { + sunet::scriptherder::cronjob { 'check_infra_cert': + cmd => "/usr/bin/check_infra_cert_expire /etc/ssl/certs/${host}_infra.crt", + minute => '30', + hour => '8', + ok_criteria => ['exit_status=0', 'max_age=25h'], } -} \ No newline at end of file + } +} diff --git a/templates/ici_ca/check_infra_cert_expire.erb b/templates/ici_ca/check_infra_cert_expire.erb index 44abadba0..93db86c15 100644 --- a/templates/ici_ca/check_infra_cert_expire.erb +++ b/templates/ici_ca/check_infra_cert_expire.erb @@ -1,57 +1,19 @@ #!/bin/sh -# Checks if a given cert on disk will expire soon - -# Copyright 2009 Peter Palfrader -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +#It is a changed version of the original https://github.com/sergioshev/nagios-plugins/blob/master/check_cert_expire set -u set -e -# warn if expires within 2 weeks, critical if within a week or already is expired -warn=1209600 +#ok if cert has minimum 2 weeks validity or more, critical if within a week or already is expired +ok=1209600 crit=604800 usage() { - echo "Usage: $0 [-w seconds] [-c seconds] " >&2 + echo "Usage: $0 " >&2 exit 3 } - -OPTS=$(getopt -o w:c: -n "$0" -- "$@") || usage - -eval set -- "$OPTS" - -while :; do - case "$1" in - -w) warn=$2; shift 2 ;; - -c) crit=$2; shift 2 ;; - --) shift; break; ;; - *) usage ;; - esac -done -if test "$crit" -gt "$warn"; then - warn=$crit -fi - if [ "$#" != 1 ]; then usage fi @@ -65,13 +27,15 @@ fi expires=`openssl x509 -enddate -noout < "$cert"` -if openssl x509 -checkend "$warn" -noout < "$cert" ; then +if openssl x509 -checkend "$ok" -noout < "$cert" ; then echo "OK: $expires" exit 0 fi + + if openssl x509 -checkend "$crit" -noout < "$cert" ; then - echo "WARN: $expires" + echo "CRITICAL: $expires" exit 2 fi echo "CRITICAL: $expires" -exit 2 \ No newline at end of file +exit 2 From 1e3eba8fc885642bbfe1d08172f47a02c5059e28 Mon Sep 17 00:00:00 2001 From: Joao Paulo Oliveira de Araujo Rangel Pamplona Date: Wed, 5 Jun 2024 13:38:21 +0200 Subject: [PATCH 018/161] updated mdq_publisher class to support custom SSL certs --- manifests/metadata/mdq_publisher.pp | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/manifests/metadata/mdq_publisher.pp b/manifests/metadata/mdq_publisher.pp index e8bffe6cb..fd1926f34 100644 --- a/manifests/metadata/mdq_publisher.pp +++ b/manifests/metadata/mdq_publisher.pp @@ -1,7 +1,9 @@ # Wrapper to setup a MDQ-publiser class sunet::metadata::mdq_publisher( Boolean $infra_cert_from_this_class = true, - Boolean $nftables_init = true, + Boolean $nftables_init = true, + Optional[String] $publisher_cert="/etc/ssl/certs/${facts['networking']['fqdn']}_infra.crt", + Optional[String] $publisher_key="/etc/ssl/private/${facts['networking']['fqdn']}_infra.key", Optional[Array] $env=[], Optional[Integer] $valid_until=12, Optional[String] $validate_cert='/var/www/html/md/md-signer2.crt', @@ -70,20 +72,21 @@ if $infra_cert_from_this_class { sunet::ici_ca::rp { 'infra': } } - $env_infra_ca = [ - "PUBLISHER_CERT=/etc/ssl/certs/${facts['networking']['fqdn']}_infra.crt", - "PUBLISHER_KEY=/etc/ssl/private/${facts['networking']['fqdn']}_infra.key", - ] + $env_certs = [ + "PUBLISHER_CERT=${publisher_cert}", + "PUBLISHER_KEY=${publisher_key}", + ] + sunet::docker_run { 'swamid-mdq-publisher': image => 'docker.sunet.se/swamid/mdq-publisher', imagetag => $imagetag, hostname => $facts['networking']['fqdn'], volumes => [ - '/etc/ssl/mdq:/etc/certs', '/etc/ssl:/etc/ssl', - '/var/www/html:/var/www/html' + '/var/www/html:/var/www/html', + '/etc/dehydrated:/etc/dehydrated', ], - env => $env + $env_infra_ca, + env => $env + $env_certs, uid_gid_consistency => false, ports => ['443:443'], } From bfb46c01880490e85d21fa2bff306addb2f05b7d Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Thu, 13 Jun 2024 15:16:59 +0200 Subject: [PATCH 019/161] Fix ports --- manifests/mail/postfix.pp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/manifests/mail/postfix.pp b/manifests/mail/postfix.pp index b8fd95a21..d1d911852 100644 --- a/manifests/mail/postfix.pp +++ b/manifests/mail/postfix.pp @@ -40,16 +40,16 @@ compose_filename => 'docker-compose.yml', description => 'Postfix', } - $ports = [25] - $ports.each|$port| { + $restricted_ports = [25] + $restricted_ports.each|$port| { sunet::nftables::docker_expose { "mail_port_${port}": allow_clients => $relay_hosts, port => $port, iif => $interface, } } - $ports = [587] - $ports.each|$port| { + $open_ports = [587] + $open_ports.each|$port| { sunet::nftables::docker_expose { "mail_port_${port}": allow_clients => 'any', port => $port, From 4ebcc1c1d9fbe325dbb3841c06eacf00981b0755 Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Mon, 17 Jun 2024 08:44:33 +0200 Subject: [PATCH 020/161] Take my destination from config --- manifests/mail/postfix.pp | 1 + templates/mail/postfix/main.erb.cf | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/manifests/mail/postfix.pp b/manifests/mail/postfix.pp index d1d911852..9c1e6badc 100644 --- a/manifests/mail/postfix.pp +++ b/manifests/mail/postfix.pp @@ -6,6 +6,7 @@ Array[String] $relaying_servers, String $smtp_domain, String $interface = 'ens3', + Array[String] $mydestination = ['$myhostname', 'localhost.localdomain', 'localhost'], String $postfix_image = 'docker.sunet.se/mail/postfix', String $postfix_tag = 'SUNET-1', Array[String] $relay_servers = ['mf-tst-ng-1.sunet.se:587', 'mf-tst-ng-2.sunet.se:587'], diff --git a/templates/mail/postfix/main.erb.cf b/templates/mail/postfix/main.erb.cf index 29432200d..63b666ec9 100644 --- a/templates/mail/postfix/main.erb.cf +++ b/templates/mail/postfix/main.erb.cf @@ -40,7 +40,7 @@ alias_maps = hash:/etc/aliases alias_database = hash:/etc/aliases virtual_mailbox_domains = mysql:/config/mysql-virtual-mailbox-domains.cf virtual_mailbox_maps = mysql:/config/mysql-virtual-mailbox-maps.cf -mydestination = $myhostname, localhost.localdomain, localhost +mydestination = <%= @mydestination.join(",") %> mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128 <% @relaying_servers.each do |relay| -%> <%= relay %><% end %> mailbox_size_limit = 0 recipient_delimiter = + From 47686463a0d77cf8f49fdb786c033de84cf99d38 Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Mon, 17 Jun 2024 08:55:45 +0200 Subject: [PATCH 021/161] Decouple alias domains from virtual mailbox maps --- templates/mail/postfix/mysql-virtual-mailbox-maps.erb.cf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/mail/postfix/mysql-virtual-mailbox-maps.erb.cf b/templates/mail/postfix/mysql-virtual-mailbox-maps.erb.cf index 58f5bff79..94fe726d3 100644 --- a/templates/mail/postfix/mysql-virtual-mailbox-maps.erb.cf +++ b/templates/mail/postfix/mysql-virtual-mailbox-maps.erb.cf @@ -2,4 +2,4 @@ user = <%= @nextcloud_db_user %> password = <%= @nextcloud_mysql_password %> hosts = <%= @db_hosts %> dbname = <%= @nextcloud_db%> -query = SELECT UNIQUE(1) FROM oc_accounts_data WHERE value = REPLACE('%s', <%= @alias_domains %>) and name = 'email' +query = SELECT UNIQUE(1) FROM oc_accounts_data WHERE value = REPLACE('%s', <%= @smtp_domain %>, 'sunet.se') and name = 'email' From b8663433b34a9c1f17bf5e95d2a885b5fbfddeee Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Mon, 17 Jun 2024 09:01:18 +0200 Subject: [PATCH 022/161] Add bespoke variables --- manifests/mail/postfix.pp | 2 ++ templates/mail/postfix/mysql-virtual-mailbox-maps.erb.cf | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/manifests/mail/postfix.pp b/manifests/mail/postfix.pp index 9c1e6badc..4cd8111d7 100644 --- a/manifests/mail/postfix.pp +++ b/manifests/mail/postfix.pp @@ -1,9 +1,11 @@ # Postfix for SUNET class sunet::mail::postfix( + String $account_domain, String $alias_domains, String $environment, String $imap_domain, Array[String] $relaying_servers, + String $short_domain, String $smtp_domain, String $interface = 'ens3', Array[String] $mydestination = ['$myhostname', 'localhost.localdomain', 'localhost'], diff --git a/templates/mail/postfix/mysql-virtual-mailbox-maps.erb.cf b/templates/mail/postfix/mysql-virtual-mailbox-maps.erb.cf index 94fe726d3..b6d8f42e8 100644 --- a/templates/mail/postfix/mysql-virtual-mailbox-maps.erb.cf +++ b/templates/mail/postfix/mysql-virtual-mailbox-maps.erb.cf @@ -2,4 +2,4 @@ user = <%= @nextcloud_db_user %> password = <%= @nextcloud_mysql_password %> hosts = <%= @db_hosts %> dbname = <%= @nextcloud_db%> -query = SELECT UNIQUE(1) FROM oc_accounts_data WHERE value = REPLACE('%s', <%= @smtp_domain %>, 'sunet.se') and name = 'email' +query = SELECT UNIQUE(1) FROM oc_accounts_data WHERE value = REPLACE('%s', '<%= @account_domain %>, '<%= @short_domain %>') and name = 'email' From 10dfa5a6ef8dac6409d766335a0d83a35d4087f4 Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Mon, 17 Jun 2024 08:55:45 +0200 Subject: [PATCH 023/161] Decouple alias domains from virtual mailbox maps --- templates/mail/postfix/mysql-virtual-mailbox-maps.erb.cf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/mail/postfix/mysql-virtual-mailbox-maps.erb.cf b/templates/mail/postfix/mysql-virtual-mailbox-maps.erb.cf index b6d8f42e8..94fe726d3 100644 --- a/templates/mail/postfix/mysql-virtual-mailbox-maps.erb.cf +++ b/templates/mail/postfix/mysql-virtual-mailbox-maps.erb.cf @@ -2,4 +2,4 @@ user = <%= @nextcloud_db_user %> password = <%= @nextcloud_mysql_password %> hosts = <%= @db_hosts %> dbname = <%= @nextcloud_db%> -query = SELECT UNIQUE(1) FROM oc_accounts_data WHERE value = REPLACE('%s', '<%= @account_domain %>, '<%= @short_domain %>') and name = 'email' +query = SELECT UNIQUE(1) FROM oc_accounts_data WHERE value = REPLACE('%s', <%= @smtp_domain %>, 'sunet.se') and name = 'email' From ad1b74f02f7e08953c8066dbd3a5176b47f843e4 Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Mon, 17 Jun 2024 09:01:18 +0200 Subject: [PATCH 024/161] Add bespoke variables --- templates/mail/postfix/mysql-virtual-mailbox-maps.erb.cf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/mail/postfix/mysql-virtual-mailbox-maps.erb.cf b/templates/mail/postfix/mysql-virtual-mailbox-maps.erb.cf index 94fe726d3..b6d8f42e8 100644 --- a/templates/mail/postfix/mysql-virtual-mailbox-maps.erb.cf +++ b/templates/mail/postfix/mysql-virtual-mailbox-maps.erb.cf @@ -2,4 +2,4 @@ user = <%= @nextcloud_db_user %> password = <%= @nextcloud_mysql_password %> hosts = <%= @db_hosts %> dbname = <%= @nextcloud_db%> -query = SELECT UNIQUE(1) FROM oc_accounts_data WHERE value = REPLACE('%s', <%= @smtp_domain %>, 'sunet.se') and name = 'email' +query = SELECT UNIQUE(1) FROM oc_accounts_data WHERE value = REPLACE('%s', '<%= @account_domain %>, '<%= @short_domain %>') and name = 'email' From 5123eb5bb43fa0d4808e7f942fa5c1a33844b4a8 Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Mon, 17 Jun 2024 09:22:38 +0200 Subject: [PATCH 025/161] Missing quote --- templates/mail/postfix/mysql-virtual-mailbox-maps.erb.cf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/mail/postfix/mysql-virtual-mailbox-maps.erb.cf b/templates/mail/postfix/mysql-virtual-mailbox-maps.erb.cf index b6d8f42e8..69ba82738 100644 --- a/templates/mail/postfix/mysql-virtual-mailbox-maps.erb.cf +++ b/templates/mail/postfix/mysql-virtual-mailbox-maps.erb.cf @@ -2,4 +2,4 @@ user = <%= @nextcloud_db_user %> password = <%= @nextcloud_mysql_password %> hosts = <%= @db_hosts %> dbname = <%= @nextcloud_db%> -query = SELECT UNIQUE(1) FROM oc_accounts_data WHERE value = REPLACE('%s', '<%= @account_domain %>, '<%= @short_domain %>') and name = 'email' +query = SELECT UNIQUE(1) FROM oc_accounts_data WHERE value = REPLACE('%s', '<%= @account_domain %>', '<%= @short_domain %>') and name = 'email' From 6dd0b02034af8f43ff043bd03f311c229201edb7 Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Mon, 17 Jun 2024 09:35:23 +0200 Subject: [PATCH 026/161] Try local_recipient_maps = --- templates/mail/postfix/main.erb.cf | 1 + 1 file changed, 1 insertion(+) diff --git a/templates/mail/postfix/main.erb.cf b/templates/mail/postfix/main.erb.cf index 63b666ec9..cf159e1f8 100644 --- a/templates/mail/postfix/main.erb.cf +++ b/templates/mail/postfix/main.erb.cf @@ -36,6 +36,7 @@ smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache smtpd_relay_restrictions = permit_mynetworks permit_sasl_authenticated reject_unauth_destination smtpd_client_restrictions = permit_mynetworks, permit_sasl_authenticated, reject myhostname = <%= @hostname %> +local_recipient_maps = alias_maps = hash:/etc/aliases alias_database = hash:/etc/aliases virtual_mailbox_domains = mysql:/config/mysql-virtual-mailbox-domains.cf From f548ded7c8c86f25a8bc645268645fdaed62c2b8 Mon Sep 17 00:00:00 2001 From: Patrik Lundin Date: Mon, 17 Jun 2024 10:08:27 +0200 Subject: [PATCH 027/161] Bump default knubbis-fleetlock version --- manifests/knubbis/fleetlock_standalone.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/knubbis/fleetlock_standalone.pp b/manifests/knubbis/fleetlock_standalone.pp index 54cdc4755..951efe882 100644 --- a/manifests/knubbis/fleetlock_standalone.pp +++ b/manifests/knubbis/fleetlock_standalone.pp @@ -15,7 +15,7 @@ # @param domain The domain where the fleetlock server will supply its services # @param letsencrypt_prod Should the server request real letsencrypt certificates class sunet::knubbis::fleetlock_standalone( - String $knubbis_fleetlock_version="v0.0.16", + String $knubbis_fleetlock_version="v0.0.17", String $etcd_version="v3.5.8", String $cfssl_helper_version="v0.0.1", String $etcdctl_helper_version="v0.0.1", From 3f11af1c3c2eb8f52c4f541c117d7ab859cc43c1 Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Mon, 17 Jun 2024 10:32:25 +0200 Subject: [PATCH 028/161] Why not both? --- templates/mail/postfix/mysql-virtual-mailbox-maps.erb.cf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/mail/postfix/mysql-virtual-mailbox-maps.erb.cf b/templates/mail/postfix/mysql-virtual-mailbox-maps.erb.cf index 5da5f34bd..6ebc992c4 100644 --- a/templates/mail/postfix/mysql-virtual-mailbox-maps.erb.cf +++ b/templates/mail/postfix/mysql-virtual-mailbox-maps.erb.cf @@ -2,4 +2,4 @@ user = <%= @nextcloud_db_user %> password = <%= @nextcloud_mysql_password %> hosts = <%= @db_hosts %> dbname = <%= @nextcloud_db%> -query = SELECT UNIQUE(1) FROM oc_accounts_data WHERE value = '%s@<%= @short_domain %>' and name = 'email' +query = SELECT UNIQUE(1) FROM oc_accounts_data WHERE (value = '%s@<%= @short_domain %>' OR value = REPLACE('%s', '<%= @account_domain %>', '<%= @short_domain %>')) AND name = 'email' From a806c15b3e1bc742de1fa6761388a8f28de0e766 Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Mon, 17 Jun 2024 12:13:48 +0200 Subject: [PATCH 029/161] Make it possible to set hostname to fqdn --- manifests/frontend/load_balancer.pp | 1 + templates/frontend/docker-compose_template.erb | 3 +++ 2 files changed, 4 insertions(+) diff --git a/manifests/frontend/load_balancer.pp b/manifests/frontend/load_balancer.pp index 5a54c819e..4daf6e309 100644 --- a/manifests/frontend/load_balancer.pp +++ b/manifests/frontend/load_balancer.pp @@ -13,6 +13,7 @@ Integer $haproxy = $base_uidgid + 10, Integer $telegraf = $base_uidgid + 11, Integer $varnish = $base_uidgid + 12, + Boolean $set_fqdn = false, ) { $config = lookup('sunet_frontend', undef, undef, undef) if $config =~ Hash[String, Hash] { diff --git a/templates/frontend/docker-compose_template.erb b/templates/frontend/docker-compose_template.erb index 6f571d46e..80d0da273 100644 --- a/templates/frontend/docker-compose_template.erb +++ b/templates/frontend/docker-compose_template.erb @@ -3,6 +3,9 @@ version: '3' services: haproxy: image: '<%= @haproxy_image %>:<%= @haproxy_imagetag %>' +<%- if @set_fqdn %> + hostname: <%= @facts['fqdn'] %> +<%- end -%> expose: <% @exposed_ports.each do |port| -%> - "<%= port %>" From 29a3bdae3affe0ca5c69634d7bd11152a3b2730b Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Mon, 17 Jun 2024 12:45:54 +0200 Subject: [PATCH 030/161] Syntax for boolean does not work. I don't think this is the problem, but I am willing to try --- templates/frontend/docker-compose_template.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/frontend/docker-compose_template.erb b/templates/frontend/docker-compose_template.erb index 80d0da273..49a4c1768 100644 --- a/templates/frontend/docker-compose_template.erb +++ b/templates/frontend/docker-compose_template.erb @@ -3,7 +3,7 @@ version: '3' services: haproxy: image: '<%= @haproxy_image %>:<%= @haproxy_imagetag %>' -<%- if @set_fqdn %> +<% if @set_fqdn -%> hostname: <%= @facts['fqdn'] %> <%- end -%> expose: From e76aae780497443321da51d06807f41e202b0013 Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Mon, 17 Jun 2024 13:27:14 +0200 Subject: [PATCH 031/161] Remove local_alias_maps --- templates/mail/postfix/main.erb.cf | 1 - 1 file changed, 1 deletion(-) diff --git a/templates/mail/postfix/main.erb.cf b/templates/mail/postfix/main.erb.cf index cf159e1f8..63b666ec9 100644 --- a/templates/mail/postfix/main.erb.cf +++ b/templates/mail/postfix/main.erb.cf @@ -36,7 +36,6 @@ smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache smtpd_relay_restrictions = permit_mynetworks permit_sasl_authenticated reject_unauth_destination smtpd_client_restrictions = permit_mynetworks, permit_sasl_authenticated, reject myhostname = <%= @hostname %> -local_recipient_maps = alias_maps = hash:/etc/aliases alias_database = hash:/etc/aliases virtual_mailbox_domains = mysql:/config/mysql-virtual-mailbox-domains.cf From e625212c91fdacaccf224a1a788906cf6f17c006 Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Mon, 17 Jun 2024 14:27:32 +0200 Subject: [PATCH 032/161] Add relay-recipient-maps --- manifests/mail/postfix.pp | 1 + templates/mail/postfix/main.erb.cf | 1 + templates/mail/postfix/mysql-relay-recipient-maps.erb.cf | 5 +++++ 3 files changed, 7 insertions(+) create mode 100644 templates/mail/postfix/mysql-relay-recipient-maps.erb.cf diff --git a/manifests/mail/postfix.pp b/manifests/mail/postfix.pp index 4cd8111d7..9bbb377a0 100644 --- a/manifests/mail/postfix.pp +++ b/manifests/mail/postfix.pp @@ -65,6 +65,7 @@ $config_files = [ 'main', 'master', + 'mysql-relay-recipient-maps', 'mysql-virtual-mailbox-domains', 'mysql-virtual-mailbox-maps' ] diff --git a/templates/mail/postfix/main.erb.cf b/templates/mail/postfix/main.erb.cf index 63b666ec9..c380f461c 100644 --- a/templates/mail/postfix/main.erb.cf +++ b/templates/mail/postfix/main.erb.cf @@ -40,6 +40,7 @@ alias_maps = hash:/etc/aliases alias_database = hash:/etc/aliases virtual_mailbox_domains = mysql:/config/mysql-virtual-mailbox-domains.cf virtual_mailbox_maps = mysql:/config/mysql-virtual-mailbox-maps.cf +relay_recipient_maps = mysql:/config/mysql-relay-recipient-maps.cf mydestination = <%= @mydestination.join(",") %> mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128 <% @relaying_servers.each do |relay| -%> <%= relay %><% end %> mailbox_size_limit = 0 diff --git a/templates/mail/postfix/mysql-relay-recipient-maps.erb.cf b/templates/mail/postfix/mysql-relay-recipient-maps.erb.cf new file mode 100644 index 000000000..8723db85e --- /dev/null +++ b/templates/mail/postfix/mysql-relay-recipient-maps.erb.cf @@ -0,0 +1,5 @@ +user = <%= @nextcloud_db_user %> +password = <%= @nextcloud_mysql_password %> +hosts = <%= @db_hosts %> +dbname = <%= @nextcloud_db%> +query = SELECT UNIQUE(REPLACE(value, '<%= @short_domain %>', '<%= @account_domain %>')) FROM oc_accounts_data WHERE name = 'email' From 2896309cfbe4fdf80497b14e3fe10801179f8c5c Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Mon, 17 Jun 2024 14:42:45 +0200 Subject: [PATCH 033/161] Add virtual alias maps --- templates/mail/postfix/main.erb.cf | 2 +- ...ay-recipient-maps.erb.cf => mysql-virtual-alias-maps.erb.cf} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename templates/mail/postfix/{mysql-relay-recipient-maps.erb.cf => mysql-virtual-alias-maps.erb.cf} (86%) diff --git a/templates/mail/postfix/main.erb.cf b/templates/mail/postfix/main.erb.cf index c380f461c..d61ad97b4 100644 --- a/templates/mail/postfix/main.erb.cf +++ b/templates/mail/postfix/main.erb.cf @@ -38,9 +38,9 @@ smtpd_client_restrictions = permit_mynetworks, permit_sasl_authenticated, reject myhostname = <%= @hostname %> alias_maps = hash:/etc/aliases alias_database = hash:/etc/aliases +virtual_alias_maps = mysql:/config/mysql-virtual-alias-maps.cf virtual_mailbox_domains = mysql:/config/mysql-virtual-mailbox-domains.cf virtual_mailbox_maps = mysql:/config/mysql-virtual-mailbox-maps.cf -relay_recipient_maps = mysql:/config/mysql-relay-recipient-maps.cf mydestination = <%= @mydestination.join(",") %> mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128 <% @relaying_servers.each do |relay| -%> <%= relay %><% end %> mailbox_size_limit = 0 diff --git a/templates/mail/postfix/mysql-relay-recipient-maps.erb.cf b/templates/mail/postfix/mysql-virtual-alias-maps.erb.cf similarity index 86% rename from templates/mail/postfix/mysql-relay-recipient-maps.erb.cf rename to templates/mail/postfix/mysql-virtual-alias-maps.erb.cf index 8723db85e..8ebe4326f 100644 --- a/templates/mail/postfix/mysql-relay-recipient-maps.erb.cf +++ b/templates/mail/postfix/mysql-virtual-alias-maps.erb.cf @@ -2,4 +2,4 @@ user = <%= @nextcloud_db_user %> password = <%= @nextcloud_mysql_password %> hosts = <%= @db_hosts %> dbname = <%= @nextcloud_db%> -query = SELECT UNIQUE(REPLACE(value, '<%= @short_domain %>', '<%= @account_domain %>')) FROM oc_accounts_data WHERE name = 'email' +query = SELECT UNIQUE(REPLACE(value, '<%= @short_domain %>', '<%= @account_domain %>')) FROM oc_accounts_data WHERE name = 'email' AND value = '%s@<%= @short_domain %>'; From 1d51e637f81a4700492f1e03e82826eb9f41d6cf Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Mon, 17 Jun 2024 14:27:32 +0200 Subject: [PATCH 034/161] Add relay-recipient-maps --- templates/mail/postfix/main.erb.cf | 1 + templates/mail/postfix/mysql-relay-recipient-maps.erb.cf | 5 +++++ 2 files changed, 6 insertions(+) create mode 100644 templates/mail/postfix/mysql-relay-recipient-maps.erb.cf diff --git a/templates/mail/postfix/main.erb.cf b/templates/mail/postfix/main.erb.cf index d61ad97b4..36a87bca2 100644 --- a/templates/mail/postfix/main.erb.cf +++ b/templates/mail/postfix/main.erb.cf @@ -41,6 +41,7 @@ alias_database = hash:/etc/aliases virtual_alias_maps = mysql:/config/mysql-virtual-alias-maps.cf virtual_mailbox_domains = mysql:/config/mysql-virtual-mailbox-domains.cf virtual_mailbox_maps = mysql:/config/mysql-virtual-mailbox-maps.cf +relay_recipient_maps = mysql:/config/mysql-relay-recipient-maps.cf mydestination = <%= @mydestination.join(",") %> mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128 <% @relaying_servers.each do |relay| -%> <%= relay %><% end %> mailbox_size_limit = 0 diff --git a/templates/mail/postfix/mysql-relay-recipient-maps.erb.cf b/templates/mail/postfix/mysql-relay-recipient-maps.erb.cf new file mode 100644 index 000000000..8723db85e --- /dev/null +++ b/templates/mail/postfix/mysql-relay-recipient-maps.erb.cf @@ -0,0 +1,5 @@ +user = <%= @nextcloud_db_user %> +password = <%= @nextcloud_mysql_password %> +hosts = <%= @db_hosts %> +dbname = <%= @nextcloud_db%> +query = SELECT UNIQUE(REPLACE(value, '<%= @short_domain %>', '<%= @account_domain %>')) FROM oc_accounts_data WHERE name = 'email' From b88822d318f0b5a94d451f4a39b7fa5cfb2fbbd7 Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Mon, 17 Jun 2024 14:42:45 +0200 Subject: [PATCH 035/161] Add virtual alias maps --- templates/mail/postfix/main.erb.cf | 1 - templates/mail/postfix/mysql-relay-recipient-maps.erb.cf | 5 ----- 2 files changed, 6 deletions(-) delete mode 100644 templates/mail/postfix/mysql-relay-recipient-maps.erb.cf diff --git a/templates/mail/postfix/main.erb.cf b/templates/mail/postfix/main.erb.cf index 36a87bca2..d61ad97b4 100644 --- a/templates/mail/postfix/main.erb.cf +++ b/templates/mail/postfix/main.erb.cf @@ -41,7 +41,6 @@ alias_database = hash:/etc/aliases virtual_alias_maps = mysql:/config/mysql-virtual-alias-maps.cf virtual_mailbox_domains = mysql:/config/mysql-virtual-mailbox-domains.cf virtual_mailbox_maps = mysql:/config/mysql-virtual-mailbox-maps.cf -relay_recipient_maps = mysql:/config/mysql-relay-recipient-maps.cf mydestination = <%= @mydestination.join(",") %> mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128 <% @relaying_servers.each do |relay| -%> <%= relay %><% end %> mailbox_size_limit = 0 diff --git a/templates/mail/postfix/mysql-relay-recipient-maps.erb.cf b/templates/mail/postfix/mysql-relay-recipient-maps.erb.cf deleted file mode 100644 index 8723db85e..000000000 --- a/templates/mail/postfix/mysql-relay-recipient-maps.erb.cf +++ /dev/null @@ -1,5 +0,0 @@ -user = <%= @nextcloud_db_user %> -password = <%= @nextcloud_mysql_password %> -hosts = <%= @db_hosts %> -dbname = <%= @nextcloud_db%> -query = SELECT UNIQUE(REPLACE(value, '<%= @short_domain %>', '<%= @account_domain %>')) FROM oc_accounts_data WHERE name = 'email' From eb4bd7b12f89c75abbecd1bcacae8a72378e487f Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Mon, 17 Jun 2024 14:46:42 +0200 Subject: [PATCH 036/161] Fix file name --- manifests/mail/postfix.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/mail/postfix.pp b/manifests/mail/postfix.pp index 9bbb377a0..a47b19d43 100644 --- a/manifests/mail/postfix.pp +++ b/manifests/mail/postfix.pp @@ -65,7 +65,7 @@ $config_files = [ 'main', 'master', - 'mysql-relay-recipient-maps', + 'mysql-virtual-alias-maps', 'mysql-virtual-mailbox-domains', 'mysql-virtual-mailbox-maps' ] From 282153a8c46a804ae53a14ecb5f345b9a65c95e7 Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Mon, 17 Jun 2024 14:53:06 +0200 Subject: [PATCH 037/161] Test another format --- templates/mail/postfix/mysql-virtual-alias-maps.erb.cf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/mail/postfix/mysql-virtual-alias-maps.erb.cf b/templates/mail/postfix/mysql-virtual-alias-maps.erb.cf index 8ebe4326f..8c276cf2a 100644 --- a/templates/mail/postfix/mysql-virtual-alias-maps.erb.cf +++ b/templates/mail/postfix/mysql-virtual-alias-maps.erb.cf @@ -2,4 +2,4 @@ user = <%= @nextcloud_db_user %> password = <%= @nextcloud_mysql_password %> hosts = <%= @db_hosts %> dbname = <%= @nextcloud_db%> -query = SELECT UNIQUE(REPLACE(value, '<%= @short_domain %>', '<%= @account_domain %>')) FROM oc_accounts_data WHERE name = 'email' AND value = '%s@<%= @short_domain %>'; +query = SELECT UNIQUE(value) FROM oc_accounts_data WHERE name = 'email' AND value = REPLACE('%s', '<%= @account_domain %>','<%= @short_domain %>'); From f18f51ee7644167225a6b3a337d676b025c17b8c Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Mon, 17 Jun 2024 14:27:32 +0200 Subject: [PATCH 038/161] Add relay-recipient-maps --- templates/mail/postfix/main.erb.cf | 1 + templates/mail/postfix/mysql-relay-recipient-maps.erb.cf | 5 +++++ 2 files changed, 6 insertions(+) create mode 100644 templates/mail/postfix/mysql-relay-recipient-maps.erb.cf diff --git a/templates/mail/postfix/main.erb.cf b/templates/mail/postfix/main.erb.cf index d61ad97b4..36a87bca2 100644 --- a/templates/mail/postfix/main.erb.cf +++ b/templates/mail/postfix/main.erb.cf @@ -41,6 +41,7 @@ alias_database = hash:/etc/aliases virtual_alias_maps = mysql:/config/mysql-virtual-alias-maps.cf virtual_mailbox_domains = mysql:/config/mysql-virtual-mailbox-domains.cf virtual_mailbox_maps = mysql:/config/mysql-virtual-mailbox-maps.cf +relay_recipient_maps = mysql:/config/mysql-relay-recipient-maps.cf mydestination = <%= @mydestination.join(",") %> mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128 <% @relaying_servers.each do |relay| -%> <%= relay %><% end %> mailbox_size_limit = 0 diff --git a/templates/mail/postfix/mysql-relay-recipient-maps.erb.cf b/templates/mail/postfix/mysql-relay-recipient-maps.erb.cf new file mode 100644 index 000000000..8723db85e --- /dev/null +++ b/templates/mail/postfix/mysql-relay-recipient-maps.erb.cf @@ -0,0 +1,5 @@ +user = <%= @nextcloud_db_user %> +password = <%= @nextcloud_mysql_password %> +hosts = <%= @db_hosts %> +dbname = <%= @nextcloud_db%> +query = SELECT UNIQUE(REPLACE(value, '<%= @short_domain %>', '<%= @account_domain %>')) FROM oc_accounts_data WHERE name = 'email' From 76c5a3836e42581935cdabe80d2a4b25728c3984 Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Mon, 17 Jun 2024 14:42:45 +0200 Subject: [PATCH 039/161] Add virtual alias maps --- templates/mail/postfix/main.erb.cf | 1 - templates/mail/postfix/mysql-relay-recipient-maps.erb.cf | 5 ----- 2 files changed, 6 deletions(-) delete mode 100644 templates/mail/postfix/mysql-relay-recipient-maps.erb.cf diff --git a/templates/mail/postfix/main.erb.cf b/templates/mail/postfix/main.erb.cf index 36a87bca2..d61ad97b4 100644 --- a/templates/mail/postfix/main.erb.cf +++ b/templates/mail/postfix/main.erb.cf @@ -41,7 +41,6 @@ alias_database = hash:/etc/aliases virtual_alias_maps = mysql:/config/mysql-virtual-alias-maps.cf virtual_mailbox_domains = mysql:/config/mysql-virtual-mailbox-domains.cf virtual_mailbox_maps = mysql:/config/mysql-virtual-mailbox-maps.cf -relay_recipient_maps = mysql:/config/mysql-relay-recipient-maps.cf mydestination = <%= @mydestination.join(",") %> mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128 <% @relaying_servers.each do |relay| -%> <%= relay %><% end %> mailbox_size_limit = 0 diff --git a/templates/mail/postfix/mysql-relay-recipient-maps.erb.cf b/templates/mail/postfix/mysql-relay-recipient-maps.erb.cf deleted file mode 100644 index 8723db85e..000000000 --- a/templates/mail/postfix/mysql-relay-recipient-maps.erb.cf +++ /dev/null @@ -1,5 +0,0 @@ -user = <%= @nextcloud_db_user %> -password = <%= @nextcloud_mysql_password %> -hosts = <%= @db_hosts %> -dbname = <%= @nextcloud_db%> -query = SELECT UNIQUE(REPLACE(value, '<%= @short_domain %>', '<%= @account_domain %>')) FROM oc_accounts_data WHERE name = 'email' From 4e6c870d0589bfc3cf10c423b61509eda0c733c7 Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Mon, 17 Jun 2024 16:07:22 +0200 Subject: [PATCH 040/161] Explicit check --- templates/frontend/docker-compose_template.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/frontend/docker-compose_template.erb b/templates/frontend/docker-compose_template.erb index 49a4c1768..74dabc8c4 100644 --- a/templates/frontend/docker-compose_template.erb +++ b/templates/frontend/docker-compose_template.erb @@ -3,7 +3,7 @@ version: '3' services: haproxy: image: '<%= @haproxy_image %>:<%= @haproxy_imagetag %>' -<% if @set_fqdn -%> +<% if @set_fqdn == true -%> hostname: <%= @facts['fqdn'] %> <%- end -%> expose: From 2fc85e952fe45608c3a5d30d8c7d3bca6f9f2251 Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Mon, 17 Jun 2024 16:22:22 +0200 Subject: [PATCH 041/161] Should go in websites2 --- manifests/frontend/load_balancer.pp | 1 - manifests/frontend/load_balancer/website2.pp | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/frontend/load_balancer.pp b/manifests/frontend/load_balancer.pp index 4daf6e309..5a54c819e 100644 --- a/manifests/frontend/load_balancer.pp +++ b/manifests/frontend/load_balancer.pp @@ -13,7 +13,6 @@ Integer $haproxy = $base_uidgid + 10, Integer $telegraf = $base_uidgid + 11, Integer $varnish = $base_uidgid + 12, - Boolean $set_fqdn = false, ) { $config = lookup('sunet_frontend', undef, undef, undef) if $config =~ Hash[String, Hash] { diff --git a/manifests/frontend/load_balancer/website2.pp b/manifests/frontend/load_balancer/website2.pp index 7d270e770..60d4fbc3e 100644 --- a/manifests/frontend/load_balancer/website2.pp +++ b/manifests/frontend/load_balancer/website2.pp @@ -105,6 +105,7 @@ $haproxy_imagetag = pick($config['haproxy_imagetag'], 'stable') $haproxy_volumes = pick($config['haproxy_volumes'], false) $multinode_port = pick_default($config['multinode_port'], false) + $set_fqdn = pick($config['set_fqdn'], false) $statsd_enabled = pick($config['statsd_enabled'], true) $statsd_host = pick($::ipaddress_docker0, $::ipaddress) $varnish_config = pick($config['varnish_config'], '/opt/frontend/config/common/default.vcl') From 7135fb88fb6e30bee1fb925ca22d3f60bb20cfd4 Mon Sep 17 00:00:00 2001 From: Maria Haider Date: Thu, 29 Aug 2024 13:05:14 +0200 Subject: [PATCH 042/161] open just allowed ports --- manifests/nftables/docker_expose.pp | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/manifests/nftables/docker_expose.pp b/manifests/nftables/docker_expose.pp index 31e7066ea..75d56bd35 100644 --- a/manifests/nftables/docker_expose.pp +++ b/manifests/nftables/docker_expose.pp @@ -52,9 +52,18 @@ notify => Service['nftables'], ; } - sunet::nftables::allow { "expose-allow-${safe_name}": - from => 'any', - port => $port, + if ($allow_clients =~ Array[String, 1]) or ($allow_clients =~ String[1]) { + sunet::nftables::allow { "expose-allow-${safe_name}": + from => $allow_clients, + port => $port, + proto => $proto, + } + } else { + sunet::nftables::allow { "expose-allow-${safe_name}": + from => any, + port => $port, + proto => $proto, + } } } } From 9b194a372946abfddea27b7440432b88937c920a Mon Sep 17 00:00:00 2001 From: Maria Haider Date: Fri, 30 Aug 2024 13:43:42 +0200 Subject: [PATCH 043/161] if clause not needed --- manifests/nftables/docker_expose.pp | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/manifests/nftables/docker_expose.pp b/manifests/nftables/docker_expose.pp index 75d56bd35..429a39f07 100644 --- a/manifests/nftables/docker_expose.pp +++ b/manifests/nftables/docker_expose.pp @@ -52,18 +52,10 @@ notify => Service['nftables'], ; } - if ($allow_clients =~ Array[String, 1]) or ($allow_clients =~ String[1]) { - sunet::nftables::allow { "expose-allow-${safe_name}": - from => $allow_clients, - port => $port, - proto => $proto, - } - } else { - sunet::nftables::allow { "expose-allow-${safe_name}": - from => any, - port => $port, - proto => $proto, - } + sunet::nftables::allow { "expose-allow-${safe_name}": + from => $allow_clients, + port => $port, + proto => $proto, } } } From 616968eac3e3e8a32dfee5e8871eb392e542420e Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Wed, 4 Sep 2024 08:34:39 +0200 Subject: [PATCH 044/161] fix wrong variable name --- manifests/mail/postfix.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/mail/postfix.pp b/manifests/mail/postfix.pp index 4cd8111d7..5e5496625 100644 --- a/manifests/mail/postfix.pp +++ b/manifests/mail/postfix.pp @@ -19,7 +19,7 @@ $config = lookup($environment) $db_hosts = join($config['db_hosts'], ' ') - $relay_hosts = join($relay_servers, ', ') + $relay_hosts = join($relaying_servers, ', ') $nextcloud_db = 'nextcloud' $nextcloud_db_user ='nextcloud' $nextcloud_mysql_password = lookup('nextcloud_mysql_password') From c1bd16608cf24e663ae2c9b7636c7086245d0a49 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Mon, 2 Sep 2024 12:58:18 +0200 Subject: [PATCH 045/161] Mimic good old `docker_run` Which always pull images on (re)start --- templates/dockerhost/docker-compose.yml.erb | 1 + 1 file changed, 1 insertion(+) diff --git a/templates/dockerhost/docker-compose.yml.erb b/templates/dockerhost/docker-compose.yml.erb index 9ebe81487..38d2f7c60 100644 --- a/templates/dockerhost/docker-compose.yml.erb +++ b/templates/dockerhost/docker-compose.yml.erb @@ -51,6 +51,7 @@ services: <% end -%> <% end -%> image: <%= @image_tag %> + pull_policy: always networks: docker: From 6bdd87602b341cf5a06df4d7bd23d8f305ef51cd Mon Sep 17 00:00:00 2001 From: Maria Haider Date: Thu, 29 Aug 2024 13:05:14 +0200 Subject: [PATCH 046/161] open just allowed ports --- manifests/nftables/docker_expose.pp | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/manifests/nftables/docker_expose.pp b/manifests/nftables/docker_expose.pp index 31e7066ea..75d56bd35 100644 --- a/manifests/nftables/docker_expose.pp +++ b/manifests/nftables/docker_expose.pp @@ -52,9 +52,18 @@ notify => Service['nftables'], ; } - sunet::nftables::allow { "expose-allow-${safe_name}": - from => 'any', - port => $port, + if ($allow_clients =~ Array[String, 1]) or ($allow_clients =~ String[1]) { + sunet::nftables::allow { "expose-allow-${safe_name}": + from => $allow_clients, + port => $port, + proto => $proto, + } + } else { + sunet::nftables::allow { "expose-allow-${safe_name}": + from => any, + port => $port, + proto => $proto, + } } } } From 6c9cacb58edd44bdc5f08bfc9455a2468c6b6424 Mon Sep 17 00:00:00 2001 From: Maria Haider Date: Fri, 30 Aug 2024 13:43:42 +0200 Subject: [PATCH 047/161] if clause not needed --- manifests/nftables/docker_expose.pp | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/manifests/nftables/docker_expose.pp b/manifests/nftables/docker_expose.pp index 75d56bd35..429a39f07 100644 --- a/manifests/nftables/docker_expose.pp +++ b/manifests/nftables/docker_expose.pp @@ -52,18 +52,10 @@ notify => Service['nftables'], ; } - if ($allow_clients =~ Array[String, 1]) or ($allow_clients =~ String[1]) { - sunet::nftables::allow { "expose-allow-${safe_name}": - from => $allow_clients, - port => $port, - proto => $proto, - } - } else { - sunet::nftables::allow { "expose-allow-${safe_name}": - from => any, - port => $port, - proto => $proto, - } + sunet::nftables::allow { "expose-allow-${safe_name}": + from => $allow_clients, + port => $port, + proto => $proto, } } } From 9997bfc0a6345184dac1e33af5c539499479aa53 Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Wed, 4 Sep 2024 08:34:39 +0200 Subject: [PATCH 048/161] fix wrong variable name --- manifests/mail/postfix.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/mail/postfix.pp b/manifests/mail/postfix.pp index 4cd8111d7..5e5496625 100644 --- a/manifests/mail/postfix.pp +++ b/manifests/mail/postfix.pp @@ -19,7 +19,7 @@ $config = lookup($environment) $db_hosts = join($config['db_hosts'], ' ') - $relay_hosts = join($relay_servers, ', ') + $relay_hosts = join($relaying_servers, ', ') $nextcloud_db = 'nextcloud' $nextcloud_db_user ='nextcloud' $nextcloud_mysql_password = lookup('nextcloud_mysql_password') From d1ff214bab969719a4da45223db1793f9bd83575 Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Wed, 4 Sep 2024 08:42:37 +0200 Subject: [PATCH 049/161] fix wrong variable name --- manifests/mail/postfix.pp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/manifests/mail/postfix.pp b/manifests/mail/postfix.pp index 5e5496625..004171a92 100644 --- a/manifests/mail/postfix.pp +++ b/manifests/mail/postfix.pp @@ -19,7 +19,7 @@ $config = lookup($environment) $db_hosts = join($config['db_hosts'], ' ') - $relay_hosts = join($relaying_servers, ', ') + $relay_hosts = join($relay_servers, ', ') $nextcloud_db = 'nextcloud' $nextcloud_db_user ='nextcloud' $nextcloud_mysql_password = lookup('nextcloud_mysql_password') @@ -46,7 +46,7 @@ $restricted_ports = [25] $restricted_ports.each|$port| { sunet::nftables::docker_expose { "mail_port_${port}": - allow_clients => $relay_hosts, + allow_clients => $relaying_servers, port => $port, iif => $interface, } From 5bb876d37ea7a621f1378d7070c53a4f2290de3e Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Wed, 4 Sep 2024 09:04:36 +0200 Subject: [PATCH 050/161] Strip out brackets --- manifests/mail/postfix.pp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/manifests/mail/postfix.pp b/manifests/mail/postfix.pp index 88d2a13e3..4165e8735 100644 --- a/manifests/mail/postfix.pp +++ b/manifests/mail/postfix.pp @@ -20,6 +20,9 @@ $config = lookup($environment) $db_hosts = join($config['db_hosts'], ' ') $relay_hosts = join($relaying_servers, ', ') + $incomming_servers = $relaying_servers.map |$server| { + regsubst($server, '[\\[\\]]', '', 'G') + } $nextcloud_db = 'nextcloud' $nextcloud_db_user ='nextcloud' $nextcloud_mysql_password = lookup('nextcloud_mysql_password') @@ -46,7 +49,7 @@ $restricted_ports = [25] $restricted_ports.each|$port| { sunet::nftables::docker_expose { "mail_port_${port}": - allow_clients => $relaying_servers, + allow_clients => $incomming_servers, port => $port, iif => $interface, } From c2b1d5fc5a72bd6312bbff1ff5afaa7a1a528615 Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Wed, 4 Sep 2024 09:04:36 +0200 Subject: [PATCH 051/161] Strip out brackets --- manifests/mail/postfix.pp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/manifests/mail/postfix.pp b/manifests/mail/postfix.pp index 004171a92..93404f082 100644 --- a/manifests/mail/postfix.pp +++ b/manifests/mail/postfix.pp @@ -20,6 +20,9 @@ $config = lookup($environment) $db_hosts = join($config['db_hosts'], ' ') $relay_hosts = join($relay_servers, ', ') + $incomming_servers = $relaying_servers.map |$server| { + regsubst($server, '[\\[\\]]', '', 'G') + } $nextcloud_db = 'nextcloud' $nextcloud_db_user ='nextcloud' $nextcloud_mysql_password = lookup('nextcloud_mysql_password') @@ -46,7 +49,7 @@ $restricted_ports = [25] $restricted_ports.each|$port| { sunet::nftables::docker_expose { "mail_port_${port}": - allow_clients => $relaying_servers, + allow_clients => $incomming_servers, port => $port, iif => $interface, } From 2e2ef666607fb504c36a59d5080f140a264e5a2f Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Tue, 10 Sep 2024 14:52:19 +0200 Subject: [PATCH 052/161] Use any instead of 0.0.0.0 --- manifests/rediscluster.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/rediscluster.pp b/manifests/rediscluster.pp index 6bb7ee76a..86cf97442 100644 --- a/manifests/rediscluster.pp +++ b/manifests/rediscluster.pp @@ -81,7 +81,7 @@ } } else { sunet::misc::ufw_allow { "redis_port_${i}": - from => '0.0.0.0/0', + from => 'any', port => [$redisportnum,$clusterportnum], } } From c4f9aba9d824797e19d3709779fa9b697b44c699 Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Tue, 10 Sep 2024 15:00:28 +0200 Subject: [PATCH 053/161] Use all defaults --- manifests/rediscluster.pp | 1 - 1 file changed, 1 deletion(-) diff --git a/manifests/rediscluster.pp b/manifests/rediscluster.pp index 86cf97442..4e7e2ad11 100644 --- a/manifests/rediscluster.pp +++ b/manifests/rediscluster.pp @@ -81,7 +81,6 @@ } } else { sunet::misc::ufw_allow { "redis_port_${i}": - from => 'any', port => [$redisportnum,$clusterportnum], } } From 560387614e8b3cbfb1efe02128a9b1ba566399bc Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Thu, 12 Sep 2024 12:46:14 +0200 Subject: [PATCH 054/161] Split rules --- manifests/rediscluster.pp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/manifests/rediscluster.pp b/manifests/rediscluster.pp index 4f8446177..7bf7ea78b 100644 --- a/manifests/rediscluster.pp +++ b/manifests/rediscluster.pp @@ -80,9 +80,13 @@ } } } else { - sunet::misc::ufw_allow { "redis_port_${i}": + sunet::misc::ufw_allow { "redis_port_${i}_v6": port => [$redisportnum,$clusterportnum], - from => ['::/0', '0.0.0.0/0'], + from => '::/0', + } + sunet::misc::ufw_allow { "redis_port_${i}_v4": + port => [$redisportnum,$clusterportnum], + from => '0.0.0.0/0', } } } From e409fefe381335ec52a2d4b26a8b674c6bf5c936 Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Thu, 12 Sep 2024 13:00:48 +0200 Subject: [PATCH 055/161] Add ipv6 to redict --- manifests/redictcluster.pp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/manifests/redictcluster.pp b/manifests/redictcluster.pp index 90a81e838..ddb6cd444 100644 --- a/manifests/redictcluster.pp +++ b/manifests/redictcluster.pp @@ -82,7 +82,11 @@ } } } else { - sunet::misc::ufw_allow { "redict_port_${i}": + sunet::misc::ufw_allow { "redict_port_${i}_v6": + from => '::/0', + port => [$redictportnum,$clusterportnum], + } + sunet::misc::ufw_allow { "redict_port_${i}_v4": from => '0.0.0.0/0', port => [$redictportnum,$clusterportnum], } From 577ae495d50f156f174cfee0ef7380bd5b0dc220 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Tue, 17 Sep 2024 13:54:53 +0200 Subject: [PATCH 056/161] Do we need a define? --- manifests/mariadb.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/mariadb.pp b/manifests/mariadb.pp index e6993a469..009c95a7a 100644 --- a/manifests/mariadb.pp +++ b/manifests/mariadb.pp @@ -1,5 +1,5 @@ # Mariadb cluster class for SUNET -define sunet::mariadb( +class sunet::mariadb( $mariadb_version=latest, $bootstrap=0, $ports = [3306, 4444, 4567, 4568], From 9d7c1f69155500e8909fda898c5e3d1e4bf6b89b Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Tue, 17 Sep 2024 14:04:55 +0200 Subject: [PATCH 057/161] Define for transition towards class? --- manifests/mariadb.pp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/manifests/mariadb.pp b/manifests/mariadb.pp index 009c95a7a..07fcb4027 100644 --- a/manifests/mariadb.pp +++ b/manifests/mariadb.pp @@ -1,3 +1,10 @@ +# Mariadb cluster definefor SUNET +define sunet::mariadb(){ + + require sunet::mariadb + +} + # Mariadb cluster class for SUNET class sunet::mariadb( $mariadb_version=latest, From 808dae3e79f6501a745373690e54bbe383f73a7d Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Tue, 17 Sep 2024 14:07:38 +0200 Subject: [PATCH 058/161] Name collition --- manifests/mariadb.pp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/manifests/mariadb.pp b/manifests/mariadb.pp index 07fcb4027..55498490f 100644 --- a/manifests/mariadb.pp +++ b/manifests/mariadb.pp @@ -1,12 +1,12 @@ # Mariadb cluster definefor SUNET define sunet::mariadb(){ - require sunet::mariadb + require sunet::mariadb::server } # Mariadb cluster class for SUNET -class sunet::mariadb( +class sunet::mariadb::server( $mariadb_version=latest, $bootstrap=0, $ports = [3306, 4444, 4567, 4568], From 2ececf61656ea2af1462c06a95972855f9e8df67 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Tue, 17 Sep 2024 14:11:27 +0200 Subject: [PATCH 059/161] Show deprecation warnings --- manifests/mariadb.pp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/manifests/mariadb.pp b/manifests/mariadb.pp index 55498490f..d727c33f1 100644 --- a/manifests/mariadb.pp +++ b/manifests/mariadb.pp @@ -1,8 +1,7 @@ # Mariadb cluster definefor SUNET define sunet::mariadb(){ - + warning('Please transition to the class "sunet::mariadb::server" instead of this define') require sunet::mariadb::server - } # Mariadb cluster class for SUNET From 438065200732c56bf04e4039b5ee8772b74aae32 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Tue, 17 Sep 2024 14:39:10 +0200 Subject: [PATCH 060/161] Puppet recommends split out files --- manifests/mariadb/init.pp | 5 +++++ manifests/{mariadb.pp => mariadb/server.pp} | 6 ------ 2 files changed, 5 insertions(+), 6 deletions(-) create mode 100644 manifests/mariadb/init.pp rename manifests/{mariadb.pp => mariadb/server.pp} (95%) diff --git a/manifests/mariadb/init.pp b/manifests/mariadb/init.pp new file mode 100644 index 000000000..170bb4fe1 --- /dev/null +++ b/manifests/mariadb/init.pp @@ -0,0 +1,5 @@ +# Mariadb cluster definefor SUNET +define sunet::mariadb(){ + warning('Please transition to the class "sunet::mariadb::server" instead of this define') + require sunet::mariadb::server +} diff --git a/manifests/mariadb.pp b/manifests/mariadb/server.pp similarity index 95% rename from manifests/mariadb.pp rename to manifests/mariadb/server.pp index d727c33f1..370a74329 100644 --- a/manifests/mariadb.pp +++ b/manifests/mariadb/server.pp @@ -1,9 +1,3 @@ -# Mariadb cluster definefor SUNET -define sunet::mariadb(){ - warning('Please transition to the class "sunet::mariadb::server" instead of this define') - require sunet::mariadb::server -} - # Mariadb cluster class for SUNET class sunet::mariadb::server( $mariadb_version=latest, From f78955a004ba3d044be13d77446dda5ce5b9b5ab Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Thu, 19 Sep 2024 12:26:21 +0200 Subject: [PATCH 061/161] Make it possible to run multiple instances At the same time add simple class good for most. --- manifests/mariadb/init.pp | 110 ++++++++++++++++++++++++++++++++++-- manifests/mariadb/server.pp | 107 ----------------------------------- manifests/mariadb/simple.pp | 15 +++++ 3 files changed, 121 insertions(+), 111 deletions(-) delete mode 100644 manifests/mariadb/server.pp create mode 100644 manifests/mariadb/simple.pp diff --git a/manifests/mariadb/init.pp b/manifests/mariadb/init.pp index 170bb4fe1..e6993a469 100644 --- a/manifests/mariadb/init.pp +++ b/manifests/mariadb/init.pp @@ -1,5 +1,107 @@ -# Mariadb cluster definefor SUNET -define sunet::mariadb(){ - warning('Please transition to the class "sunet::mariadb::server" instead of this define') - require sunet::mariadb::server +# Mariadb cluster class for SUNET +define sunet::mariadb( + $mariadb_version=latest, + $bootstrap=0, + $ports = [3306, 4444, 4567, 4568], + $dns = undef, +) +{ + $mariadb_root_password = lookup('mariadb_root_password', undef, undef,'NOT_SET_IN_HIERA') + $mariadb_user = lookup('mariadb_user', undef, undef,undef) + $mariadb_user_password = lookup('mariadb_user_password', undef, undef,undef) + $mariadb_database = lookup('mariadb_database', undef, undef,undef) + $mariadb_backup_password = lookup('mariadb_root_password', undef, undef,'NOT_SET_IN_HIERA') + $clients = lookup('mariadb_clients', undef, undef,['127.0.0.1']) + $cluster_nodes = lookup('mariadb_cluster_nodes', undef, undef,[]) + $mariadb_dir = '/opt/mariadb' + $server_id = 1000 + Integer($facts['networking']['hostname'][-1]) + + # Hack to not clash with docker_compose which tries to create the same directory + exec {'mariadb_dir_create': + command => "mkdir -p ${mariadb_dir}", + unless => "test -d ${mariadb_dir}", + } + + $dirs = ['datadir', 'init', 'conf', 'backups', 'scripts' ] + $dirs.each |$dir| { + ensure_resource('file',"${mariadb_dir}/${dir}", { ensure => directory, recurse => true } ) + } + + $_from = $clients + $cluster_nodes + sunet::misc::ufw_allow { 'mariadb_ports': + from => $_from, + port => $ports, + } + + file { '/usr/local/bin/purge-binlogs': + ensure => present, + content => template('sunet/mariadb/purge-binlogs.erb.sh'), + mode => '0744', + owner => 999, + group => 999, + } + file { '/usr/local/bin/run_manual_backup_dump': + ensure => present, + content => template('sunet/mariadb/run_manual_backup_dump.erb.sh'), + mode => '0744', + owner => 999, + group => 999, + } + sunet::scriptherder::cronjob { 'purge_binlogs': + cmd => '/usr/local/bin/purge-binlogs', + hour => '6', + minute => '0', + ok_criteria => ['exit_status=0','max_age=2d'], + warn_criteria => ['exit_status=1','max_age=3d'], + } + file { '/usr/local/bin/cluster-size': + ensure => present, + content => template('sunet/mariadb/cluster-size.erb.sh'), + mode => '0744', + } + file { '/usr/local/bin/cluster-status': + ensure => present, + content => template('sunet/mariadb/cluster-status.erb.sh'), + mode => '0744', + } + file { '/etc/sudoers.d/99-size-test': + ensure => file, + content => "script ALL=(root) NOPASSWD: /usr/local/bin/cluster-size\n", + mode => '0440', + owner => 'root', + group => 'root', + } + file { '/etc/sudoers.d/99-status-test': + ensure => file, + content => "script ALL=(root) NOPASSWD: /usr/local/bin/cluster-status\n", + mode => '0440', + owner => 'root', + group => 'root', + } + + $sql_files = ['02-backup_user.sql'] + $sql_files.each |$sql_file|{ + file { "${mariadb_dir}/init/${sql_file}": + ensure => present, + content => template("sunet/mariadb/${sql_file}.erb"), + mode => '0744', + } + } + file { "${mariadb_dir}/conf/credentials.cnf": + ensure => present, + content => template('sunet/mariadb/credentials.cnf.erb'), + mode => '0744', + } + file { "${mariadb_dir}/conf/my.cnf": + ensure => present, + content => template('sunet/mariadb/my.cnf.erb'), + mode => '0744', + } + $docker_compose = sunet::docker_compose { 'sunet_mariadb_docker_compose': + content => template('sunet/mariadb/docker-compose_mariadb.yml.erb'), + service_name => 'mariadb', + compose_dir => '/opt/', + compose_filename => 'docker-compose.yml', + description => 'Mariadb server', + } } diff --git a/manifests/mariadb/server.pp b/manifests/mariadb/server.pp deleted file mode 100644 index 370a74329..000000000 --- a/manifests/mariadb/server.pp +++ /dev/null @@ -1,107 +0,0 @@ -# Mariadb cluster class for SUNET -class sunet::mariadb::server( - $mariadb_version=latest, - $bootstrap=0, - $ports = [3306, 4444, 4567, 4568], - $dns = undef, -) -{ - $mariadb_root_password = lookup('mariadb_root_password', undef, undef,'NOT_SET_IN_HIERA') - $mariadb_user = lookup('mariadb_user', undef, undef,undef) - $mariadb_user_password = lookup('mariadb_user_password', undef, undef,undef) - $mariadb_database = lookup('mariadb_database', undef, undef,undef) - $mariadb_backup_password = lookup('mariadb_root_password', undef, undef,'NOT_SET_IN_HIERA') - $clients = lookup('mariadb_clients', undef, undef,['127.0.0.1']) - $cluster_nodes = lookup('mariadb_cluster_nodes', undef, undef,[]) - $mariadb_dir = '/opt/mariadb' - $server_id = 1000 + Integer($facts['networking']['hostname'][-1]) - - # Hack to not clash with docker_compose which tries to create the same directory - exec {'mariadb_dir_create': - command => "mkdir -p ${mariadb_dir}", - unless => "test -d ${mariadb_dir}", - } - - $dirs = ['datadir', 'init', 'conf', 'backups', 'scripts' ] - $dirs.each |$dir| { - ensure_resource('file',"${mariadb_dir}/${dir}", { ensure => directory, recurse => true } ) - } - - $_from = $clients + $cluster_nodes - sunet::misc::ufw_allow { 'mariadb_ports': - from => $_from, - port => $ports, - } - - file { '/usr/local/bin/purge-binlogs': - ensure => present, - content => template('sunet/mariadb/purge-binlogs.erb.sh'), - mode => '0744', - owner => 999, - group => 999, - } - file { '/usr/local/bin/run_manual_backup_dump': - ensure => present, - content => template('sunet/mariadb/run_manual_backup_dump.erb.sh'), - mode => '0744', - owner => 999, - group => 999, - } - sunet::scriptherder::cronjob { 'purge_binlogs': - cmd => '/usr/local/bin/purge-binlogs', - hour => '6', - minute => '0', - ok_criteria => ['exit_status=0','max_age=2d'], - warn_criteria => ['exit_status=1','max_age=3d'], - } - file { '/usr/local/bin/cluster-size': - ensure => present, - content => template('sunet/mariadb/cluster-size.erb.sh'), - mode => '0744', - } - file { '/usr/local/bin/cluster-status': - ensure => present, - content => template('sunet/mariadb/cluster-status.erb.sh'), - mode => '0744', - } - file { '/etc/sudoers.d/99-size-test': - ensure => file, - content => "script ALL=(root) NOPASSWD: /usr/local/bin/cluster-size\n", - mode => '0440', - owner => 'root', - group => 'root', - } - file { '/etc/sudoers.d/99-status-test': - ensure => file, - content => "script ALL=(root) NOPASSWD: /usr/local/bin/cluster-status\n", - mode => '0440', - owner => 'root', - group => 'root', - } - - $sql_files = ['02-backup_user.sql'] - $sql_files.each |$sql_file|{ - file { "${mariadb_dir}/init/${sql_file}": - ensure => present, - content => template("sunet/mariadb/${sql_file}.erb"), - mode => '0744', - } - } - file { "${mariadb_dir}/conf/credentials.cnf": - ensure => present, - content => template('sunet/mariadb/credentials.cnf.erb'), - mode => '0744', - } - file { "${mariadb_dir}/conf/my.cnf": - ensure => present, - content => template('sunet/mariadb/my.cnf.erb'), - mode => '0744', - } - $docker_compose = sunet::docker_compose { 'sunet_mariadb_docker_compose': - content => template('sunet/mariadb/docker-compose_mariadb.yml.erb'), - service_name => 'mariadb', - compose_dir => '/opt/', - compose_filename => 'docker-compose.yml', - description => 'Mariadb server', - } -} diff --git a/manifests/mariadb/simple.pp b/manifests/mariadb/simple.pp new file mode 100644 index 000000000..3f3ef86fe --- /dev/null +++ b/manifests/mariadb/simple.pp @@ -0,0 +1,15 @@ +# A simple class to setup mariadb +# Simple but elegant class for a more civilized age +class sunet::mariadb::simple( + String $mariadb_version=latest, + Integer $bootstrap=0, + Array[Integer] $ports = [3306, 4444, 4567, 4568], + Array[String] $dns = [], +){ + sunet::mariadb { 'sunet_mariadb_simple': + mariadb_version => $mariadb_version, + bootstrap => $bootstrap, + ports => $ports, + dns => $dns, + } +} From 13ea66523e5114f85d165902c5e1a1c9c85634e5 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Thu, 19 Sep 2024 12:44:45 +0200 Subject: [PATCH 062/161] Defines can't be in init.pp --- manifests/{mariadb/init.pp => mariadb.pp} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename manifests/{mariadb/init.pp => mariadb.pp} (100%) diff --git a/manifests/mariadb/init.pp b/manifests/mariadb.pp similarity index 100% rename from manifests/mariadb/init.pp rename to manifests/mariadb.pp From f0c5cb94d0591f25e808cfe29d1e6173551c57fa Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Thu, 19 Sep 2024 12:51:12 +0200 Subject: [PATCH 063/161] Only add resolvers if there is data in the array --- templates/mariadb/docker-compose_mariadb.yml.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/mariadb/docker-compose_mariadb.yml.erb b/templates/mariadb/docker-compose_mariadb.yml.erb index 9309dfebe..59d82d918 100644 --- a/templates/mariadb/docker-compose_mariadb.yml.erb +++ b/templates/mariadb/docker-compose_mariadb.yml.erb @@ -12,7 +12,7 @@ services: - /opt/mariadb/init:/docker-entrypoint-initdb.d - /opt/mariadb/scripts:/scripts network_mode: host -<%- if @dns -%> +<%- if !@dns.empty? -%> dns: - <%= @dns %> <%- end -%> From 45abfe45dbcdb6ae3fb12dded943beb62e7b51ed Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Thu, 19 Sep 2024 12:54:38 +0200 Subject: [PATCH 064/161] Expand the list --- templates/mariadb/docker-compose_mariadb.yml.erb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/templates/mariadb/docker-compose_mariadb.yml.erb b/templates/mariadb/docker-compose_mariadb.yml.erb index 59d82d918..d92245767 100644 --- a/templates/mariadb/docker-compose_mariadb.yml.erb +++ b/templates/mariadb/docker-compose_mariadb.yml.erb @@ -14,7 +14,9 @@ services: network_mode: host <%- if !@dns.empty? -%> dns: - - <%= @dns %> +<% @dns.each do |resolver| -%> + - '<%= resolver %>' +<% end -%> <%- end -%> environment: - MYSQL_ROOT_PASSWORD=<%= @mariadb_root_password %> From e726eeb619bfd55ec65bd9e86539dca64b279543 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Thu, 19 Sep 2024 14:21:48 +0200 Subject: [PATCH 065/161] Typing --- manifests/mariadb.pp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/manifests/mariadb.pp b/manifests/mariadb.pp index e6993a469..b96f305ed 100644 --- a/manifests/mariadb.pp +++ b/manifests/mariadb.pp @@ -1,9 +1,9 @@ # Mariadb cluster class for SUNET define sunet::mariadb( - $mariadb_version=latest, - $bootstrap=0, - $ports = [3306, 4444, 4567, 4568], - $dns = undef, + String $mariadb_version=latest, + Integer $bootstrap=0, + Array[Integer] $ports = [3306, 4444, 4567, 4568], + Array[String] $dns = [], ) { $mariadb_root_password = lookup('mariadb_root_password', undef, undef,'NOT_SET_IN_HIERA') From 0f4fcaad54fdd5a3ab5de0606498d5cd27d24f0d Mon Sep 17 00:00:00 2001 From: Maria Haider Date: Thu, 29 Aug 2024 13:05:14 +0200 Subject: [PATCH 066/161] open just allowed ports --- manifests/nftables/docker_expose.pp | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/manifests/nftables/docker_expose.pp b/manifests/nftables/docker_expose.pp index 429a39f07..75d56bd35 100644 --- a/manifests/nftables/docker_expose.pp +++ b/manifests/nftables/docker_expose.pp @@ -52,10 +52,18 @@ notify => Service['nftables'], ; } - sunet::nftables::allow { "expose-allow-${safe_name}": - from => $allow_clients, - port => $port, - proto => $proto, + if ($allow_clients =~ Array[String, 1]) or ($allow_clients =~ String[1]) { + sunet::nftables::allow { "expose-allow-${safe_name}": + from => $allow_clients, + port => $port, + proto => $proto, + } + } else { + sunet::nftables::allow { "expose-allow-${safe_name}": + from => any, + port => $port, + proto => $proto, + } } } } From a35eac872e87ced63c98257449d3a9dafc4f8284 Mon Sep 17 00:00:00 2001 From: Maria Haider Date: Fri, 30 Aug 2024 13:43:42 +0200 Subject: [PATCH 067/161] if clause not needed --- manifests/nftables/docker_expose.pp | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/manifests/nftables/docker_expose.pp b/manifests/nftables/docker_expose.pp index 75d56bd35..429a39f07 100644 --- a/manifests/nftables/docker_expose.pp +++ b/manifests/nftables/docker_expose.pp @@ -52,18 +52,10 @@ notify => Service['nftables'], ; } - if ($allow_clients =~ Array[String, 1]) or ($allow_clients =~ String[1]) { - sunet::nftables::allow { "expose-allow-${safe_name}": - from => $allow_clients, - port => $port, - proto => $proto, - } - } else { - sunet::nftables::allow { "expose-allow-${safe_name}": - from => any, - port => $port, - proto => $proto, - } + sunet::nftables::allow { "expose-allow-${safe_name}": + from => $allow_clients, + port => $port, + proto => $proto, } } } From 30e863920752dddf322b5c44d061f409a1f08d1b Mon Sep 17 00:00:00 2001 From: Patrik Lundin Date: Tue, 24 Sep 2024 12:19:33 +0200 Subject: [PATCH 068/161] Upgrade knubbis-fleetlock to v0.0.18 --- manifests/knubbis/fleetlock_standalone.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/knubbis/fleetlock_standalone.pp b/manifests/knubbis/fleetlock_standalone.pp index 951efe882..1b722ca0a 100644 --- a/manifests/knubbis/fleetlock_standalone.pp +++ b/manifests/knubbis/fleetlock_standalone.pp @@ -15,7 +15,7 @@ # @param domain The domain where the fleetlock server will supply its services # @param letsencrypt_prod Should the server request real letsencrypt certificates class sunet::knubbis::fleetlock_standalone( - String $knubbis_fleetlock_version="v0.0.17", + String $knubbis_fleetlock_version="v0.0.18", String $etcd_version="v3.5.8", String $cfssl_helper_version="v0.0.1", String $etcdctl_helper_version="v0.0.1", From 833155f4fdc9fa5c7e6b5d8804b949304e2bb6f5 Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Tue, 1 Oct 2024 17:13:13 +0200 Subject: [PATCH 069/161] Make it possible to specify image and tag --- manifests/redictcluster.pp | 13 ++++++++----- manifests/rediscluster.pp | 2 ++ templates/redictcluster/docker-compose.yml.erb | 2 +- templates/rediscluster/docker-compose.yml.erb | 2 +- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/manifests/redictcluster.pp b/manifests/redictcluster.pp index ddb6cd444..8a4c794f2 100644 --- a/manifests/redictcluster.pp +++ b/manifests/redictcluster.pp @@ -6,6 +6,9 @@ Optional[String] $cluster_announce_ip = '', Optional[Boolean] $automatic_rectify = false, Optional[Boolean] $prevent_reboot = false, + Optional[String] $image = 'registry.redict.io/redict', + Optional[String] $tag = '7-bookworm', + ) { @@ -17,9 +20,9 @@ } # Allow the user to use the explicit string ipaddress or ipaddress6 to use the corresponding facts if $__cluster_announce_ip == 'ipaddress' { - $_cluster_announce_ip = $facts['ipaddress'] + $_cluster_announce_ip = $facts['networking']['ip'] } elsif $__cluster_announce_ip == 'ipaddress6' { - $_cluster_announce_ip = $facts['ipaddress6'] + $_cluster_announce_ip = $facts['networking']['ip6'] } else { $_cluster_announce_ip = $__cluster_announce_ip } @@ -66,9 +69,9 @@ $redictportnum = 6379 + $i file { "/opt/redict/node-${i}": - ensure => directory, - owner => '999', - group => '999', + ensure => directory, + owner => '999', + group => '999', } -> file { "/opt/redict/node-${i}/server.conf": ensure => present, diff --git a/manifests/rediscluster.pp b/manifests/rediscluster.pp index 7bf7ea78b..c0352c298 100644 --- a/manifests/rediscluster.pp +++ b/manifests/rediscluster.pp @@ -6,6 +6,8 @@ Optional[String] $cluster_announce_ip = '', Optional[Boolean] $automatic_rectify = false, Optional[Boolean] $prevent_reboot = false, + Optional[String] $image = 'redis', + Optional[String] $tag = '7-bookworm', ) { diff --git a/templates/redictcluster/docker-compose.yml.erb b/templates/redictcluster/docker-compose.yml.erb index 4420df2b2..93e895873 100644 --- a/templates/redictcluster/docker-compose.yml.erb +++ b/templates/redictcluster/docker-compose.yml.erb @@ -7,7 +7,7 @@ services: <% joinport = 16379 + i %> redict-node-<%= i %>: container_name: redict-node-<%= i %> - image: registry.redict.io/redict:7-bookworm + image: <%= @image %>:<%= @tag %> dns: - 89.46.20.75 - 89.46.21.29 diff --git a/templates/rediscluster/docker-compose.yml.erb b/templates/rediscluster/docker-compose.yml.erb index 6110c0b21..07b510a88 100644 --- a/templates/rediscluster/docker-compose.yml.erb +++ b/templates/rediscluster/docker-compose.yml.erb @@ -7,7 +7,7 @@ services: <% joinport = 16379 + i %> redis-node-<%= i %>: container_name: redis-node-<%= i %> - image: redis:7-bookworm + image: <%= @image %>:<%= @tag %> dns: - 89.46.20.75 - 89.46.21.29 From c727ed938362f5fc3793fbbbd4afa69c7f6311e9 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Tue, 17 Sep 2024 13:54:53 +0200 Subject: [PATCH 070/161] Do we need a define? --- manifests/mariadb.pp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/manifests/mariadb.pp b/manifests/mariadb.pp index b96f305ed..009c95a7a 100644 --- a/manifests/mariadb.pp +++ b/manifests/mariadb.pp @@ -1,9 +1,9 @@ # Mariadb cluster class for SUNET -define sunet::mariadb( - String $mariadb_version=latest, - Integer $bootstrap=0, - Array[Integer] $ports = [3306, 4444, 4567, 4568], - Array[String] $dns = [], +class sunet::mariadb( + $mariadb_version=latest, + $bootstrap=0, + $ports = [3306, 4444, 4567, 4568], + $dns = undef, ) { $mariadb_root_password = lookup('mariadb_root_password', undef, undef,'NOT_SET_IN_HIERA') From bce376338fb9b98ddb44d52123cab5fbdaf3dc08 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Tue, 17 Sep 2024 14:04:55 +0200 Subject: [PATCH 071/161] Define for transition towards class? --- manifests/mariadb.pp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/manifests/mariadb.pp b/manifests/mariadb.pp index 009c95a7a..07fcb4027 100644 --- a/manifests/mariadb.pp +++ b/manifests/mariadb.pp @@ -1,3 +1,10 @@ +# Mariadb cluster definefor SUNET +define sunet::mariadb(){ + + require sunet::mariadb + +} + # Mariadb cluster class for SUNET class sunet::mariadb( $mariadb_version=latest, From be456944c10054e6068b42ee10ccb6c1282e4d0a Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Tue, 17 Sep 2024 14:07:38 +0200 Subject: [PATCH 072/161] Name collition --- manifests/mariadb.pp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/manifests/mariadb.pp b/manifests/mariadb.pp index 07fcb4027..55498490f 100644 --- a/manifests/mariadb.pp +++ b/manifests/mariadb.pp @@ -1,12 +1,12 @@ # Mariadb cluster definefor SUNET define sunet::mariadb(){ - require sunet::mariadb + require sunet::mariadb::server } # Mariadb cluster class for SUNET -class sunet::mariadb( +class sunet::mariadb::server( $mariadb_version=latest, $bootstrap=0, $ports = [3306, 4444, 4567, 4568], From cf4e79ad6fc5abecefaa3c87b4b56f847d4ed389 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Tue, 17 Sep 2024 14:11:27 +0200 Subject: [PATCH 073/161] Show deprecation warnings --- manifests/mariadb.pp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/manifests/mariadb.pp b/manifests/mariadb.pp index 55498490f..d727c33f1 100644 --- a/manifests/mariadb.pp +++ b/manifests/mariadb.pp @@ -1,8 +1,7 @@ # Mariadb cluster definefor SUNET define sunet::mariadb(){ - + warning('Please transition to the class "sunet::mariadb::server" instead of this define') require sunet::mariadb::server - } # Mariadb cluster class for SUNET From fdde3d917b2352fe0736dcec3bed8a459453258a Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Tue, 17 Sep 2024 14:39:10 +0200 Subject: [PATCH 074/161] Puppet recommends split out files --- manifests/mariadb/init.pp | 5 +++++ manifests/{mariadb.pp => mariadb/server.pp} | 6 ------ 2 files changed, 5 insertions(+), 6 deletions(-) create mode 100644 manifests/mariadb/init.pp rename manifests/{mariadb.pp => mariadb/server.pp} (95%) diff --git a/manifests/mariadb/init.pp b/manifests/mariadb/init.pp new file mode 100644 index 000000000..170bb4fe1 --- /dev/null +++ b/manifests/mariadb/init.pp @@ -0,0 +1,5 @@ +# Mariadb cluster definefor SUNET +define sunet::mariadb(){ + warning('Please transition to the class "sunet::mariadb::server" instead of this define') + require sunet::mariadb::server +} diff --git a/manifests/mariadb.pp b/manifests/mariadb/server.pp similarity index 95% rename from manifests/mariadb.pp rename to manifests/mariadb/server.pp index d727c33f1..370a74329 100644 --- a/manifests/mariadb.pp +++ b/manifests/mariadb/server.pp @@ -1,9 +1,3 @@ -# Mariadb cluster definefor SUNET -define sunet::mariadb(){ - warning('Please transition to the class "sunet::mariadb::server" instead of this define') - require sunet::mariadb::server -} - # Mariadb cluster class for SUNET class sunet::mariadb::server( $mariadb_version=latest, From 8888cbef724db5ee1364cb51ec6539974263d887 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Thu, 19 Sep 2024 12:26:21 +0200 Subject: [PATCH 075/161] Make it possible to run multiple instances At the same time add simple class good for most. --- manifests/mariadb/init.pp | 110 ++++++++++++++++++++++++++++++++++-- manifests/mariadb/server.pp | 107 ----------------------------------- 2 files changed, 106 insertions(+), 111 deletions(-) delete mode 100644 manifests/mariadb/server.pp diff --git a/manifests/mariadb/init.pp b/manifests/mariadb/init.pp index 170bb4fe1..e6993a469 100644 --- a/manifests/mariadb/init.pp +++ b/manifests/mariadb/init.pp @@ -1,5 +1,107 @@ -# Mariadb cluster definefor SUNET -define sunet::mariadb(){ - warning('Please transition to the class "sunet::mariadb::server" instead of this define') - require sunet::mariadb::server +# Mariadb cluster class for SUNET +define sunet::mariadb( + $mariadb_version=latest, + $bootstrap=0, + $ports = [3306, 4444, 4567, 4568], + $dns = undef, +) +{ + $mariadb_root_password = lookup('mariadb_root_password', undef, undef,'NOT_SET_IN_HIERA') + $mariadb_user = lookup('mariadb_user', undef, undef,undef) + $mariadb_user_password = lookup('mariadb_user_password', undef, undef,undef) + $mariadb_database = lookup('mariadb_database', undef, undef,undef) + $mariadb_backup_password = lookup('mariadb_root_password', undef, undef,'NOT_SET_IN_HIERA') + $clients = lookup('mariadb_clients', undef, undef,['127.0.0.1']) + $cluster_nodes = lookup('mariadb_cluster_nodes', undef, undef,[]) + $mariadb_dir = '/opt/mariadb' + $server_id = 1000 + Integer($facts['networking']['hostname'][-1]) + + # Hack to not clash with docker_compose which tries to create the same directory + exec {'mariadb_dir_create': + command => "mkdir -p ${mariadb_dir}", + unless => "test -d ${mariadb_dir}", + } + + $dirs = ['datadir', 'init', 'conf', 'backups', 'scripts' ] + $dirs.each |$dir| { + ensure_resource('file',"${mariadb_dir}/${dir}", { ensure => directory, recurse => true } ) + } + + $_from = $clients + $cluster_nodes + sunet::misc::ufw_allow { 'mariadb_ports': + from => $_from, + port => $ports, + } + + file { '/usr/local/bin/purge-binlogs': + ensure => present, + content => template('sunet/mariadb/purge-binlogs.erb.sh'), + mode => '0744', + owner => 999, + group => 999, + } + file { '/usr/local/bin/run_manual_backup_dump': + ensure => present, + content => template('sunet/mariadb/run_manual_backup_dump.erb.sh'), + mode => '0744', + owner => 999, + group => 999, + } + sunet::scriptherder::cronjob { 'purge_binlogs': + cmd => '/usr/local/bin/purge-binlogs', + hour => '6', + minute => '0', + ok_criteria => ['exit_status=0','max_age=2d'], + warn_criteria => ['exit_status=1','max_age=3d'], + } + file { '/usr/local/bin/cluster-size': + ensure => present, + content => template('sunet/mariadb/cluster-size.erb.sh'), + mode => '0744', + } + file { '/usr/local/bin/cluster-status': + ensure => present, + content => template('sunet/mariadb/cluster-status.erb.sh'), + mode => '0744', + } + file { '/etc/sudoers.d/99-size-test': + ensure => file, + content => "script ALL=(root) NOPASSWD: /usr/local/bin/cluster-size\n", + mode => '0440', + owner => 'root', + group => 'root', + } + file { '/etc/sudoers.d/99-status-test': + ensure => file, + content => "script ALL=(root) NOPASSWD: /usr/local/bin/cluster-status\n", + mode => '0440', + owner => 'root', + group => 'root', + } + + $sql_files = ['02-backup_user.sql'] + $sql_files.each |$sql_file|{ + file { "${mariadb_dir}/init/${sql_file}": + ensure => present, + content => template("sunet/mariadb/${sql_file}.erb"), + mode => '0744', + } + } + file { "${mariadb_dir}/conf/credentials.cnf": + ensure => present, + content => template('sunet/mariadb/credentials.cnf.erb'), + mode => '0744', + } + file { "${mariadb_dir}/conf/my.cnf": + ensure => present, + content => template('sunet/mariadb/my.cnf.erb'), + mode => '0744', + } + $docker_compose = sunet::docker_compose { 'sunet_mariadb_docker_compose': + content => template('sunet/mariadb/docker-compose_mariadb.yml.erb'), + service_name => 'mariadb', + compose_dir => '/opt/', + compose_filename => 'docker-compose.yml', + description => 'Mariadb server', + } } diff --git a/manifests/mariadb/server.pp b/manifests/mariadb/server.pp deleted file mode 100644 index 370a74329..000000000 --- a/manifests/mariadb/server.pp +++ /dev/null @@ -1,107 +0,0 @@ -# Mariadb cluster class for SUNET -class sunet::mariadb::server( - $mariadb_version=latest, - $bootstrap=0, - $ports = [3306, 4444, 4567, 4568], - $dns = undef, -) -{ - $mariadb_root_password = lookup('mariadb_root_password', undef, undef,'NOT_SET_IN_HIERA') - $mariadb_user = lookup('mariadb_user', undef, undef,undef) - $mariadb_user_password = lookup('mariadb_user_password', undef, undef,undef) - $mariadb_database = lookup('mariadb_database', undef, undef,undef) - $mariadb_backup_password = lookup('mariadb_root_password', undef, undef,'NOT_SET_IN_HIERA') - $clients = lookup('mariadb_clients', undef, undef,['127.0.0.1']) - $cluster_nodes = lookup('mariadb_cluster_nodes', undef, undef,[]) - $mariadb_dir = '/opt/mariadb' - $server_id = 1000 + Integer($facts['networking']['hostname'][-1]) - - # Hack to not clash with docker_compose which tries to create the same directory - exec {'mariadb_dir_create': - command => "mkdir -p ${mariadb_dir}", - unless => "test -d ${mariadb_dir}", - } - - $dirs = ['datadir', 'init', 'conf', 'backups', 'scripts' ] - $dirs.each |$dir| { - ensure_resource('file',"${mariadb_dir}/${dir}", { ensure => directory, recurse => true } ) - } - - $_from = $clients + $cluster_nodes - sunet::misc::ufw_allow { 'mariadb_ports': - from => $_from, - port => $ports, - } - - file { '/usr/local/bin/purge-binlogs': - ensure => present, - content => template('sunet/mariadb/purge-binlogs.erb.sh'), - mode => '0744', - owner => 999, - group => 999, - } - file { '/usr/local/bin/run_manual_backup_dump': - ensure => present, - content => template('sunet/mariadb/run_manual_backup_dump.erb.sh'), - mode => '0744', - owner => 999, - group => 999, - } - sunet::scriptherder::cronjob { 'purge_binlogs': - cmd => '/usr/local/bin/purge-binlogs', - hour => '6', - minute => '0', - ok_criteria => ['exit_status=0','max_age=2d'], - warn_criteria => ['exit_status=1','max_age=3d'], - } - file { '/usr/local/bin/cluster-size': - ensure => present, - content => template('sunet/mariadb/cluster-size.erb.sh'), - mode => '0744', - } - file { '/usr/local/bin/cluster-status': - ensure => present, - content => template('sunet/mariadb/cluster-status.erb.sh'), - mode => '0744', - } - file { '/etc/sudoers.d/99-size-test': - ensure => file, - content => "script ALL=(root) NOPASSWD: /usr/local/bin/cluster-size\n", - mode => '0440', - owner => 'root', - group => 'root', - } - file { '/etc/sudoers.d/99-status-test': - ensure => file, - content => "script ALL=(root) NOPASSWD: /usr/local/bin/cluster-status\n", - mode => '0440', - owner => 'root', - group => 'root', - } - - $sql_files = ['02-backup_user.sql'] - $sql_files.each |$sql_file|{ - file { "${mariadb_dir}/init/${sql_file}": - ensure => present, - content => template("sunet/mariadb/${sql_file}.erb"), - mode => '0744', - } - } - file { "${mariadb_dir}/conf/credentials.cnf": - ensure => present, - content => template('sunet/mariadb/credentials.cnf.erb'), - mode => '0744', - } - file { "${mariadb_dir}/conf/my.cnf": - ensure => present, - content => template('sunet/mariadb/my.cnf.erb'), - mode => '0744', - } - $docker_compose = sunet::docker_compose { 'sunet_mariadb_docker_compose': - content => template('sunet/mariadb/docker-compose_mariadb.yml.erb'), - service_name => 'mariadb', - compose_dir => '/opt/', - compose_filename => 'docker-compose.yml', - description => 'Mariadb server', - } -} From 74455e4da4f87a4d0568a314330b15744741db2c Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Thu, 19 Sep 2024 12:44:45 +0200 Subject: [PATCH 076/161] Defines can't be in init.pp --- manifests/{mariadb/init.pp => mariadb.pp} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename manifests/{mariadb/init.pp => mariadb.pp} (100%) diff --git a/manifests/mariadb/init.pp b/manifests/mariadb.pp similarity index 100% rename from manifests/mariadb/init.pp rename to manifests/mariadb.pp From e74c329a85f7e3a53615e17aa3e5e55d22b2aefa Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Thu, 19 Sep 2024 14:21:48 +0200 Subject: [PATCH 077/161] Typing --- manifests/mariadb.pp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/manifests/mariadb.pp b/manifests/mariadb.pp index e6993a469..b96f305ed 100644 --- a/manifests/mariadb.pp +++ b/manifests/mariadb.pp @@ -1,9 +1,9 @@ # Mariadb cluster class for SUNET define sunet::mariadb( - $mariadb_version=latest, - $bootstrap=0, - $ports = [3306, 4444, 4567, 4568], - $dns = undef, + String $mariadb_version=latest, + Integer $bootstrap=0, + Array[Integer] $ports = [3306, 4444, 4567, 4568], + Array[String] $dns = [], ) { $mariadb_root_password = lookup('mariadb_root_password', undef, undef,'NOT_SET_IN_HIERA') From 9c4c08db10d1991fec45c4267f5668ed8b2f76d4 Mon Sep 17 00:00:00 2001 From: Maria Haider Date: Thu, 29 Aug 2024 13:05:14 +0200 Subject: [PATCH 078/161] open just allowed ports --- manifests/nftables/docker_expose.pp | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/manifests/nftables/docker_expose.pp b/manifests/nftables/docker_expose.pp index 429a39f07..75d56bd35 100644 --- a/manifests/nftables/docker_expose.pp +++ b/manifests/nftables/docker_expose.pp @@ -52,10 +52,18 @@ notify => Service['nftables'], ; } - sunet::nftables::allow { "expose-allow-${safe_name}": - from => $allow_clients, - port => $port, - proto => $proto, + if ($allow_clients =~ Array[String, 1]) or ($allow_clients =~ String[1]) { + sunet::nftables::allow { "expose-allow-${safe_name}": + from => $allow_clients, + port => $port, + proto => $proto, + } + } else { + sunet::nftables::allow { "expose-allow-${safe_name}": + from => any, + port => $port, + proto => $proto, + } } } } From f06be146a8cb88d0fa227d691f90567284a502be Mon Sep 17 00:00:00 2001 From: Maria Haider Date: Fri, 30 Aug 2024 13:43:42 +0200 Subject: [PATCH 079/161] if clause not needed --- manifests/nftables/docker_expose.pp | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/manifests/nftables/docker_expose.pp b/manifests/nftables/docker_expose.pp index 75d56bd35..429a39f07 100644 --- a/manifests/nftables/docker_expose.pp +++ b/manifests/nftables/docker_expose.pp @@ -52,18 +52,10 @@ notify => Service['nftables'], ; } - if ($allow_clients =~ Array[String, 1]) or ($allow_clients =~ String[1]) { - sunet::nftables::allow { "expose-allow-${safe_name}": - from => $allow_clients, - port => $port, - proto => $proto, - } - } else { - sunet::nftables::allow { "expose-allow-${safe_name}": - from => any, - port => $port, - proto => $proto, - } + sunet::nftables::allow { "expose-allow-${safe_name}": + from => $allow_clients, + port => $port, + proto => $proto, } } } From 77e857eaee341d06c5b7d87057525e1e980e18f5 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Fri, 20 Sep 2024 15:11:34 +0200 Subject: [PATCH 080/161] Initial attempt of tab completion for scriptherder --- .../scriptherder-tab-completion.sh | 43 +++++++++++++++++++ manifests/scriptherder/init.pp | 4 ++ 2 files changed, 47 insertions(+) create mode 100644 files/scriptherder/scriptherder-tab-completion.sh diff --git a/files/scriptherder/scriptherder-tab-completion.sh b/files/scriptherder/scriptherder-tab-completion.sh new file mode 100644 index 000000000..3344a3ed3 --- /dev/null +++ b/files/scriptherder/scriptherder-tab-completion.sh @@ -0,0 +1,43 @@ +# shellcheck disable=SC2148 +_scriptherder() { + local cur prev OPTS + + COMPREPLY=() + cur="${COMP_WORDS[COMP_CWORD]}" + prev="${COMP_WORDS[COMP_CWORD - 1]}" + + case $prev in + 'check' | 'ls' | 'lastlog' | 'lastfaillog') + local JOBS + # shellcheck disable=SC2086 + JOBS="$(basename -s '.ini' /etc/scriptherder/check/${2}*)" + mapfile -t COMPREPLY < <(compgen -W "${JOBS}" -- "${cur}") + return 0 + ;; + 'wrap') + # not supported + return 0 + ;; + + # Allow access to cur below + 'scriptherder') + true + ;; + *) + return 0 + ;; + esac + + case $cur in + *) + OPTS="check + ls + lastlog + lastfaillog + wrap" + mapfile -t COMPREPLY < <(compgen -W "${OPTS[*]}" -- "${cur}") + return 0 + ;; + esac +} +complete -F _scriptherder scriptherder diff --git a/manifests/scriptherder/init.pp b/manifests/scriptherder/init.pp index c0c373135..aeb2ff71e 100644 --- a/manifests/scriptherder/init.pp +++ b/manifests/scriptherder/init.pp @@ -22,6 +22,10 @@ mode => '0755', source => 'puppet:///modules/sunet/scriptherder/scriptherder.py', } + file { '/etc/bash_completion.d/scriptherder-tab-completion.sh': + mode => '0755', + source => 'puppet:///modules/sunet/scriptherder/scriptherder-tab-completion.sh', + } } } if $nrpe { From f9a24cac8f140b0e5c086a3cbff3b9660116f44a Mon Sep 17 00:00:00 2001 From: Patrik Lundin Date: Thu, 3 Oct 2024 12:57:24 +0200 Subject: [PATCH 081/161] baas2: Same encoding string everywhere While "utf8" works it seems python docs in general talks about "utf-8" and it is what we are using in all other places in the code so make this the same also. --- files/baas2/sunet-baas2-bootstrap | 2 +- files/baas2/sunet-baas2-tbmr-bootstrap | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/files/baas2/sunet-baas2-bootstrap b/files/baas2/sunet-baas2-bootstrap index 9b49f57ad..864eb449e 100755 --- a/files/baas2/sunet-baas2-bootstrap +++ b/files/baas2/sunet-baas2-bootstrap @@ -80,7 +80,7 @@ def get_installed_version() -> Union[str, None]: check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - encoding="utf8", + encoding="utf-8", ) except subprocess.CalledProcessError as exc: # If the package is not installed this is OK, we will install it diff --git a/files/baas2/sunet-baas2-tbmr-bootstrap b/files/baas2/sunet-baas2-tbmr-bootstrap index e125c40d5..e74bf171f 100755 --- a/files/baas2/sunet-baas2-tbmr-bootstrap +++ b/files/baas2/sunet-baas2-tbmr-bootstrap @@ -52,7 +52,7 @@ def get_installed_version() -> Union[str, None]: check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - encoding="utf8", + encoding="utf-8", ) except subprocess.CalledProcessError as exc: # If the package is not installed this is OK, we will install it From 11dc3874ecdb2b4fa14c37531ff4f9a5a4a9381e Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Tue, 1 Oct 2024 13:55:46 +0200 Subject: [PATCH 082/161] Monitor GPG keys for cosmos --- files/nagios/check_gpg_keys.sh | 101 ++++++++++++++++++++ manifests/naemon_monitor.pp | 7 ++ manifests/nagios/nrpe_check_gpg_keys_bin.pp | 10 ++ 3 files changed, 118 insertions(+) create mode 100755 files/nagios/check_gpg_keys.sh create mode 100644 manifests/nagios/nrpe_check_gpg_keys_bin.pp diff --git a/files/nagios/check_gpg_keys.sh b/files/nagios/check_gpg_keys.sh new file mode 100755 index 000000000..bcfc3a7df --- /dev/null +++ b/files/nagios/check_gpg_keys.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash + +set -o pipefail + +if [ -z "$1" ]; then + echo "UNKOWN: A directory is required as \$1" + exit 3 +fi + +DIRECTORY="$1" + +if [ ! -d "${DIRECTORY}" ]; then + echo "UNKOWN: Unknown directory (${DIRECTORY})" + exit 3 +fi + +WARNING=$(date --date="+ 14 days" +%s) +CRITICAL=$(date --date="+ 7 days" +%s) + +CRIT=0 +WARN=0 + +EXPIRING=() +INVALIDS=() + +NUM_KEYS=0 +INFINITE_KEYS=0 + +PREFIX="OK" +EXIT=0 + +for key in "${DIRECTORY}"/*.pub; do + ((NUM_KEYS++)) + + if ! expirey_date=$(gpg --fixed-list-mode --with-colons --show-keys "${key}" 2>/dev/null | grep -e '^pub:' | cut -d : -f 7); then + INVALIDS+=("${key}") + continue + fi + if [ "$(echo "${expirey_date}" | wc -l)" -ne 1 ]; then + INVALIDS+=("${key}") + continue + fi + + if [ -z "${expirey_date}" ]; then + ((INFINITE_KEYS++)) + continue + elif ! echo "${expirey_date}" | grep -qP '^\d{10}$'; then + echo "Warning: Can't parse ${key} for validity" + ((WARN++)) + INVALIDS+=("${key}") + continue + fi + + if [ "${expirey_date}" -lt "${CRITICAL}" ]; then + ((CRIT++)) + EXPIRING+=("${key}") + elif [ "${expirey_date}" -lt "${WARNING}" ]; then + ((WARN++)) + EXPIRING+=("${key}") + fi +done + +NUM_EXPIRING=${#EXPIRING[@]} +NUM_INVALID=${#INVALIDS[@]} + +if [ $CRIT -ne 0 ]; then + PREFIX="CRITICAL" + EXIT=1 +elif [ $WARN -ne 0 ]; then + PREFIX="WARNING" + EXIT=2 +elif [ "$NUM_INVALID" -ne 0 ]; then + PREFIX="WARNING" + EXIT=2 +fi + +NON_OK_STRING=() +if [ "${NUM_EXPIRING}" -ne 0 ]; then + NON_OK_STRING+=("Expiring/expired gpg keys (${NUM_EXPIRING}): ${EXPIRING[*]}") +fi + +if [ "${NUM_INVALID}" -ne 0 ]; then + NON_OK_STRING+=("Invalid gpg keys (${NUM_INVALID}): ${INVALIDS[*]}") +fi + +NON_OK_OUTPUT="" +for string in "${NON_OK_STRING[@]}"; do + if [ -z "$NON_OK_OUTPUT" ]; then + NON_OK_OUTPUT="${string}" + else + NON_OK_OUTPUT="${NON_OK_OUTPUT}, ${string}" + fi +done + +OUTPUT_STRING="No gpg keys are about to expire" +if [ -n "${NON_OK_OUTPUT}" ]; then + OUTPUT_STRING=${NON_OK_OUTPUT} +fi + +echo "${PREFIX}: ${OUTPUT_STRING} | expiring_keys=${NUM_EXPIRING} invalid_keys=${NUM_INVALID} infinite_keys=${INFINITE_KEYS} total_keys=${NUM_KEYS}" +exit "${EXIT}" diff --git a/manifests/naemon_monitor.pp b/manifests/naemon_monitor.pp index 7849af96c..df687334a 100644 --- a/manifests/naemon_monitor.pp +++ b/manifests/naemon_monitor.pp @@ -413,6 +413,13 @@ } } + require sunet::nagios::nrpe_check_cosmos_keys + nagioscfg::service {'check_cosmos_keys': + hostgroup_name => ['sunet::naemon_monitor'], + check_command => 'check_nrpe!check_cosmos_keys', + description => 'GPG keys used by cosmos', + } + file { '/etc/naemon/conf.d/cosmos/naemon-hostgroups.cfg': ensure => file, mode => '0644', diff --git a/manifests/nagios/nrpe_check_gpg_keys_bin.pp b/manifests/nagios/nrpe_check_gpg_keys_bin.pp new file mode 100644 index 000000000..dfeec5a26 --- /dev/null +++ b/manifests/nagios/nrpe_check_gpg_keys_bin.pp @@ -0,0 +1,10 @@ +# Binary for check_gpg_keys +class sunet::nagios::nrpe_check_gpg_keys_bin ( +) { + file { '/usr/lib/nagios/plugins/check_gpg_keys': + ensure => 'file', + mode => '0755', + owner => 'root', + content => file('sunet/nagios/check_gpg_keys.sh') + } +} From 8fd85f22c36006bfca37befdb61ae017a8c9a98b Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Tue, 1 Oct 2024 13:59:36 +0200 Subject: [PATCH 083/161] Forgot to add the cosmos check --- manifests/nagios/nrpe_check_cosmos_keys.pp | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 manifests/nagios/nrpe_check_cosmos_keys.pp diff --git a/manifests/nagios/nrpe_check_cosmos_keys.pp b/manifests/nagios/nrpe_check_cosmos_keys.pp new file mode 100644 index 000000000..3508e1900 --- /dev/null +++ b/manifests/nagios/nrpe_check_cosmos_keys.pp @@ -0,0 +1,10 @@ +# Check gpg keys used by cosmos +define sunet::nagios::nrpe_check_cosmos_keys ( +) { + + include sunet::nagios::nrpe_check_gpg_keys_bin + + sunet::nagios::nrpe_command { 'check_cosmos_keys': + command_line => '/usr/lib/nagios/plugins/check_gpg_keys /etc/cosmos/keys', + } +} From 021f041261aaaa51d43dd0b9dd707c3f0affa652 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Tue, 1 Oct 2024 14:02:20 +0200 Subject: [PATCH 084/161] Is class better for requirements? --- manifests/nagios/nrpe_check_cosmos_keys.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/nagios/nrpe_check_cosmos_keys.pp b/manifests/nagios/nrpe_check_cosmos_keys.pp index 3508e1900..0bb2dd9cf 100644 --- a/manifests/nagios/nrpe_check_cosmos_keys.pp +++ b/manifests/nagios/nrpe_check_cosmos_keys.pp @@ -1,5 +1,5 @@ # Check gpg keys used by cosmos -define sunet::nagios::nrpe_check_cosmos_keys ( +class sunet::nagios::nrpe_check_cosmos_keys ( ) { include sunet::nagios::nrpe_check_gpg_keys_bin From 827e19fc2decfd5fbcdfb4b0c1151dee6ae4ccbe Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Tue, 1 Oct 2024 14:06:15 +0200 Subject: [PATCH 085/161] Flipped values around --- files/nagios/check_gpg_keys.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/files/nagios/check_gpg_keys.sh b/files/nagios/check_gpg_keys.sh index bcfc3a7df..3e23d121f 100755 --- a/files/nagios/check_gpg_keys.sh +++ b/files/nagios/check_gpg_keys.sh @@ -65,13 +65,13 @@ NUM_INVALID=${#INVALIDS[@]} if [ $CRIT -ne 0 ]; then PREFIX="CRITICAL" - EXIT=1 + EXIT=2 elif [ $WARN -ne 0 ]; then PREFIX="WARNING" - EXIT=2 + EXIT=1 elif [ "$NUM_INVALID" -ne 0 ]; then PREFIX="WARNING" - EXIT=2 + EXIT=1 fi NON_OK_STRING=() From a168e041898f2f27a70ef0e5aebfdb386c314e2c Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Tue, 1 Oct 2024 14:14:41 +0200 Subject: [PATCH 086/161] Improve naming --- manifests/naemon_monitor.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/naemon_monitor.pp b/manifests/naemon_monitor.pp index df687334a..c96ed4910 100644 --- a/manifests/naemon_monitor.pp +++ b/manifests/naemon_monitor.pp @@ -417,7 +417,7 @@ nagioscfg::service {'check_cosmos_keys': hostgroup_name => ['sunet::naemon_monitor'], check_command => 'check_nrpe!check_cosmos_keys', - description => 'GPG keys used by cosmos', + description => 'Cosmos GPG keys', } file { '/etc/naemon/conf.d/cosmos/naemon-hostgroups.cfg': From d770493a133806705ce86324019bebd32bd19df7 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Tue, 1 Oct 2024 14:40:59 +0200 Subject: [PATCH 087/161] Monitor keys that signs our metadata --- manifests/metadata/metadata_repo.pp | 6 ++++++ manifests/naemon_monitor.pp | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/manifests/metadata/metadata_repo.pp b/manifests/metadata/metadata_repo.pp index 56484bf2d..a600bc7b0 100644 --- a/manifests/metadata/metadata_repo.pp +++ b/manifests/metadata/metadata_repo.pp @@ -35,6 +35,12 @@ ok_criteria => ['exit_status=0', 'max_age=15m'], warn_criteria => ['exit_status=0', 'max_age=1h'], } + + include sunet::nagios::nrpe_check_gpg_keys_bin + + sunet::nagios::nrpe_command { 'check_cosmos_keys': + command_line => "/usr/lib/nagios/plugins/check_gpg_keys ${cache_dir}/keys", + } } } else { vcsrepo { '/opt/metadata': diff --git a/manifests/naemon_monitor.pp b/manifests/naemon_monitor.pp index c96ed4910..a0a49f249 100644 --- a/manifests/naemon_monitor.pp +++ b/manifests/naemon_monitor.pp @@ -420,6 +420,12 @@ description => 'Cosmos GPG keys', } + nagioscfg::service {'check_metadata_keys': + hostgroup_name => ['sunet::metadata::metadata_repo'], + check_command => 'check_nrpe!check_metadata_keys', + description => 'Metadata GPG keys', + } + file { '/etc/naemon/conf.d/cosmos/naemon-hostgroups.cfg': ensure => file, mode => '0644', From e7595a6ece72ca4b8a064f231c0dd698d0abc389 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Tue, 1 Oct 2024 15:03:01 +0200 Subject: [PATCH 088/161] Wrong name --- manifests/metadata/metadata_repo.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/metadata/metadata_repo.pp b/manifests/metadata/metadata_repo.pp index a600bc7b0..5b31b9ce8 100644 --- a/manifests/metadata/metadata_repo.pp +++ b/manifests/metadata/metadata_repo.pp @@ -38,7 +38,7 @@ include sunet::nagios::nrpe_check_gpg_keys_bin - sunet::nagios::nrpe_command { 'check_cosmos_keys': + sunet::nagios::nrpe_command { 'check_metadata_keys': command_line => "/usr/lib/nagios/plugins/check_gpg_keys ${cache_dir}/keys", } } From 5bc27fb9f6309cefdb3c8aae73091a2e8c21b677 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Tue, 1 Oct 2024 15:40:23 +0200 Subject: [PATCH 089/161] Wrong location --- manifests/metadata/metadata_repo.pp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/manifests/metadata/metadata_repo.pp b/manifests/metadata/metadata_repo.pp index 5b31b9ce8..3a9a649b4 100644 --- a/manifests/metadata/metadata_repo.pp +++ b/manifests/metadata/metadata_repo.pp @@ -28,6 +28,12 @@ } -> package { ['make','gnupg2']: ensure => latest } + + include sunet::nagios::nrpe_check_gpg_keys_bin + sunet::nagios::nrpe_command { 'check_metadata_keys': + command_line => "/usr/lib/nagios/plugins/check_gpg_keys ${cache_dir}/keys", + } + if $update_by_cron { sunet::scriptherder::cronjob { 'verify_and_update': cmd => "${cache_dir}/scripts/do-update.sh", @@ -36,11 +42,6 @@ warn_criteria => ['exit_status=0', 'max_age=1h'], } - include sunet::nagios::nrpe_check_gpg_keys_bin - - sunet::nagios::nrpe_command { 'check_metadata_keys': - command_line => "/usr/lib/nagios/plugins/check_gpg_keys ${cache_dir}/keys", - } } } else { vcsrepo { '/opt/metadata': From 5d05b38b111607f21ada83c32fe8ed4b3c109567 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Tue, 1 Oct 2024 15:45:06 +0200 Subject: [PATCH 090/161] Linting --- files/nagios/check_gpg_keys.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/files/nagios/check_gpg_keys.sh b/files/nagios/check_gpg_keys.sh index 3e23d121f..9691fc74b 100755 --- a/files/nagios/check_gpg_keys.sh +++ b/files/nagios/check_gpg_keys.sh @@ -63,10 +63,10 @@ done NUM_EXPIRING=${#EXPIRING[@]} NUM_INVALID=${#INVALIDS[@]} -if [ $CRIT -ne 0 ]; then +if [ "${CRIT}" -ne 0 ]; then PREFIX="CRITICAL" EXIT=2 -elif [ $WARN -ne 0 ]; then +elif [ "${WARN}" -ne 0 ]; then PREFIX="WARNING" EXIT=1 elif [ "$NUM_INVALID" -ne 0 ]; then From 88ec46635fd3b3a052e5392e7e7bce4ebe7b0b78 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Thu, 3 Oct 2024 11:38:30 +0200 Subject: [PATCH 091/161] Enable `set -u` --- files/nagios/check_gpg_keys.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/files/nagios/check_gpg_keys.sh b/files/nagios/check_gpg_keys.sh index 9691fc74b..e91c98567 100755 --- a/files/nagios/check_gpg_keys.sh +++ b/files/nagios/check_gpg_keys.sh @@ -1,13 +1,14 @@ #!/usr/bin/env bash -set -o pipefail +set -uo pipefail -if [ -z "$1" ]; then - echo "UNKOWN: A directory is required as \$1" - exit 3 +if [ $# -ne 1 ]; then + echo "UNKOWN: A directory is required as \$1" + exit 3 fi -DIRECTORY="$1" +args=("$@") +DIRECTORY=${args[0]} if [ ! -d "${DIRECTORY}" ]; then echo "UNKOWN: Unknown directory (${DIRECTORY})" From 9f0c7664aca68e36144aa8b1414573c41047e7d5 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Thu, 3 Oct 2024 11:43:55 +0200 Subject: [PATCH 092/161] Only allow one public key per file --- files/nagios/check_gpg_keys.sh | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/files/nagios/check_gpg_keys.sh b/files/nagios/check_gpg_keys.sh index e91c98567..85f0a4071 100755 --- a/files/nagios/check_gpg_keys.sh +++ b/files/nagios/check_gpg_keys.sh @@ -33,15 +33,19 @@ EXIT=0 for key in "${DIRECTORY}"/*.pub; do ((NUM_KEYS++)) - if ! expirey_date=$(gpg --fixed-list-mode --with-colons --show-keys "${key}" 2>/dev/null | grep -e '^pub:' | cut -d : -f 7); then - INVALIDS+=("${key}") - continue - fi - if [ "$(echo "${expirey_date}" | wc -l)" -ne 1 ]; then + if ! pub_keys=$(gpg --fixed-list-mode --with-colons --show-keys "${key}" 2>/dev/null | grep -e '^pub:'); then INVALIDS+=("${key}") continue fi + # Only allow one public key per file + if [ "$(echo "${pub_keys}" | wc -l)" -ne 1 ]; then + INVALIDS+=("${key}") + continue + fi + + expirey_date=$(echo "${pub_keys}" | cut -d : -f 7) + if [ -z "${expirey_date}" ]; then ((INFINITE_KEYS++)) continue From a5159e0cc617ad165e5a474199a4ddf3007d06ce Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Thu, 3 Oct 2024 11:56:37 +0200 Subject: [PATCH 093/161] Warn if a key lacks expiration --- files/nagios/check_gpg_keys.sh | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/files/nagios/check_gpg_keys.sh b/files/nagios/check_gpg_keys.sh index 85f0a4071..d767ecc38 100755 --- a/files/nagios/check_gpg_keys.sh +++ b/files/nagios/check_gpg_keys.sh @@ -23,9 +23,9 @@ WARN=0 EXPIRING=() INVALIDS=() +INFINITIVES=() NUM_KEYS=0 -INFINITE_KEYS=0 PREFIX="OK" EXIT=0 @@ -47,7 +47,8 @@ for key in "${DIRECTORY}"/*.pub; do expirey_date=$(echo "${pub_keys}" | cut -d : -f 7) if [ -z "${expirey_date}" ]; then - ((INFINITE_KEYS++)) + INFINITIVES+=("${key}") + ((WARN++)) continue elif ! echo "${expirey_date}" | grep -qP '^\d{10}$'; then echo "Warning: Can't parse ${key} for validity" @@ -67,6 +68,7 @@ done NUM_EXPIRING=${#EXPIRING[@]} NUM_INVALID=${#INVALIDS[@]} +NUM_INFINITIVE=${#INFINITIVES[@]} if [ "${CRIT}" -ne 0 ]; then PREFIX="CRITICAL" @@ -88,6 +90,10 @@ if [ "${NUM_INVALID}" -ne 0 ]; then NON_OK_STRING+=("Invalid gpg keys (${NUM_INVALID}): ${INVALIDS[*]}") fi +if [ "${NUM_INFINITIVE}" -ne 0 ]; then + NON_OK_STRING+=("GPG keys without expiration (${NUM_INVALID}): ${INFINITIVES[*]}") +fi + NON_OK_OUTPUT="" for string in "${NON_OK_STRING[@]}"; do if [ -z "$NON_OK_OUTPUT" ]; then @@ -97,10 +103,10 @@ for string in "${NON_OK_STRING[@]}"; do fi done -OUTPUT_STRING="No gpg keys are about to expire" +OUTPUT_STRING="No GPG keys are about to expire" if [ -n "${NON_OK_OUTPUT}" ]; then OUTPUT_STRING=${NON_OK_OUTPUT} fi -echo "${PREFIX}: ${OUTPUT_STRING} | expiring_keys=${NUM_EXPIRING} invalid_keys=${NUM_INVALID} infinite_keys=${INFINITE_KEYS} total_keys=${NUM_KEYS}" +echo "${PREFIX}: ${OUTPUT_STRING} | expiring_keys=${NUM_EXPIRING} invalid_keys=${NUM_INVALID} infinite_keys=${NUM_INFINITIVE} total_keys=${NUM_KEYS}" exit "${EXIT}" From 1160fd3b48e8382ab54bd62d2f7c9496f93d2b47 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Thu, 3 Oct 2024 12:21:02 +0200 Subject: [PATCH 094/161] Enforce an uniformed filename --- files/nagios/check_gpg_keys.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/files/nagios/check_gpg_keys.sh b/files/nagios/check_gpg_keys.sh index d767ecc38..b154f7636 100755 --- a/files/nagios/check_gpg_keys.sh +++ b/files/nagios/check_gpg_keys.sh @@ -57,6 +57,15 @@ for key in "${DIRECTORY}"/*.pub; do continue fi + fingerprint=$(gpg --fixed-list-mode --with-colons --show-keys "${key}" 2>/dev/null | grep -A1 -e '^pub:' | grep -e '^fpr' |cut -d : -f 10) + filename=$(basename "${key}") + # Only allow files with the long fingerprint as suffix. E.g + # jocar-13376BF892B5871181A218E9BE4EC2EEADF2C31B.pub + if ! echo "${filename}" | grep -qP "^[^-]*-${fingerprint}.pub$"; then + ((WARN++)) + INVALIDS+=("${key}") + fi + if [ "${expirey_date}" -lt "${CRITICAL}" ]; then ((CRIT++)) EXPIRING+=("${key}") From 8930e37c67b9d1b06b10e42c156e2d4058d26e99 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Thu, 3 Oct 2024 12:23:22 +0200 Subject: [PATCH 095/161] Uniformed use of GPG --- files/nagios/check_gpg_keys.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/files/nagios/check_gpg_keys.sh b/files/nagios/check_gpg_keys.sh index b154f7636..6fe9e163b 100755 --- a/files/nagios/check_gpg_keys.sh +++ b/files/nagios/check_gpg_keys.sh @@ -92,11 +92,11 @@ fi NON_OK_STRING=() if [ "${NUM_EXPIRING}" -ne 0 ]; then - NON_OK_STRING+=("Expiring/expired gpg keys (${NUM_EXPIRING}): ${EXPIRING[*]}") + NON_OK_STRING+=("Expiring/expired GPG keys (${NUM_EXPIRING}): ${EXPIRING[*]}") fi if [ "${NUM_INVALID}" -ne 0 ]; then - NON_OK_STRING+=("Invalid gpg keys (${NUM_INVALID}): ${INVALIDS[*]}") + NON_OK_STRING+=("Invalid GPG keys (${NUM_INVALID}): ${INVALIDS[*]}") fi if [ "${NUM_INFINITIVE}" -ne 0 ]; then From e7521a916f52b31a9f7b34455c7bb3b19eb9c087 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Thu, 3 Oct 2024 12:30:01 +0200 Subject: [PATCH 096/161] Don't spread GNUPGHOME around --- files/nagios/check_gpg_keys.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/files/nagios/check_gpg_keys.sh b/files/nagios/check_gpg_keys.sh index 6fe9e163b..7c55328f3 100755 --- a/files/nagios/check_gpg_keys.sh +++ b/files/nagios/check_gpg_keys.sh @@ -30,6 +30,9 @@ NUM_KEYS=0 PREFIX="OK" EXIT=0 +TMPDIR=$(mktemp -d) +export GNUPGHOME="${TMPDIR}" + for key in "${DIRECTORY}"/*.pub; do ((NUM_KEYS++)) @@ -117,5 +120,7 @@ if [ -n "${NON_OK_OUTPUT}" ]; then OUTPUT_STRING=${NON_OK_OUTPUT} fi +rm -rf "${TMPDIR}" + echo "${PREFIX}: ${OUTPUT_STRING} | expiring_keys=${NUM_EXPIRING} invalid_keys=${NUM_INVALID} infinite_keys=${NUM_INFINITIVE} total_keys=${NUM_KEYS}" exit "${EXIT}" From 6953ae4c18cb70180f7b54065cf3e32357bf14bf Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Thu, 3 Oct 2024 13:01:00 +0200 Subject: [PATCH 097/161] Reformat --- files/nagios/check_gpg_keys.sh | 42 +++++++++++++++++----------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/files/nagios/check_gpg_keys.sh b/files/nagios/check_gpg_keys.sh index 7c55328f3..f53bdec51 100755 --- a/files/nagios/check_gpg_keys.sh +++ b/files/nagios/check_gpg_keys.sh @@ -3,8 +3,8 @@ set -uo pipefail if [ $# -ne 1 ]; then - echo "UNKOWN: A directory is required as \$1" - exit 3 + echo "UNKOWN: A directory is required as \$1" + exit 3 fi args=("$@") @@ -34,46 +34,46 @@ TMPDIR=$(mktemp -d) export GNUPGHOME="${TMPDIR}" for key in "${DIRECTORY}"/*.pub; do - ((NUM_KEYS++)) + ((NUM_KEYS++)) - if ! pub_keys=$(gpg --fixed-list-mode --with-colons --show-keys "${key}" 2>/dev/null | grep -e '^pub:'); then + if ! pub_keys=$(gpg --fixed-list-mode --with-colons --show-keys "${key}" 2>/dev/null | grep -e '^pub:'); then INVALIDS+=("${key}") continue fi - # Only allow one public key per file - if [ "$(echo "${pub_keys}" | wc -l)" -ne 1 ]; then - INVALIDS+=("${key}") - continue - fi + # Only allow one public key per file + if [ "$(echo "${pub_keys}" | wc -l)" -ne 1 ]; then + INVALIDS+=("${key}") + continue + fi - expirey_date=$(echo "${pub_keys}" | cut -d : -f 7) + expirey_date=$(echo "${pub_keys}" | cut -d : -f 7) if [ -z "${expirey_date}" ]; then INFINITIVES+=("${key}") - ((WARN++)) + ((WARN++)) continue elif ! echo "${expirey_date}" | grep -qP '^\d{10}$'; then echo "Warning: Can't parse ${key} for validity" - ((WARN++)) + ((WARN++)) INVALIDS+=("${key}") continue fi - fingerprint=$(gpg --fixed-list-mode --with-colons --show-keys "${key}" 2>/dev/null | grep -A1 -e '^pub:' | grep -e '^fpr' |cut -d : -f 10) - filename=$(basename "${key}") - # Only allow files with the long fingerprint as suffix. E.g - # jocar-13376BF892B5871181A218E9BE4EC2EEADF2C31B.pub - if ! echo "${filename}" | grep -qP "^[^-]*-${fingerprint}.pub$"; then - ((WARN++)) + fingerprint=$(gpg --fixed-list-mode --with-colons --show-keys "${key}" 2>/dev/null | grep -A1 -e '^pub:' | grep -e '^fpr' | cut -d : -f 10) + filename=$(basename "${key}") + # Only allow files with the long fingerprint as suffix. E.g + # jocar-13376BF892B5871181A218E9BE4EC2EEADF2C31B.pub + if ! echo "${filename}" | grep -qP "^[^-]*-${fingerprint}.pub$"; then + ((WARN++)) INVALIDS+=("${key}") - fi + fi if [ "${expirey_date}" -lt "${CRITICAL}" ]; then - ((CRIT++)) + ((CRIT++)) EXPIRING+=("${key}") elif [ "${expirey_date}" -lt "${WARNING}" ]; then - ((WARN++)) + ((WARN++)) EXPIRING+=("${key}") fi done From aca3a529e075be3c9fbd2ca157358a19d24b27d9 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Thu, 3 Oct 2024 14:47:04 +0200 Subject: [PATCH 098/161] Count correct keys --- files/nagios/check_gpg_keys.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/files/nagios/check_gpg_keys.sh b/files/nagios/check_gpg_keys.sh index f53bdec51..1d47e58d8 100755 --- a/files/nagios/check_gpg_keys.sh +++ b/files/nagios/check_gpg_keys.sh @@ -103,7 +103,7 @@ if [ "${NUM_INVALID}" -ne 0 ]; then fi if [ "${NUM_INFINITIVE}" -ne 0 ]; then - NON_OK_STRING+=("GPG keys without expiration (${NUM_INVALID}): ${INFINITIVES[*]}") + NON_OK_STRING+=("GPG keys without expiration (${NUM_INFINITIVE}): ${INFINITIVES[*]}") fi NON_OK_OUTPUT="" From 6fc1b003ee230ee154619c1c18a7566a266f287f Mon Sep 17 00:00:00 2001 From: Maria Haider Date: Thu, 3 Oct 2024 23:20:22 +0200 Subject: [PATCH 099/161] updated invent script --- templates/invent/invent.sh.erb | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/templates/invent/invent.sh.erb b/templates/invent/invent.sh.erb index b84aeda53..48544b3cc 100644 --- a/templates/invent/invent.sh.erb +++ b/templates/invent/invent.sh.erb @@ -44,9 +44,8 @@ if [ $(which docker) ]; then # Gather structured data docker fact docker_fact="${fact_dir}/docker_ps.json" for container in $(docker ps -q); do - docker ps --format '{{json . }}' --filter "id=${container}" | \ - jq '. |= . + '{"ImageId":$(docker inspect --format '{{json .Image }}' ${container})'}' - done | jq --jsonargs '{"docker_ps":[inputs]}' > ${docker_fact} + docker ps --format '{{json . }}' --filter "id=${container}" | jq '. |= . + '{"ImageId":$(docker inspect --format '{{json .Image }}' ${container})'}'; + done | jq -s |jq -s '{docker_ps: add}' > ${docker_fact} fi From f03d398a47ca90e396dad30e51b5f6e1cb9410dd Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Tue, 17 Sep 2024 13:54:53 +0200 Subject: [PATCH 100/161] Do we need a define? --- manifests/mariadb.pp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/manifests/mariadb.pp b/manifests/mariadb.pp index b96f305ed..009c95a7a 100644 --- a/manifests/mariadb.pp +++ b/manifests/mariadb.pp @@ -1,9 +1,9 @@ # Mariadb cluster class for SUNET -define sunet::mariadb( - String $mariadb_version=latest, - Integer $bootstrap=0, - Array[Integer] $ports = [3306, 4444, 4567, 4568], - Array[String] $dns = [], +class sunet::mariadb( + $mariadb_version=latest, + $bootstrap=0, + $ports = [3306, 4444, 4567, 4568], + $dns = undef, ) { $mariadb_root_password = lookup('mariadb_root_password', undef, undef,'NOT_SET_IN_HIERA') From 131600a31e9bef954d4344afdb3b620ab2690484 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Tue, 17 Sep 2024 14:04:55 +0200 Subject: [PATCH 101/161] Define for transition towards class? --- manifests/mariadb.pp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/manifests/mariadb.pp b/manifests/mariadb.pp index 009c95a7a..07fcb4027 100644 --- a/manifests/mariadb.pp +++ b/manifests/mariadb.pp @@ -1,3 +1,10 @@ +# Mariadb cluster definefor SUNET +define sunet::mariadb(){ + + require sunet::mariadb + +} + # Mariadb cluster class for SUNET class sunet::mariadb( $mariadb_version=latest, From 175bfb4e85d4a3b244a042c3aa25266b1c1909e6 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Tue, 17 Sep 2024 14:07:38 +0200 Subject: [PATCH 102/161] Name collition --- manifests/mariadb.pp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/manifests/mariadb.pp b/manifests/mariadb.pp index 07fcb4027..55498490f 100644 --- a/manifests/mariadb.pp +++ b/manifests/mariadb.pp @@ -1,12 +1,12 @@ # Mariadb cluster definefor SUNET define sunet::mariadb(){ - require sunet::mariadb + require sunet::mariadb::server } # Mariadb cluster class for SUNET -class sunet::mariadb( +class sunet::mariadb::server( $mariadb_version=latest, $bootstrap=0, $ports = [3306, 4444, 4567, 4568], From 6e2ae7e84300350948a60c06cc372ebeb6c2b293 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Tue, 17 Sep 2024 14:11:27 +0200 Subject: [PATCH 103/161] Show deprecation warnings --- manifests/mariadb.pp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/manifests/mariadb.pp b/manifests/mariadb.pp index 55498490f..d727c33f1 100644 --- a/manifests/mariadb.pp +++ b/manifests/mariadb.pp @@ -1,8 +1,7 @@ # Mariadb cluster definefor SUNET define sunet::mariadb(){ - + warning('Please transition to the class "sunet::mariadb::server" instead of this define') require sunet::mariadb::server - } # Mariadb cluster class for SUNET From 0ff33dc82c9c3796b67e20b05c9a9e7a74883e72 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Tue, 17 Sep 2024 14:39:10 +0200 Subject: [PATCH 104/161] Puppet recommends split out files --- manifests/mariadb/init.pp | 5 +++++ manifests/{mariadb.pp => mariadb/server.pp} | 6 ------ 2 files changed, 5 insertions(+), 6 deletions(-) create mode 100644 manifests/mariadb/init.pp rename manifests/{mariadb.pp => mariadb/server.pp} (95%) diff --git a/manifests/mariadb/init.pp b/manifests/mariadb/init.pp new file mode 100644 index 000000000..170bb4fe1 --- /dev/null +++ b/manifests/mariadb/init.pp @@ -0,0 +1,5 @@ +# Mariadb cluster definefor SUNET +define sunet::mariadb(){ + warning('Please transition to the class "sunet::mariadb::server" instead of this define') + require sunet::mariadb::server +} diff --git a/manifests/mariadb.pp b/manifests/mariadb/server.pp similarity index 95% rename from manifests/mariadb.pp rename to manifests/mariadb/server.pp index d727c33f1..370a74329 100644 --- a/manifests/mariadb.pp +++ b/manifests/mariadb/server.pp @@ -1,9 +1,3 @@ -# Mariadb cluster definefor SUNET -define sunet::mariadb(){ - warning('Please transition to the class "sunet::mariadb::server" instead of this define') - require sunet::mariadb::server -} - # Mariadb cluster class for SUNET class sunet::mariadb::server( $mariadb_version=latest, From 2a982a77251a4657d72f1aed13dd021fac565ad7 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Thu, 19 Sep 2024 12:26:21 +0200 Subject: [PATCH 105/161] Make it possible to run multiple instances At the same time add simple class good for most. --- manifests/mariadb/init.pp | 110 ++++++++++++++++++++++++++++++++++-- manifests/mariadb/server.pp | 107 ----------------------------------- 2 files changed, 106 insertions(+), 111 deletions(-) delete mode 100644 manifests/mariadb/server.pp diff --git a/manifests/mariadb/init.pp b/manifests/mariadb/init.pp index 170bb4fe1..e6993a469 100644 --- a/manifests/mariadb/init.pp +++ b/manifests/mariadb/init.pp @@ -1,5 +1,107 @@ -# Mariadb cluster definefor SUNET -define sunet::mariadb(){ - warning('Please transition to the class "sunet::mariadb::server" instead of this define') - require sunet::mariadb::server +# Mariadb cluster class for SUNET +define sunet::mariadb( + $mariadb_version=latest, + $bootstrap=0, + $ports = [3306, 4444, 4567, 4568], + $dns = undef, +) +{ + $mariadb_root_password = lookup('mariadb_root_password', undef, undef,'NOT_SET_IN_HIERA') + $mariadb_user = lookup('mariadb_user', undef, undef,undef) + $mariadb_user_password = lookup('mariadb_user_password', undef, undef,undef) + $mariadb_database = lookup('mariadb_database', undef, undef,undef) + $mariadb_backup_password = lookup('mariadb_root_password', undef, undef,'NOT_SET_IN_HIERA') + $clients = lookup('mariadb_clients', undef, undef,['127.0.0.1']) + $cluster_nodes = lookup('mariadb_cluster_nodes', undef, undef,[]) + $mariadb_dir = '/opt/mariadb' + $server_id = 1000 + Integer($facts['networking']['hostname'][-1]) + + # Hack to not clash with docker_compose which tries to create the same directory + exec {'mariadb_dir_create': + command => "mkdir -p ${mariadb_dir}", + unless => "test -d ${mariadb_dir}", + } + + $dirs = ['datadir', 'init', 'conf', 'backups', 'scripts' ] + $dirs.each |$dir| { + ensure_resource('file',"${mariadb_dir}/${dir}", { ensure => directory, recurse => true } ) + } + + $_from = $clients + $cluster_nodes + sunet::misc::ufw_allow { 'mariadb_ports': + from => $_from, + port => $ports, + } + + file { '/usr/local/bin/purge-binlogs': + ensure => present, + content => template('sunet/mariadb/purge-binlogs.erb.sh'), + mode => '0744', + owner => 999, + group => 999, + } + file { '/usr/local/bin/run_manual_backup_dump': + ensure => present, + content => template('sunet/mariadb/run_manual_backup_dump.erb.sh'), + mode => '0744', + owner => 999, + group => 999, + } + sunet::scriptherder::cronjob { 'purge_binlogs': + cmd => '/usr/local/bin/purge-binlogs', + hour => '6', + minute => '0', + ok_criteria => ['exit_status=0','max_age=2d'], + warn_criteria => ['exit_status=1','max_age=3d'], + } + file { '/usr/local/bin/cluster-size': + ensure => present, + content => template('sunet/mariadb/cluster-size.erb.sh'), + mode => '0744', + } + file { '/usr/local/bin/cluster-status': + ensure => present, + content => template('sunet/mariadb/cluster-status.erb.sh'), + mode => '0744', + } + file { '/etc/sudoers.d/99-size-test': + ensure => file, + content => "script ALL=(root) NOPASSWD: /usr/local/bin/cluster-size\n", + mode => '0440', + owner => 'root', + group => 'root', + } + file { '/etc/sudoers.d/99-status-test': + ensure => file, + content => "script ALL=(root) NOPASSWD: /usr/local/bin/cluster-status\n", + mode => '0440', + owner => 'root', + group => 'root', + } + + $sql_files = ['02-backup_user.sql'] + $sql_files.each |$sql_file|{ + file { "${mariadb_dir}/init/${sql_file}": + ensure => present, + content => template("sunet/mariadb/${sql_file}.erb"), + mode => '0744', + } + } + file { "${mariadb_dir}/conf/credentials.cnf": + ensure => present, + content => template('sunet/mariadb/credentials.cnf.erb'), + mode => '0744', + } + file { "${mariadb_dir}/conf/my.cnf": + ensure => present, + content => template('sunet/mariadb/my.cnf.erb'), + mode => '0744', + } + $docker_compose = sunet::docker_compose { 'sunet_mariadb_docker_compose': + content => template('sunet/mariadb/docker-compose_mariadb.yml.erb'), + service_name => 'mariadb', + compose_dir => '/opt/', + compose_filename => 'docker-compose.yml', + description => 'Mariadb server', + } } diff --git a/manifests/mariadb/server.pp b/manifests/mariadb/server.pp deleted file mode 100644 index 370a74329..000000000 --- a/manifests/mariadb/server.pp +++ /dev/null @@ -1,107 +0,0 @@ -# Mariadb cluster class for SUNET -class sunet::mariadb::server( - $mariadb_version=latest, - $bootstrap=0, - $ports = [3306, 4444, 4567, 4568], - $dns = undef, -) -{ - $mariadb_root_password = lookup('mariadb_root_password', undef, undef,'NOT_SET_IN_HIERA') - $mariadb_user = lookup('mariadb_user', undef, undef,undef) - $mariadb_user_password = lookup('mariadb_user_password', undef, undef,undef) - $mariadb_database = lookup('mariadb_database', undef, undef,undef) - $mariadb_backup_password = lookup('mariadb_root_password', undef, undef,'NOT_SET_IN_HIERA') - $clients = lookup('mariadb_clients', undef, undef,['127.0.0.1']) - $cluster_nodes = lookup('mariadb_cluster_nodes', undef, undef,[]) - $mariadb_dir = '/opt/mariadb' - $server_id = 1000 + Integer($facts['networking']['hostname'][-1]) - - # Hack to not clash with docker_compose which tries to create the same directory - exec {'mariadb_dir_create': - command => "mkdir -p ${mariadb_dir}", - unless => "test -d ${mariadb_dir}", - } - - $dirs = ['datadir', 'init', 'conf', 'backups', 'scripts' ] - $dirs.each |$dir| { - ensure_resource('file',"${mariadb_dir}/${dir}", { ensure => directory, recurse => true } ) - } - - $_from = $clients + $cluster_nodes - sunet::misc::ufw_allow { 'mariadb_ports': - from => $_from, - port => $ports, - } - - file { '/usr/local/bin/purge-binlogs': - ensure => present, - content => template('sunet/mariadb/purge-binlogs.erb.sh'), - mode => '0744', - owner => 999, - group => 999, - } - file { '/usr/local/bin/run_manual_backup_dump': - ensure => present, - content => template('sunet/mariadb/run_manual_backup_dump.erb.sh'), - mode => '0744', - owner => 999, - group => 999, - } - sunet::scriptherder::cronjob { 'purge_binlogs': - cmd => '/usr/local/bin/purge-binlogs', - hour => '6', - minute => '0', - ok_criteria => ['exit_status=0','max_age=2d'], - warn_criteria => ['exit_status=1','max_age=3d'], - } - file { '/usr/local/bin/cluster-size': - ensure => present, - content => template('sunet/mariadb/cluster-size.erb.sh'), - mode => '0744', - } - file { '/usr/local/bin/cluster-status': - ensure => present, - content => template('sunet/mariadb/cluster-status.erb.sh'), - mode => '0744', - } - file { '/etc/sudoers.d/99-size-test': - ensure => file, - content => "script ALL=(root) NOPASSWD: /usr/local/bin/cluster-size\n", - mode => '0440', - owner => 'root', - group => 'root', - } - file { '/etc/sudoers.d/99-status-test': - ensure => file, - content => "script ALL=(root) NOPASSWD: /usr/local/bin/cluster-status\n", - mode => '0440', - owner => 'root', - group => 'root', - } - - $sql_files = ['02-backup_user.sql'] - $sql_files.each |$sql_file|{ - file { "${mariadb_dir}/init/${sql_file}": - ensure => present, - content => template("sunet/mariadb/${sql_file}.erb"), - mode => '0744', - } - } - file { "${mariadb_dir}/conf/credentials.cnf": - ensure => present, - content => template('sunet/mariadb/credentials.cnf.erb'), - mode => '0744', - } - file { "${mariadb_dir}/conf/my.cnf": - ensure => present, - content => template('sunet/mariadb/my.cnf.erb'), - mode => '0744', - } - $docker_compose = sunet::docker_compose { 'sunet_mariadb_docker_compose': - content => template('sunet/mariadb/docker-compose_mariadb.yml.erb'), - service_name => 'mariadb', - compose_dir => '/opt/', - compose_filename => 'docker-compose.yml', - description => 'Mariadb server', - } -} From 4d2f0ab42b995bc188f62c7e5e5573566899bbe1 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Thu, 19 Sep 2024 12:44:45 +0200 Subject: [PATCH 106/161] Defines can't be in init.pp --- manifests/{mariadb/init.pp => mariadb.pp} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename manifests/{mariadb/init.pp => mariadb.pp} (100%) diff --git a/manifests/mariadb/init.pp b/manifests/mariadb.pp similarity index 100% rename from manifests/mariadb/init.pp rename to manifests/mariadb.pp From 197b6feba727cfec7672b50f2709e164a83253d7 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Thu, 19 Sep 2024 14:21:48 +0200 Subject: [PATCH 107/161] Typing --- manifests/mariadb.pp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/manifests/mariadb.pp b/manifests/mariadb.pp index e6993a469..b96f305ed 100644 --- a/manifests/mariadb.pp +++ b/manifests/mariadb.pp @@ -1,9 +1,9 @@ # Mariadb cluster class for SUNET define sunet::mariadb( - $mariadb_version=latest, - $bootstrap=0, - $ports = [3306, 4444, 4567, 4568], - $dns = undef, + String $mariadb_version=latest, + Integer $bootstrap=0, + Array[Integer] $ports = [3306, 4444, 4567, 4568], + Array[String] $dns = [], ) { $mariadb_root_password = lookup('mariadb_root_password', undef, undef,'NOT_SET_IN_HIERA') From bcf88b68468c5166344215ac9994922f5813e458 Mon Sep 17 00:00:00 2001 From: Maria Haider Date: Thu, 29 Aug 2024 13:05:14 +0200 Subject: [PATCH 108/161] open just allowed ports --- manifests/nftables/docker_expose.pp | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/manifests/nftables/docker_expose.pp b/manifests/nftables/docker_expose.pp index 429a39f07..75d56bd35 100644 --- a/manifests/nftables/docker_expose.pp +++ b/manifests/nftables/docker_expose.pp @@ -52,10 +52,18 @@ notify => Service['nftables'], ; } - sunet::nftables::allow { "expose-allow-${safe_name}": - from => $allow_clients, - port => $port, - proto => $proto, + if ($allow_clients =~ Array[String, 1]) or ($allow_clients =~ String[1]) { + sunet::nftables::allow { "expose-allow-${safe_name}": + from => $allow_clients, + port => $port, + proto => $proto, + } + } else { + sunet::nftables::allow { "expose-allow-${safe_name}": + from => any, + port => $port, + proto => $proto, + } } } } From 743dfc172dd4f52f1bc31ba29e6fc8bcb23058aa Mon Sep 17 00:00:00 2001 From: Maria Haider Date: Fri, 30 Aug 2024 13:43:42 +0200 Subject: [PATCH 109/161] if clause not needed --- manifests/nftables/docker_expose.pp | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/manifests/nftables/docker_expose.pp b/manifests/nftables/docker_expose.pp index 75d56bd35..429a39f07 100644 --- a/manifests/nftables/docker_expose.pp +++ b/manifests/nftables/docker_expose.pp @@ -52,18 +52,10 @@ notify => Service['nftables'], ; } - if ($allow_clients =~ Array[String, 1]) or ($allow_clients =~ String[1]) { - sunet::nftables::allow { "expose-allow-${safe_name}": - from => $allow_clients, - port => $port, - proto => $proto, - } - } else { - sunet::nftables::allow { "expose-allow-${safe_name}": - from => any, - port => $port, - proto => $proto, - } + sunet::nftables::allow { "expose-allow-${safe_name}": + from => $allow_clients, + port => $port, + proto => $proto, } } } From 104c7d1f7ae59f18dec51089ec68b25b9aa939e9 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Tue, 1 Oct 2024 13:55:46 +0200 Subject: [PATCH 110/161] Monitor GPG keys for cosmos --- files/nagios/check_gpg_keys.sh | 65 +++++++++++----------------------- manifests/naemon_monitor.pp | 8 +---- 2 files changed, 21 insertions(+), 52 deletions(-) diff --git a/files/nagios/check_gpg_keys.sh b/files/nagios/check_gpg_keys.sh index 1d47e58d8..bcfc3a7df 100755 --- a/files/nagios/check_gpg_keys.sh +++ b/files/nagios/check_gpg_keys.sh @@ -1,14 +1,13 @@ #!/usr/bin/env bash -set -uo pipefail +set -o pipefail -if [ $# -ne 1 ]; then +if [ -z "$1" ]; then echo "UNKOWN: A directory is required as \$1" exit 3 fi -args=("$@") -DIRECTORY=${args[0]} +DIRECTORY="$1" if [ ! -d "${DIRECTORY}" ]; then echo "UNKOWN: Unknown directory (${DIRECTORY})" @@ -23,87 +22,65 @@ WARN=0 EXPIRING=() INVALIDS=() -INFINITIVES=() NUM_KEYS=0 +INFINITE_KEYS=0 PREFIX="OK" EXIT=0 -TMPDIR=$(mktemp -d) -export GNUPGHOME="${TMPDIR}" - for key in "${DIRECTORY}"/*.pub; do - ((NUM_KEYS++)) + ((NUM_KEYS++)) - if ! pub_keys=$(gpg --fixed-list-mode --with-colons --show-keys "${key}" 2>/dev/null | grep -e '^pub:'); then + if ! expirey_date=$(gpg --fixed-list-mode --with-colons --show-keys "${key}" 2>/dev/null | grep -e '^pub:' | cut -d : -f 7); then INVALIDS+=("${key}") continue fi - - # Only allow one public key per file - if [ "$(echo "${pub_keys}" | wc -l)" -ne 1 ]; then + if [ "$(echo "${expirey_date}" | wc -l)" -ne 1 ]; then INVALIDS+=("${key}") continue fi - expirey_date=$(echo "${pub_keys}" | cut -d : -f 7) - if [ -z "${expirey_date}" ]; then - INFINITIVES+=("${key}") - ((WARN++)) + ((INFINITE_KEYS++)) continue elif ! echo "${expirey_date}" | grep -qP '^\d{10}$'; then echo "Warning: Can't parse ${key} for validity" - ((WARN++)) + ((WARN++)) INVALIDS+=("${key}") continue fi - fingerprint=$(gpg --fixed-list-mode --with-colons --show-keys "${key}" 2>/dev/null | grep -A1 -e '^pub:' | grep -e '^fpr' | cut -d : -f 10) - filename=$(basename "${key}") - # Only allow files with the long fingerprint as suffix. E.g - # jocar-13376BF892B5871181A218E9BE4EC2EEADF2C31B.pub - if ! echo "${filename}" | grep -qP "^[^-]*-${fingerprint}.pub$"; then - ((WARN++)) - INVALIDS+=("${key}") - fi - if [ "${expirey_date}" -lt "${CRITICAL}" ]; then - ((CRIT++)) + ((CRIT++)) EXPIRING+=("${key}") elif [ "${expirey_date}" -lt "${WARNING}" ]; then - ((WARN++)) + ((WARN++)) EXPIRING+=("${key}") fi done NUM_EXPIRING=${#EXPIRING[@]} NUM_INVALID=${#INVALIDS[@]} -NUM_INFINITIVE=${#INFINITIVES[@]} -if [ "${CRIT}" -ne 0 ]; then +if [ $CRIT -ne 0 ]; then PREFIX="CRITICAL" - EXIT=2 -elif [ "${WARN}" -ne 0 ]; then - PREFIX="WARNING" EXIT=1 +elif [ $WARN -ne 0 ]; then + PREFIX="WARNING" + EXIT=2 elif [ "$NUM_INVALID" -ne 0 ]; then PREFIX="WARNING" - EXIT=1 + EXIT=2 fi NON_OK_STRING=() if [ "${NUM_EXPIRING}" -ne 0 ]; then - NON_OK_STRING+=("Expiring/expired GPG keys (${NUM_EXPIRING}): ${EXPIRING[*]}") + NON_OK_STRING+=("Expiring/expired gpg keys (${NUM_EXPIRING}): ${EXPIRING[*]}") fi if [ "${NUM_INVALID}" -ne 0 ]; then - NON_OK_STRING+=("Invalid GPG keys (${NUM_INVALID}): ${INVALIDS[*]}") -fi - -if [ "${NUM_INFINITIVE}" -ne 0 ]; then - NON_OK_STRING+=("GPG keys without expiration (${NUM_INFINITIVE}): ${INFINITIVES[*]}") + NON_OK_STRING+=("Invalid gpg keys (${NUM_INVALID}): ${INVALIDS[*]}") fi NON_OK_OUTPUT="" @@ -115,12 +92,10 @@ for string in "${NON_OK_STRING[@]}"; do fi done -OUTPUT_STRING="No GPG keys are about to expire" +OUTPUT_STRING="No gpg keys are about to expire" if [ -n "${NON_OK_OUTPUT}" ]; then OUTPUT_STRING=${NON_OK_OUTPUT} fi -rm -rf "${TMPDIR}" - -echo "${PREFIX}: ${OUTPUT_STRING} | expiring_keys=${NUM_EXPIRING} invalid_keys=${NUM_INVALID} infinite_keys=${NUM_INFINITIVE} total_keys=${NUM_KEYS}" +echo "${PREFIX}: ${OUTPUT_STRING} | expiring_keys=${NUM_EXPIRING} invalid_keys=${NUM_INVALID} infinite_keys=${INFINITE_KEYS} total_keys=${NUM_KEYS}" exit "${EXIT}" diff --git a/manifests/naemon_monitor.pp b/manifests/naemon_monitor.pp index a0a49f249..df687334a 100644 --- a/manifests/naemon_monitor.pp +++ b/manifests/naemon_monitor.pp @@ -417,13 +417,7 @@ nagioscfg::service {'check_cosmos_keys': hostgroup_name => ['sunet::naemon_monitor'], check_command => 'check_nrpe!check_cosmos_keys', - description => 'Cosmos GPG keys', - } - - nagioscfg::service {'check_metadata_keys': - hostgroup_name => ['sunet::metadata::metadata_repo'], - check_command => 'check_nrpe!check_metadata_keys', - description => 'Metadata GPG keys', + description => 'GPG keys used by cosmos', } file { '/etc/naemon/conf.d/cosmos/naemon-hostgroups.cfg': From 6b933767c3741f74042db949d2ad7a3164996d74 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Tue, 1 Oct 2024 13:59:36 +0200 Subject: [PATCH 111/161] Forgot to add the cosmos check --- manifests/nagios/nrpe_check_cosmos_keys.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/nagios/nrpe_check_cosmos_keys.pp b/manifests/nagios/nrpe_check_cosmos_keys.pp index 0bb2dd9cf..3508e1900 100644 --- a/manifests/nagios/nrpe_check_cosmos_keys.pp +++ b/manifests/nagios/nrpe_check_cosmos_keys.pp @@ -1,5 +1,5 @@ # Check gpg keys used by cosmos -class sunet::nagios::nrpe_check_cosmos_keys ( +define sunet::nagios::nrpe_check_cosmos_keys ( ) { include sunet::nagios::nrpe_check_gpg_keys_bin From dbf868db48bdde752811dbf9d414c37cd2d7f832 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Tue, 1 Oct 2024 14:02:20 +0200 Subject: [PATCH 112/161] Is class better for requirements? --- manifests/nagios/nrpe_check_cosmos_keys.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/nagios/nrpe_check_cosmos_keys.pp b/manifests/nagios/nrpe_check_cosmos_keys.pp index 3508e1900..0bb2dd9cf 100644 --- a/manifests/nagios/nrpe_check_cosmos_keys.pp +++ b/manifests/nagios/nrpe_check_cosmos_keys.pp @@ -1,5 +1,5 @@ # Check gpg keys used by cosmos -define sunet::nagios::nrpe_check_cosmos_keys ( +class sunet::nagios::nrpe_check_cosmos_keys ( ) { include sunet::nagios::nrpe_check_gpg_keys_bin From aebe825b31d4ed38e8f318de644e54c1aefaed5f Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Tue, 1 Oct 2024 14:06:15 +0200 Subject: [PATCH 113/161] Flipped values around --- files/nagios/check_gpg_keys.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/files/nagios/check_gpg_keys.sh b/files/nagios/check_gpg_keys.sh index bcfc3a7df..3e23d121f 100755 --- a/files/nagios/check_gpg_keys.sh +++ b/files/nagios/check_gpg_keys.sh @@ -65,13 +65,13 @@ NUM_INVALID=${#INVALIDS[@]} if [ $CRIT -ne 0 ]; then PREFIX="CRITICAL" - EXIT=1 + EXIT=2 elif [ $WARN -ne 0 ]; then PREFIX="WARNING" - EXIT=2 + EXIT=1 elif [ "$NUM_INVALID" -ne 0 ]; then PREFIX="WARNING" - EXIT=2 + EXIT=1 fi NON_OK_STRING=() From 9246490b7309fcda42a9bdb1362fa3c8e82e4826 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Tue, 1 Oct 2024 14:14:41 +0200 Subject: [PATCH 114/161] Improve naming --- manifests/naemon_monitor.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/naemon_monitor.pp b/manifests/naemon_monitor.pp index df687334a..c96ed4910 100644 --- a/manifests/naemon_monitor.pp +++ b/manifests/naemon_monitor.pp @@ -417,7 +417,7 @@ nagioscfg::service {'check_cosmos_keys': hostgroup_name => ['sunet::naemon_monitor'], check_command => 'check_nrpe!check_cosmos_keys', - description => 'GPG keys used by cosmos', + description => 'Cosmos GPG keys', } file { '/etc/naemon/conf.d/cosmos/naemon-hostgroups.cfg': From 2b9cd8b5b26f3d3492c2041c3bd5c3a61728bbb4 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Tue, 1 Oct 2024 14:40:59 +0200 Subject: [PATCH 115/161] Monitor keys that signs our metadata --- manifests/metadata/metadata_repo.pp | 5 +++++ manifests/naemon_monitor.pp | 6 ++++++ 2 files changed, 11 insertions(+) diff --git a/manifests/metadata/metadata_repo.pp b/manifests/metadata/metadata_repo.pp index 3a9a649b4..8fbb97f2c 100644 --- a/manifests/metadata/metadata_repo.pp +++ b/manifests/metadata/metadata_repo.pp @@ -42,6 +42,11 @@ warn_criteria => ['exit_status=0', 'max_age=1h'], } + include sunet::nagios::nrpe_check_gpg_keys_bin + + sunet::nagios::nrpe_command { 'check_cosmos_keys': + command_line => "/usr/lib/nagios/plugins/check_gpg_keys ${cache_dir}/keys", + } } } else { vcsrepo { '/opt/metadata': diff --git a/manifests/naemon_monitor.pp b/manifests/naemon_monitor.pp index c96ed4910..a0a49f249 100644 --- a/manifests/naemon_monitor.pp +++ b/manifests/naemon_monitor.pp @@ -420,6 +420,12 @@ description => 'Cosmos GPG keys', } + nagioscfg::service {'check_metadata_keys': + hostgroup_name => ['sunet::metadata::metadata_repo'], + check_command => 'check_nrpe!check_metadata_keys', + description => 'Metadata GPG keys', + } + file { '/etc/naemon/conf.d/cosmos/naemon-hostgroups.cfg': ensure => file, mode => '0644', From 80773669aef68454829b8bdb989d77508fc6704e Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Tue, 1 Oct 2024 15:03:01 +0200 Subject: [PATCH 116/161] Wrong name --- manifests/metadata/metadata_repo.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/metadata/metadata_repo.pp b/manifests/metadata/metadata_repo.pp index 8fbb97f2c..54728adc3 100644 --- a/manifests/metadata/metadata_repo.pp +++ b/manifests/metadata/metadata_repo.pp @@ -44,7 +44,7 @@ include sunet::nagios::nrpe_check_gpg_keys_bin - sunet::nagios::nrpe_command { 'check_cosmos_keys': + sunet::nagios::nrpe_command { 'check_metadata_keys': command_line => "/usr/lib/nagios/plugins/check_gpg_keys ${cache_dir}/keys", } } From cf6e9c105c1401cbfcfcc3a0a36f19d186ac7d2e Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Tue, 1 Oct 2024 15:40:23 +0200 Subject: [PATCH 117/161] Wrong location --- manifests/metadata/metadata_repo.pp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/manifests/metadata/metadata_repo.pp b/manifests/metadata/metadata_repo.pp index 54728adc3..3a9a649b4 100644 --- a/manifests/metadata/metadata_repo.pp +++ b/manifests/metadata/metadata_repo.pp @@ -42,11 +42,6 @@ warn_criteria => ['exit_status=0', 'max_age=1h'], } - include sunet::nagios::nrpe_check_gpg_keys_bin - - sunet::nagios::nrpe_command { 'check_metadata_keys': - command_line => "/usr/lib/nagios/plugins/check_gpg_keys ${cache_dir}/keys", - } } } else { vcsrepo { '/opt/metadata': From 6792b7b2a476a91a09ac15efc60ccc6b72ad12d4 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Tue, 1 Oct 2024 15:45:06 +0200 Subject: [PATCH 118/161] Linting --- files/nagios/check_gpg_keys.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/files/nagios/check_gpg_keys.sh b/files/nagios/check_gpg_keys.sh index 3e23d121f..9691fc74b 100755 --- a/files/nagios/check_gpg_keys.sh +++ b/files/nagios/check_gpg_keys.sh @@ -63,10 +63,10 @@ done NUM_EXPIRING=${#EXPIRING[@]} NUM_INVALID=${#INVALIDS[@]} -if [ $CRIT -ne 0 ]; then +if [ "${CRIT}" -ne 0 ]; then PREFIX="CRITICAL" EXIT=2 -elif [ $WARN -ne 0 ]; then +elif [ "${WARN}" -ne 0 ]; then PREFIX="WARNING" EXIT=1 elif [ "$NUM_INVALID" -ne 0 ]; then From a89d7e369aa511fb50315fa57559560cc048fba5 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Thu, 3 Oct 2024 11:38:30 +0200 Subject: [PATCH 119/161] Enable `set -u` --- files/nagios/check_gpg_keys.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/files/nagios/check_gpg_keys.sh b/files/nagios/check_gpg_keys.sh index 9691fc74b..e91c98567 100755 --- a/files/nagios/check_gpg_keys.sh +++ b/files/nagios/check_gpg_keys.sh @@ -1,13 +1,14 @@ #!/usr/bin/env bash -set -o pipefail +set -uo pipefail -if [ -z "$1" ]; then - echo "UNKOWN: A directory is required as \$1" - exit 3 +if [ $# -ne 1 ]; then + echo "UNKOWN: A directory is required as \$1" + exit 3 fi -DIRECTORY="$1" +args=("$@") +DIRECTORY=${args[0]} if [ ! -d "${DIRECTORY}" ]; then echo "UNKOWN: Unknown directory (${DIRECTORY})" From 9a74d9ead3f5128aaee981f9934408e57f8af82d Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Thu, 3 Oct 2024 11:43:55 +0200 Subject: [PATCH 120/161] Only allow one public key per file --- files/nagios/check_gpg_keys.sh | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/files/nagios/check_gpg_keys.sh b/files/nagios/check_gpg_keys.sh index e91c98567..85f0a4071 100755 --- a/files/nagios/check_gpg_keys.sh +++ b/files/nagios/check_gpg_keys.sh @@ -33,15 +33,19 @@ EXIT=0 for key in "${DIRECTORY}"/*.pub; do ((NUM_KEYS++)) - if ! expirey_date=$(gpg --fixed-list-mode --with-colons --show-keys "${key}" 2>/dev/null | grep -e '^pub:' | cut -d : -f 7); then - INVALIDS+=("${key}") - continue - fi - if [ "$(echo "${expirey_date}" | wc -l)" -ne 1 ]; then + if ! pub_keys=$(gpg --fixed-list-mode --with-colons --show-keys "${key}" 2>/dev/null | grep -e '^pub:'); then INVALIDS+=("${key}") continue fi + # Only allow one public key per file + if [ "$(echo "${pub_keys}" | wc -l)" -ne 1 ]; then + INVALIDS+=("${key}") + continue + fi + + expirey_date=$(echo "${pub_keys}" | cut -d : -f 7) + if [ -z "${expirey_date}" ]; then ((INFINITE_KEYS++)) continue From 2dc27d1109618b3ab18c116e50dc926c6781c1b1 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Thu, 3 Oct 2024 11:56:37 +0200 Subject: [PATCH 121/161] Warn if a key lacks expiration --- files/nagios/check_gpg_keys.sh | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/files/nagios/check_gpg_keys.sh b/files/nagios/check_gpg_keys.sh index 85f0a4071..d767ecc38 100755 --- a/files/nagios/check_gpg_keys.sh +++ b/files/nagios/check_gpg_keys.sh @@ -23,9 +23,9 @@ WARN=0 EXPIRING=() INVALIDS=() +INFINITIVES=() NUM_KEYS=0 -INFINITE_KEYS=0 PREFIX="OK" EXIT=0 @@ -47,7 +47,8 @@ for key in "${DIRECTORY}"/*.pub; do expirey_date=$(echo "${pub_keys}" | cut -d : -f 7) if [ -z "${expirey_date}" ]; then - ((INFINITE_KEYS++)) + INFINITIVES+=("${key}") + ((WARN++)) continue elif ! echo "${expirey_date}" | grep -qP '^\d{10}$'; then echo "Warning: Can't parse ${key} for validity" @@ -67,6 +68,7 @@ done NUM_EXPIRING=${#EXPIRING[@]} NUM_INVALID=${#INVALIDS[@]} +NUM_INFINITIVE=${#INFINITIVES[@]} if [ "${CRIT}" -ne 0 ]; then PREFIX="CRITICAL" @@ -88,6 +90,10 @@ if [ "${NUM_INVALID}" -ne 0 ]; then NON_OK_STRING+=("Invalid gpg keys (${NUM_INVALID}): ${INVALIDS[*]}") fi +if [ "${NUM_INFINITIVE}" -ne 0 ]; then + NON_OK_STRING+=("GPG keys without expiration (${NUM_INVALID}): ${INFINITIVES[*]}") +fi + NON_OK_OUTPUT="" for string in "${NON_OK_STRING[@]}"; do if [ -z "$NON_OK_OUTPUT" ]; then @@ -97,10 +103,10 @@ for string in "${NON_OK_STRING[@]}"; do fi done -OUTPUT_STRING="No gpg keys are about to expire" +OUTPUT_STRING="No GPG keys are about to expire" if [ -n "${NON_OK_OUTPUT}" ]; then OUTPUT_STRING=${NON_OK_OUTPUT} fi -echo "${PREFIX}: ${OUTPUT_STRING} | expiring_keys=${NUM_EXPIRING} invalid_keys=${NUM_INVALID} infinite_keys=${INFINITE_KEYS} total_keys=${NUM_KEYS}" +echo "${PREFIX}: ${OUTPUT_STRING} | expiring_keys=${NUM_EXPIRING} invalid_keys=${NUM_INVALID} infinite_keys=${NUM_INFINITIVE} total_keys=${NUM_KEYS}" exit "${EXIT}" From ab5fc36a236bda3eb4333bb0c82e1cf8272296da Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Thu, 3 Oct 2024 12:21:02 +0200 Subject: [PATCH 122/161] Enforce an uniformed filename --- files/nagios/check_gpg_keys.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/files/nagios/check_gpg_keys.sh b/files/nagios/check_gpg_keys.sh index d767ecc38..b154f7636 100755 --- a/files/nagios/check_gpg_keys.sh +++ b/files/nagios/check_gpg_keys.sh @@ -57,6 +57,15 @@ for key in "${DIRECTORY}"/*.pub; do continue fi + fingerprint=$(gpg --fixed-list-mode --with-colons --show-keys "${key}" 2>/dev/null | grep -A1 -e '^pub:' | grep -e '^fpr' |cut -d : -f 10) + filename=$(basename "${key}") + # Only allow files with the long fingerprint as suffix. E.g + # jocar-13376BF892B5871181A218E9BE4EC2EEADF2C31B.pub + if ! echo "${filename}" | grep -qP "^[^-]*-${fingerprint}.pub$"; then + ((WARN++)) + INVALIDS+=("${key}") + fi + if [ "${expirey_date}" -lt "${CRITICAL}" ]; then ((CRIT++)) EXPIRING+=("${key}") From 4a950b29d4012805aea7f3b3cdb7e951cc4971a1 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Thu, 3 Oct 2024 12:23:22 +0200 Subject: [PATCH 123/161] Uniformed use of GPG --- files/nagios/check_gpg_keys.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/files/nagios/check_gpg_keys.sh b/files/nagios/check_gpg_keys.sh index b154f7636..6fe9e163b 100755 --- a/files/nagios/check_gpg_keys.sh +++ b/files/nagios/check_gpg_keys.sh @@ -92,11 +92,11 @@ fi NON_OK_STRING=() if [ "${NUM_EXPIRING}" -ne 0 ]; then - NON_OK_STRING+=("Expiring/expired gpg keys (${NUM_EXPIRING}): ${EXPIRING[*]}") + NON_OK_STRING+=("Expiring/expired GPG keys (${NUM_EXPIRING}): ${EXPIRING[*]}") fi if [ "${NUM_INVALID}" -ne 0 ]; then - NON_OK_STRING+=("Invalid gpg keys (${NUM_INVALID}): ${INVALIDS[*]}") + NON_OK_STRING+=("Invalid GPG keys (${NUM_INVALID}): ${INVALIDS[*]}") fi if [ "${NUM_INFINITIVE}" -ne 0 ]; then From 6dc23a30af37cea4e58a2f2172dba537e16f6535 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Thu, 3 Oct 2024 12:30:01 +0200 Subject: [PATCH 124/161] Don't spread GNUPGHOME around --- files/nagios/check_gpg_keys.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/files/nagios/check_gpg_keys.sh b/files/nagios/check_gpg_keys.sh index 6fe9e163b..7c55328f3 100755 --- a/files/nagios/check_gpg_keys.sh +++ b/files/nagios/check_gpg_keys.sh @@ -30,6 +30,9 @@ NUM_KEYS=0 PREFIX="OK" EXIT=0 +TMPDIR=$(mktemp -d) +export GNUPGHOME="${TMPDIR}" + for key in "${DIRECTORY}"/*.pub; do ((NUM_KEYS++)) @@ -117,5 +120,7 @@ if [ -n "${NON_OK_OUTPUT}" ]; then OUTPUT_STRING=${NON_OK_OUTPUT} fi +rm -rf "${TMPDIR}" + echo "${PREFIX}: ${OUTPUT_STRING} | expiring_keys=${NUM_EXPIRING} invalid_keys=${NUM_INVALID} infinite_keys=${NUM_INFINITIVE} total_keys=${NUM_KEYS}" exit "${EXIT}" From af511b2b588b7345d57a723950f4a53e950f7238 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Thu, 3 Oct 2024 13:01:00 +0200 Subject: [PATCH 125/161] Reformat --- files/nagios/check_gpg_keys.sh | 42 +++++++++++++++++----------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/files/nagios/check_gpg_keys.sh b/files/nagios/check_gpg_keys.sh index 7c55328f3..f53bdec51 100755 --- a/files/nagios/check_gpg_keys.sh +++ b/files/nagios/check_gpg_keys.sh @@ -3,8 +3,8 @@ set -uo pipefail if [ $# -ne 1 ]; then - echo "UNKOWN: A directory is required as \$1" - exit 3 + echo "UNKOWN: A directory is required as \$1" + exit 3 fi args=("$@") @@ -34,46 +34,46 @@ TMPDIR=$(mktemp -d) export GNUPGHOME="${TMPDIR}" for key in "${DIRECTORY}"/*.pub; do - ((NUM_KEYS++)) + ((NUM_KEYS++)) - if ! pub_keys=$(gpg --fixed-list-mode --with-colons --show-keys "${key}" 2>/dev/null | grep -e '^pub:'); then + if ! pub_keys=$(gpg --fixed-list-mode --with-colons --show-keys "${key}" 2>/dev/null | grep -e '^pub:'); then INVALIDS+=("${key}") continue fi - # Only allow one public key per file - if [ "$(echo "${pub_keys}" | wc -l)" -ne 1 ]; then - INVALIDS+=("${key}") - continue - fi + # Only allow one public key per file + if [ "$(echo "${pub_keys}" | wc -l)" -ne 1 ]; then + INVALIDS+=("${key}") + continue + fi - expirey_date=$(echo "${pub_keys}" | cut -d : -f 7) + expirey_date=$(echo "${pub_keys}" | cut -d : -f 7) if [ -z "${expirey_date}" ]; then INFINITIVES+=("${key}") - ((WARN++)) + ((WARN++)) continue elif ! echo "${expirey_date}" | grep -qP '^\d{10}$'; then echo "Warning: Can't parse ${key} for validity" - ((WARN++)) + ((WARN++)) INVALIDS+=("${key}") continue fi - fingerprint=$(gpg --fixed-list-mode --with-colons --show-keys "${key}" 2>/dev/null | grep -A1 -e '^pub:' | grep -e '^fpr' |cut -d : -f 10) - filename=$(basename "${key}") - # Only allow files with the long fingerprint as suffix. E.g - # jocar-13376BF892B5871181A218E9BE4EC2EEADF2C31B.pub - if ! echo "${filename}" | grep -qP "^[^-]*-${fingerprint}.pub$"; then - ((WARN++)) + fingerprint=$(gpg --fixed-list-mode --with-colons --show-keys "${key}" 2>/dev/null | grep -A1 -e '^pub:' | grep -e '^fpr' | cut -d : -f 10) + filename=$(basename "${key}") + # Only allow files with the long fingerprint as suffix. E.g + # jocar-13376BF892B5871181A218E9BE4EC2EEADF2C31B.pub + if ! echo "${filename}" | grep -qP "^[^-]*-${fingerprint}.pub$"; then + ((WARN++)) INVALIDS+=("${key}") - fi + fi if [ "${expirey_date}" -lt "${CRITICAL}" ]; then - ((CRIT++)) + ((CRIT++)) EXPIRING+=("${key}") elif [ "${expirey_date}" -lt "${WARNING}" ]; then - ((WARN++)) + ((WARN++)) EXPIRING+=("${key}") fi done From 7ea50d113e2f920b6487916821a66eafb0edff1f Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Thu, 3 Oct 2024 14:47:04 +0200 Subject: [PATCH 126/161] Count correct keys --- files/nagios/check_gpg_keys.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/files/nagios/check_gpg_keys.sh b/files/nagios/check_gpg_keys.sh index f53bdec51..1d47e58d8 100755 --- a/files/nagios/check_gpg_keys.sh +++ b/files/nagios/check_gpg_keys.sh @@ -103,7 +103,7 @@ if [ "${NUM_INVALID}" -ne 0 ]; then fi if [ "${NUM_INFINITIVE}" -ne 0 ]; then - NON_OK_STRING+=("GPG keys without expiration (${NUM_INVALID}): ${INFINITIVES[*]}") + NON_OK_STRING+=("GPG keys without expiration (${NUM_INFINITIVE}): ${INFINITIVES[*]}") fi NON_OK_OUTPUT="" From cd5cf624d631ca51bf3633db3572f4b009e1e076 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Fri, 4 Oct 2024 08:51:24 +0200 Subject: [PATCH 127/161] Naemon didn't like non existing groups --- manifests/naemon_monitor.pp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/manifests/naemon_monitor.pp b/manifests/naemon_monitor.pp index a0a49f249..c96ed4910 100644 --- a/manifests/naemon_monitor.pp +++ b/manifests/naemon_monitor.pp @@ -420,12 +420,6 @@ description => 'Cosmos GPG keys', } - nagioscfg::service {'check_metadata_keys': - hostgroup_name => ['sunet::metadata::metadata_repo'], - check_command => 'check_nrpe!check_metadata_keys', - description => 'Metadata GPG keys', - } - file { '/etc/naemon/conf.d/cosmos/naemon-hostgroups.cfg': ensure => file, mode => '0644', From 710a706cd17d7ec8fc4b7d4cebe0f4feead40441 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Mon, 23 Sep 2024 14:11:41 +0200 Subject: [PATCH 128/161] Initial attempt of a backup class Most of it is copied from Sunet Drive --- manifests/mariadb.pp | 3 + manifests/mariadb/backup.pp | 76 +++++++++++++++++++ .../mariadb/backup/check_replication.erb | 10 +++ templates/mariadb/backup/do_backup.erb.sh | 12 +++ .../backup/start_replica_from_init.erb.sh | 17 +++++ templates/mariadb/backup/status-test.erb | 3 + .../mariadb/docker-compose_mariadb.yml.erb | 7 ++ templates/mariadb/my.cnf.erb | 2 + 8 files changed, 130 insertions(+) create mode 100644 manifests/mariadb/backup.pp create mode 100755 templates/mariadb/backup/check_replication.erb create mode 100644 templates/mariadb/backup/do_backup.erb.sh create mode 100644 templates/mariadb/backup/start_replica_from_init.erb.sh create mode 100644 templates/mariadb/backup/status-test.erb diff --git a/manifests/mariadb.pp b/manifests/mariadb.pp index b96f305ed..1994e8f4f 100644 --- a/manifests/mariadb.pp +++ b/manifests/mariadb.pp @@ -6,6 +6,9 @@ Array[String] $dns = [], ) { + + $galera = true + $mariadb_root_password = lookup('mariadb_root_password', undef, undef,'NOT_SET_IN_HIERA') $mariadb_user = lookup('mariadb_user', undef, undef,undef) $mariadb_user_password = lookup('mariadb_user_password', undef, undef,undef) diff --git a/manifests/mariadb/backup.pp b/manifests/mariadb/backup.pp new file mode 100644 index 000000000..d532a6d57 --- /dev/null +++ b/manifests/mariadb/backup.pp @@ -0,0 +1,76 @@ +# This is a asyncronous replica of the Maria DB Cluster for SUNET +class sunet::mariadb::backup( + $tag_mariadb=undef, + $location=undef +) { + + include sunet::packages::netcat_openbsd + $dirs = [ 'datadir', 'init', 'conf', 'backups' ] + $dirs.each | $dir | { + ensure_resource('file',"/opt/mariadb/backup/${dir}", { ensure => directory, recurse => true } ) + } + + $cluster_nodes = lookup('mariadb_cluster_nodes', undef, undef,[]) + $replicate_from = cluster_nodes[0] + + # Secrets from local.eyaml + $mysql_root_password = safe_hiera('mariadb_root_password') + $backup_password = safe_hiera('mariadb_root_password') + $mysql_user_password = safe_hiera('mariadb_user_password') + + sunet::system_user {'mysql': username => 'mysql', group => 'mysql' } + + $sql_files = ['02-backup_user.sql'] + $sql_files.each |$sql_file|{ + file { "/opt/mariadb/backup/init/${sql_file}": + ensure => present, + content => template("sunet/mariadb/${sql_file}.erb"), + mode => '0744', + } + } + $conf_files = ['credentials.cnf', 'my.cnf'] + $conf_files.each |$conf_file|{ + file { "/opt/mariadb/backup/conf/${conf_file}": + ensure => present, + content => template("sunet/mariadb/backup/${conf_file}.erb"), + mode => '0744', + } + } + file { '/opt/mariadb/backup/start_replica_from_init.sh': + ensure => present, + content => template('sunet/mariadb/backup/start_replica_from_init.erb.sh'), + mode => '0744', + } + # XXX trigger needed + file { '/opt/mariadb/backup/do_backup.sh': + ensure => present, + content => template('sunet/mariadb/backup/do_backup.erb.sh'), + mode => '0744', + } + + file { '/usr/local/bin/check_replication': + ensure => present, + content => template('sunet/mariadb/backup/check_replication.erb'), + mode => '0744', + } + file { '/usr/local/bin/status-test': + ensure => present, + content => template('sunet/mariadb/backup/status-test.erb'), + mode => '0744', + } + file { '/etc/sudoers.d/99-status-test': + ensure => file, + content => "script ALL=(root) NOPASSWD: /usr/local/bin/status-test\n", + mode => '0440', + owner => 'root', + group => 'root', + } + sunet::docker_compose { 'mariadb_backup': + content => template('sunet/mariadb/docker-compose_mariadb.yml.erb'), + service_name => 'mariadb_backup', + compose_dir => '/opt/', + compose_filename => 'docker-compose.yml', + description => 'Mariadb replica', + } + +} diff --git a/templates/mariadb/backup/check_replication.erb b/templates/mariadb/backup/check_replication.erb new file mode 100755 index 000000000..0672c60e0 --- /dev/null +++ b/templates/mariadb/backup/check_replication.erb @@ -0,0 +1,10 @@ +#!/bin/bash + +result="$(docker exec mariadb_backup_mariadb_backup_1 mysql -p<%= @mariadb_root_password %> -BN -e 'show status like "slave_running"')" +if [[ "${result}" == "Slave_running ON" ]]; then + echo "OK: Replica running" + exit 0 +else + echo "CRITICAL: Replica not running" + exit 2 +fi diff --git a/templates/mariadb/backup/do_backup.erb.sh b/templates/mariadb/backup/do_backup.erb.sh new file mode 100644 index 000000000..bc1d43861 --- /dev/null +++ b/templates/mariadb/backup/do_backup.erb.sh @@ -0,0 +1,12 @@ +#!/bin/bash +stream_name="mariadb-stream-$(date +%Y-%m-%dT%H.%M.%S).gz" +dump_name="mariadb-dump-$(date +%Y-%m-%dT%H.%M.%S).sql.gz" +backup_dir="/opt/mariadb/backups/$(date +%Y/%m/%d)" +mkdir -p "${backup_dir}" + +buopts="--slave-info --safe-slave-backup" +dumpopts="--dump-slave" +mysql -p"${MYSQL_ROOT_PASSWORD}" -e "stop slave" +mariadb-backup --backup ${buopts} -u root -p"${MYSQL_ROOT_PASSWORD}" --stream=xbstream | gzip >"${backup_dir}/${stream_name}" +mysqldump --all-databases --single-transaction ${dumpopts} -u root -p${MYSQL_ROOT_PASSWORD} | gzip >"${backup_dir}/${dump_name}" +mysql -p${MYSQL_ROOT_PASSWORD} -e "start slave" diff --git a/templates/mariadb/backup/start_replica_from_init.erb.sh b/templates/mariadb/backup/start_replica_from_init.erb.sh new file mode 100644 index 000000000..c41c12575 --- /dev/null +++ b/templates/mariadb/backup/start_replica_from_init.erb.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +mysql="mysql -u root -p${MYSQL_ROOT_PASSWORD}" +init_file='/backups/init.sql.gz' +if [[ -f ${init_file} ]]; then + ${mysql} -e "STOP SLAVE;RESET SLAVE;" + master_command=$(zgrep 'CHANGE MASTER TO MASTER_LOG_FILE' ${init_file} | sed -e 's/^-- //' -e 's/;$//') + master_command="${master_command}, MASTER_HOST='<%= @repliacte_from %>', MASTER_USER='backup'" + master_command="${master_command}, MASTER_PASSWORD='<%= @mariadb_backup_password%>', MASTER_SSL=1" + master_command="${master_command}, MASTER_CONNECT_RETRY=20" + zcat ${init_file} | ${mysql} + ${mysql} -e "${master_command}" + ${mysql} -e "START SLAVE" + sleep 3s + ${mysql} -e "SHOW SLAVE STATUS\G" +fi + +exit 0 diff --git a/templates/mariadb/backup/status-test.erb b/templates/mariadb/backup/status-test.erb new file mode 100644 index 000000000..197e50bc8 --- /dev/null +++ b/templates/mariadb/backup/status-test.erb @@ -0,0 +1,3 @@ +#!/bin/bash + +docker exec mariadb_backup_mariadb_backup_1 mysql -u root -p'<%= @mariadb_root_password %>' -N -B -e "show status like 'Slave_running'" diff --git a/templates/mariadb/docker-compose_mariadb.yml.erb b/templates/mariadb/docker-compose_mariadb.yml.erb index d92245767..22d635dc8 100644 --- a/templates/mariadb/docker-compose_mariadb.yml.erb +++ b/templates/mariadb/docker-compose_mariadb.yml.erb @@ -11,7 +11,12 @@ services: - /opt/mariadb/datadir:/var/lib/mysql - /opt/mariadb/init:/docker-entrypoint-initdb.d - /opt/mariadb/scripts:/scripts +<%- if @backup -%> + - /opt/mariadb_backup/start_replica_from_init.sh:/start_replica_from_init.sh +<% end -%> +<%- if @galera -%> network_mode: host +<% end -%> <%- if !@dns.empty? -%> dns: <% @dns.each do |resolver| -%> @@ -29,8 +34,10 @@ services: <%- if @mariadb_database -%> - MYSQL_DATABASE=<%= @mariadb_database %> <%- end -%> +<%- if @galera -%> - BOOTSTRAP=<%= @bootstrap %> - FORCE_BOOTSTRAP=0 command: "--wsrep_cluster_address=gcomm://<%= @cluster_nodes.join(',') %>" tty: true +<%- end -%> diff --git a/templates/mariadb/my.cnf.erb b/templates/mariadb/my.cnf.erb index 47556ed19..e80e89cb0 100644 --- a/templates/mariadb/my.cnf.erb +++ b/templates/mariadb/my.cnf.erb @@ -34,6 +34,7 @@ innodb_rollback_on_timeout = 1 innodb_write_io_threads = 4 # CPU dependent transaction_isolation = 'READ-COMMITTED' +<% if @galera -%> # Galera wsrep_cluster_name = "Sunet_MariaDB_Cluster" wsrep_gtid_domain_id = 1000 # same on all Galera nodes in the same segment @@ -45,3 +46,4 @@ wsrep_provider_options = "gcache.size=2G;gmcast.segment=0" # gmcast.seg wsrep_slave_threads = 4 # CPU dependent wsrep_sst_method = mariabackup wsrep_sync_wait = 1 +<% end -%> From f3456f6751b5338cd98a0afd710bcdf620e1eef3 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Mon, 23 Sep 2024 14:36:20 +0200 Subject: [PATCH 129/161] Share config with cluster class --- manifests/mariadb/backup.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/mariadb/backup.pp b/manifests/mariadb/backup.pp index d532a6d57..3aaf52b53 100644 --- a/manifests/mariadb/backup.pp +++ b/manifests/mariadb/backup.pp @@ -32,7 +32,7 @@ $conf_files.each |$conf_file|{ file { "/opt/mariadb/backup/conf/${conf_file}": ensure => present, - content => template("sunet/mariadb/backup/${conf_file}.erb"), + content => template("sunet/mariadb/${conf_file}.erb"), mode => '0744', } } From 76c35dde6a1e5e463c6bf77bf4f97cc40539e533 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Mon, 23 Sep 2024 14:40:03 +0200 Subject: [PATCH 130/161] Option to change resolvers inside container --- manifests/mariadb/backup.pp | 1 + 1 file changed, 1 insertion(+) diff --git a/manifests/mariadb/backup.pp b/manifests/mariadb/backup.pp index 3aaf52b53..c20db9f15 100644 --- a/manifests/mariadb/backup.pp +++ b/manifests/mariadb/backup.pp @@ -2,6 +2,7 @@ class sunet::mariadb::backup( $tag_mariadb=undef, $location=undef + Array[String] $dns = [], ) { include sunet::packages::netcat_openbsd From 2368a3b992212560cd262e7f522b5cfd50350b3e Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Mon, 23 Sep 2024 14:40:24 +0200 Subject: [PATCH 131/161] Not in use outside Drive --- manifests/mariadb/backup.pp | 1 - 1 file changed, 1 deletion(-) diff --git a/manifests/mariadb/backup.pp b/manifests/mariadb/backup.pp index c20db9f15..ec94aab40 100644 --- a/manifests/mariadb/backup.pp +++ b/manifests/mariadb/backup.pp @@ -1,7 +1,6 @@ # This is a asyncronous replica of the Maria DB Cluster for SUNET class sunet::mariadb::backup( $tag_mariadb=undef, - $location=undef Array[String] $dns = [], ) { From cc4b24164d9f4cc3e3594b596af2c8142837d01b Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Mon, 23 Sep 2024 14:42:10 +0200 Subject: [PATCH 132/161] Use variable already in Compose file --- manifests/mariadb/backup.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/mariadb/backup.pp b/manifests/mariadb/backup.pp index ec94aab40..5ff9157e0 100644 --- a/manifests/mariadb/backup.pp +++ b/manifests/mariadb/backup.pp @@ -1,6 +1,6 @@ # This is a asyncronous replica of the Maria DB Cluster for SUNET class sunet::mariadb::backup( - $tag_mariadb=undef, + String $mariadb_version=latest, Array[String] $dns = [], ) { From 496055df3a22aff47534552aa759d4d7615541a2 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Mon, 23 Sep 2024 14:43:59 +0200 Subject: [PATCH 133/161] Create basedir --- manifests/mariadb/backup.pp | 1 + 1 file changed, 1 insertion(+) diff --git a/manifests/mariadb/backup.pp b/manifests/mariadb/backup.pp index 5ff9157e0..c37f39e51 100644 --- a/manifests/mariadb/backup.pp +++ b/manifests/mariadb/backup.pp @@ -5,6 +5,7 @@ ) { include sunet::packages::netcat_openbsd + ensure_resource('file',"/opt/mariadb/backup/", { ensure => directory, recurse => true } ) $dirs = [ 'datadir', 'init', 'conf', 'backups' ] $dirs.each | $dir | { ensure_resource('file',"/opt/mariadb/backup/${dir}", { ensure => directory, recurse => true } ) From 76c65a1a77d45f19c9d4de0c1af96a62a13853a2 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Mon, 23 Sep 2024 14:59:50 +0200 Subject: [PATCH 134/161] Run as a plain mariadb --- manifests/mariadb/backup.pp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/manifests/mariadb/backup.pp b/manifests/mariadb/backup.pp index c37f39e51..69b615ae2 100644 --- a/manifests/mariadb/backup.pp +++ b/manifests/mariadb/backup.pp @@ -66,9 +66,9 @@ owner => 'root', group => 'root', } - sunet::docker_compose { 'mariadb_backup': + sunet::docker_compose { 'mariadb': content => template('sunet/mariadb/docker-compose_mariadb.yml.erb'), - service_name => 'mariadb_backup', + service_name => 'mariadb', compose_dir => '/opt/', compose_filename => 'docker-compose.yml', description => 'Mariadb replica', From a262ddeb128c339304747ff6b9cd6b132121ae80 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Tue, 24 Sep 2024 09:47:54 +0200 Subject: [PATCH 135/161] Use the correct variable names --- manifests/mariadb/backup.pp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/manifests/mariadb/backup.pp b/manifests/mariadb/backup.pp index 69b615ae2..f2bd657d3 100644 --- a/manifests/mariadb/backup.pp +++ b/manifests/mariadb/backup.pp @@ -15,9 +15,9 @@ $replicate_from = cluster_nodes[0] # Secrets from local.eyaml - $mysql_root_password = safe_hiera('mariadb_root_password') - $backup_password = safe_hiera('mariadb_root_password') - $mysql_user_password = safe_hiera('mariadb_user_password') + $mariadb_root_password = safe_hiera('mariadb_root_password') + $mariadb_backup_password = safe_hiera('mariadb_root_password') + $mariadb_user_password = safe_hiera('mariadb_user_password') sunet::system_user {'mysql': username => 'mysql', group => 'mysql' } From 9bd508f13479f7dc31b8a26d8d11f9e9b7e3d95d Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Thu, 26 Sep 2024 14:17:26 +0200 Subject: [PATCH 136/161] Put script in a location where it can be reached by container --- manifests/mariadb/backup.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/mariadb/backup.pp b/manifests/mariadb/backup.pp index f2bd657d3..136a6f952 100644 --- a/manifests/mariadb/backup.pp +++ b/manifests/mariadb/backup.pp @@ -37,7 +37,7 @@ mode => '0744', } } - file { '/opt/mariadb/backup/start_replica_from_init.sh': + file { '/opt/mariadb/scripts/start_replica_from_init.sh': ensure => present, content => template('sunet/mariadb/backup/start_replica_from_init.erb.sh'), mode => '0744', From dbd322c7b59a2e1a2f2400794df5f889711e4e53 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Thu, 26 Sep 2024 14:30:45 +0200 Subject: [PATCH 137/161] Tyop --- templates/mariadb/backup/start_replica_from_init.erb.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/mariadb/backup/start_replica_from_init.erb.sh b/templates/mariadb/backup/start_replica_from_init.erb.sh index c41c12575..3f6242c34 100644 --- a/templates/mariadb/backup/start_replica_from_init.erb.sh +++ b/templates/mariadb/backup/start_replica_from_init.erb.sh @@ -4,7 +4,7 @@ init_file='/backups/init.sql.gz' if [[ -f ${init_file} ]]; then ${mysql} -e "STOP SLAVE;RESET SLAVE;" master_command=$(zgrep 'CHANGE MASTER TO MASTER_LOG_FILE' ${init_file} | sed -e 's/^-- //' -e 's/;$//') - master_command="${master_command}, MASTER_HOST='<%= @repliacte_from %>', MASTER_USER='backup'" + master_command="${master_command}, MASTER_HOST='<%= @replicate_from %>', MASTER_USER='backup'" master_command="${master_command}, MASTER_PASSWORD='<%= @mariadb_backup_password%>', MASTER_SSL=1" master_command="${master_command}, MASTER_CONNECT_RETRY=20" zcat ${init_file} | ${mysql} From cdafd2b25890bea15027a4709e43985f10227ce3 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Thu, 26 Sep 2024 14:38:00 +0200 Subject: [PATCH 138/161] Syntax error --- manifests/mariadb/backup.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/mariadb/backup.pp b/manifests/mariadb/backup.pp index 136a6f952..769568b51 100644 --- a/manifests/mariadb/backup.pp +++ b/manifests/mariadb/backup.pp @@ -12,7 +12,7 @@ } $cluster_nodes = lookup('mariadb_cluster_nodes', undef, undef,[]) - $replicate_from = cluster_nodes[0] + $replicate_from = $cluster_nodes[0] # Secrets from local.eyaml $mariadb_root_password = safe_hiera('mariadb_root_password') From d68c89298728c301dbf9a9b78b56cdffcf935bbf Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Thu, 26 Sep 2024 15:23:59 +0200 Subject: [PATCH 139/161] Put config where mariadb reads --- manifests/mariadb/backup.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/mariadb/backup.pp b/manifests/mariadb/backup.pp index 769568b51..2d38c461a 100644 --- a/manifests/mariadb/backup.pp +++ b/manifests/mariadb/backup.pp @@ -31,7 +31,7 @@ } $conf_files = ['credentials.cnf', 'my.cnf'] $conf_files.each |$conf_file|{ - file { "/opt/mariadb/backup/conf/${conf_file}": + file { "/opt/mariadb/conf/${conf_file}": ensure => present, content => template("sunet/mariadb/${conf_file}.erb"), mode => '0744', From 17ba2d1949c26cec6db7f1995eff9686443a48b1 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Fri, 27 Sep 2024 10:05:38 +0200 Subject: [PATCH 140/161] server_id must be unique Previous method always returned 1001. --- manifests/mariadb.pp | 1 - templates/mariadb/my.cnf.erb | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/manifests/mariadb.pp b/manifests/mariadb.pp index 1994e8f4f..490e6da46 100644 --- a/manifests/mariadb.pp +++ b/manifests/mariadb.pp @@ -17,7 +17,6 @@ $clients = lookup('mariadb_clients', undef, undef,['127.0.0.1']) $cluster_nodes = lookup('mariadb_cluster_nodes', undef, undef,[]) $mariadb_dir = '/opt/mariadb' - $server_id = 1000 + Integer($facts['networking']['hostname'][-1]) # Hack to not clash with docker_compose which tries to create the same directory exec {'mariadb_dir_create': diff --git a/templates/mariadb/my.cnf.erb b/templates/mariadb/my.cnf.erb index e80e89cb0..f64591e8c 100644 --- a/templates/mariadb/my.cnf.erb +++ b/templates/mariadb/my.cnf.erb @@ -18,7 +18,7 @@ gtid_ignore_duplicates = ON gtid_strict_mode = ON log_bin = binlog log_slave_updates = ON -server_id = <%= @server_id %> +server_id = <%= @facts['networking']['ip'].split(".").map(&:to_i).pack('CCCC').unpack('N')[0] %> # Innodb innodb_autoinc_lock_mode = 2 From 3d97e7a0b8e437d5f4066da4a6a823f9d59090ff Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Fri, 27 Sep 2024 11:07:04 +0200 Subject: [PATCH 141/161] Static relay logs --- templates/mariadb/my.cnf.erb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/templates/mariadb/my.cnf.erb b/templates/mariadb/my.cnf.erb index f64591e8c..a66cb4aa0 100644 --- a/templates/mariadb/my.cnf.erb +++ b/templates/mariadb/my.cnf.erb @@ -19,6 +19,8 @@ gtid_strict_mode = ON log_bin = binlog log_slave_updates = ON server_id = <%= @facts['networking']['ip'].split(".").map(&:to_i).pack('CCCC').unpack('N')[0] %> +# Default hostname base relay_log is no good in containers +relay_log = 'relay-log' # Innodb innodb_autoinc_lock_mode = 2 From 8e9bc88a6bf589ca08f707275ea9785f5b82ae86 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Fri, 27 Sep 2024 11:07:56 +0200 Subject: [PATCH 142/161] Explain the witch craft --- templates/mariadb/my.cnf.erb | 1 + 1 file changed, 1 insertion(+) diff --git a/templates/mariadb/my.cnf.erb b/templates/mariadb/my.cnf.erb index a66cb4aa0..30c0664e6 100644 --- a/templates/mariadb/my.cnf.erb +++ b/templates/mariadb/my.cnf.erb @@ -18,6 +18,7 @@ gtid_ignore_duplicates = ON gtid_strict_mode = ON log_bin = binlog log_slave_updates = ON +# Use IP adress as decimal in order to create an unique server id server_id = <%= @facts['networking']['ip'].split(".").map(&:to_i).pack('CCCC').unpack('N')[0] %> # Default hostname base relay_log is no good in containers relay_log = 'relay-log' From 2ef4a89e9f5040857852e21d588388fb2687b5b3 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Fri, 27 Sep 2024 11:42:29 +0200 Subject: [PATCH 143/161] Match cluster class names --- templates/mariadb/backup/status-test.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/mariadb/backup/status-test.erb b/templates/mariadb/backup/status-test.erb index 197e50bc8..8094c9eff 100644 --- a/templates/mariadb/backup/status-test.erb +++ b/templates/mariadb/backup/status-test.erb @@ -1,3 +1,3 @@ #!/bin/bash -docker exec mariadb_backup_mariadb_backup_1 mysql -u root -p'<%= @mariadb_root_password %>' -N -B -e "show status like 'Slave_running'" +docker exec mariadb-db-1 mysql -u root -p'<%= @mariadb_root_password %>' -N -B -e "show status like 'Slave_running'" From 63d016ec81915b1e881ef23faf5eed285a8f96d4 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Fri, 27 Sep 2024 11:44:40 +0200 Subject: [PATCH 144/161] Match cluster names --- templates/mariadb/backup/check_replication.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/mariadb/backup/check_replication.erb b/templates/mariadb/backup/check_replication.erb index 0672c60e0..035dc4a6d 100755 --- a/templates/mariadb/backup/check_replication.erb +++ b/templates/mariadb/backup/check_replication.erb @@ -1,6 +1,6 @@ #!/bin/bash -result="$(docker exec mariadb_backup_mariadb_backup_1 mysql -p<%= @mariadb_root_password %> -BN -e 'show status like "slave_running"')" +result="$(docker exec mariadb-db-1 mysql -p<%= @mariadb_root_password %> -BN -e 'show status like "slave_running"')" if [[ "${result}" == "Slave_running ON" ]]; then echo "OK: Replica running" exit 0 From 8a484a7b1e79f5c8663cbdf5f8315c7ace7d1873 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Fri, 27 Sep 2024 11:49:05 +0200 Subject: [PATCH 145/161] Allow container access --- manifests/mariadb/backup.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/mariadb/backup.pp b/manifests/mariadb/backup.pp index 2d38c461a..3cb3105b2 100644 --- a/manifests/mariadb/backup.pp +++ b/manifests/mariadb/backup.pp @@ -43,7 +43,7 @@ mode => '0744', } # XXX trigger needed - file { '/opt/mariadb/backup/do_backup.sh': + file { '/opt/mariadb/scripts/do_backup.sh': ensure => present, content => template('sunet/mariadb/backup/do_backup.erb.sh'), mode => '0744', From b906aa3b7cf21ff00c5e0cae5cc48f042fc08782 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Fri, 27 Sep 2024 11:51:20 +0200 Subject: [PATCH 146/161] Run inside container --- templates/mariadb/backup/do_backup.erb.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/mariadb/backup/do_backup.erb.sh b/templates/mariadb/backup/do_backup.erb.sh index bc1d43861..67a77d04a 100644 --- a/templates/mariadb/backup/do_backup.erb.sh +++ b/templates/mariadb/backup/do_backup.erb.sh @@ -1,7 +1,7 @@ #!/bin/bash stream_name="mariadb-stream-$(date +%Y-%m-%dT%H.%M.%S).gz" dump_name="mariadb-dump-$(date +%Y-%m-%dT%H.%M.%S).sql.gz" -backup_dir="/opt/mariadb/backups/$(date +%Y/%m/%d)" +backup_dir="/backups/$(date +%Y/%m/%d)" mkdir -p "${backup_dir}" buopts="--slave-info --safe-slave-backup" From a630d4fd9f60999693706baf17c7d4c5d0ce588b Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Fri, 27 Sep 2024 12:15:56 +0200 Subject: [PATCH 147/161] Allow NRPE to check the replication --- manifests/mariadb/backup.pp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/manifests/mariadb/backup.pp b/manifests/mariadb/backup.pp index 3cb3105b2..1c707e3ff 100644 --- a/manifests/mariadb/backup.pp +++ b/manifests/mariadb/backup.pp @@ -2,6 +2,7 @@ class sunet::mariadb::backup( String $mariadb_version=latest, Array[String] $dns = [], + Boolean $nrpe = true; ) { include sunet::packages::netcat_openbsd @@ -66,6 +67,14 @@ owner => 'root', group => 'root', } + sunet::sudoer {'nagios_run_replication_command': + user_name => 'nagios', + collection => 'nrpe_replication_check', + command_line => '/usr/local/bin/check_replication' + } + sunet::nagios::nrpe_command {'check_async_replication': + command_line => '/usr/bin/sudo /usr/local/bin/check_replication' + } sunet::docker_compose { 'mariadb': content => template('sunet/mariadb/docker-compose_mariadb.yml.erb'), service_name => 'mariadb', From a34aebce04db47885980ab16a05058e74d9efc55 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Fri, 27 Sep 2024 12:17:47 +0200 Subject: [PATCH 148/161] This is not perl :( --- manifests/mariadb/backup.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/mariadb/backup.pp b/manifests/mariadb/backup.pp index 1c707e3ff..fd5e1eab6 100644 --- a/manifests/mariadb/backup.pp +++ b/manifests/mariadb/backup.pp @@ -2,7 +2,7 @@ class sunet::mariadb::backup( String $mariadb_version=latest, Array[String] $dns = [], - Boolean $nrpe = true; + Boolean $nrpe = true, ) { include sunet::packages::netcat_openbsd From 3acd4de63d42fa43f9316cbda372cb2e435b5cab Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Fri, 27 Sep 2024 12:18:02 +0200 Subject: [PATCH 149/161] Wrap in feature flag --- manifests/mariadb/backup.pp | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/manifests/mariadb/backup.pp b/manifests/mariadb/backup.pp index fd5e1eab6..e8cb36d0f 100644 --- a/manifests/mariadb/backup.pp +++ b/manifests/mariadb/backup.pp @@ -67,13 +67,16 @@ owner => 'root', group => 'root', } - sunet::sudoer {'nagios_run_replication_command': - user_name => 'nagios', - collection => 'nrpe_replication_check', - command_line => '/usr/local/bin/check_replication' - } - sunet::nagios::nrpe_command {'check_async_replication': - command_line => '/usr/bin/sudo /usr/local/bin/check_replication' + + if $nrpe { + sunet::sudoer {'nagios_run_replication_command': + user_name => 'nagios', + collection => 'nrpe_replication_check', + command_line => '/usr/local/bin/check_replication' + } + sunet::nagios::nrpe_command {'check_async_replication': + command_line => '/usr/bin/sudo /usr/local/bin/check_replication' + } } sunet::docker_compose { 'mariadb': content => template('sunet/mariadb/docker-compose_mariadb.yml.erb'), From 907997f0be0025c211e311213dc43a7733073777 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Fri, 27 Sep 2024 13:12:19 +0200 Subject: [PATCH 150/161] Don't reinvent the wheel - use a define! --- manifests/mariadb/backup.pp | 34 ++++------------------------------ 1 file changed, 4 insertions(+), 30 deletions(-) diff --git a/manifests/mariadb/backup.pp b/manifests/mariadb/backup.pp index e8cb36d0f..14743686a 100644 --- a/manifests/mariadb/backup.pp +++ b/manifests/mariadb/backup.pp @@ -5,11 +5,10 @@ Boolean $nrpe = true, ) { - include sunet::packages::netcat_openbsd - ensure_resource('file',"/opt/mariadb/backup/", { ensure => directory, recurse => true } ) - $dirs = [ 'datadir', 'init', 'conf', 'backups' ] - $dirs.each | $dir | { - ensure_resource('file',"/opt/mariadb/backup/${dir}", { ensure => directory, recurse => true } ) + sunet::mariadb { 'sunet_mariadb_simple': + mariadb_version => $mariadb_version, + ports => [3306], + dns => $dns, } $cluster_nodes = lookup('mariadb_cluster_nodes', undef, undef,[]) @@ -20,24 +19,6 @@ $mariadb_backup_password = safe_hiera('mariadb_root_password') $mariadb_user_password = safe_hiera('mariadb_user_password') - sunet::system_user {'mysql': username => 'mysql', group => 'mysql' } - - $sql_files = ['02-backup_user.sql'] - $sql_files.each |$sql_file|{ - file { "/opt/mariadb/backup/init/${sql_file}": - ensure => present, - content => template("sunet/mariadb/${sql_file}.erb"), - mode => '0744', - } - } - $conf_files = ['credentials.cnf', 'my.cnf'] - $conf_files.each |$conf_file|{ - file { "/opt/mariadb/conf/${conf_file}": - ensure => present, - content => template("sunet/mariadb/${conf_file}.erb"), - mode => '0744', - } - } file { '/opt/mariadb/scripts/start_replica_from_init.sh': ensure => present, content => template('sunet/mariadb/backup/start_replica_from_init.erb.sh'), @@ -78,12 +59,5 @@ command_line => '/usr/bin/sudo /usr/local/bin/check_replication' } } - sunet::docker_compose { 'mariadb': - content => template('sunet/mariadb/docker-compose_mariadb.yml.erb'), - service_name => 'mariadb', - compose_dir => '/opt/', - compose_filename => 'docker-compose.yml', - description => 'Mariadb replica', - } } From 06acae7f5a60867d9f300d8a014b55a0c9f38241 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Fri, 27 Sep 2024 13:14:14 +0200 Subject: [PATCH 151/161] Options --- manifests/mariadb.pp | 3 +-- manifests/mariadb/backup.pp | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/manifests/mariadb.pp b/manifests/mariadb.pp index 490e6da46..69fb91bbc 100644 --- a/manifests/mariadb.pp +++ b/manifests/mariadb.pp @@ -4,11 +4,10 @@ Integer $bootstrap=0, Array[Integer] $ports = [3306, 4444, 4567, 4568], Array[String] $dns = [], + Boolean $galera = true, ) { - $galera = true - $mariadb_root_password = lookup('mariadb_root_password', undef, undef,'NOT_SET_IN_HIERA') $mariadb_user = lookup('mariadb_user', undef, undef,undef) $mariadb_user_password = lookup('mariadb_user_password', undef, undef,undef) diff --git a/manifests/mariadb/backup.pp b/manifests/mariadb/backup.pp index 14743686a..f7cb7c35c 100644 --- a/manifests/mariadb/backup.pp +++ b/manifests/mariadb/backup.pp @@ -9,6 +9,7 @@ mariadb_version => $mariadb_version, ports => [3306], dns => $dns, + galera => False, } $cluster_nodes = lookup('mariadb_cluster_nodes', undef, undef,[]) From ae94737eebcbfb1b34a0c4c3f472b4320c5552ed Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Fri, 27 Sep 2024 13:15:06 +0200 Subject: [PATCH 152/161] Syntax error --- manifests/mariadb/backup.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/mariadb/backup.pp b/manifests/mariadb/backup.pp index f7cb7c35c..dd1f37fa2 100644 --- a/manifests/mariadb/backup.pp +++ b/manifests/mariadb/backup.pp @@ -9,7 +9,7 @@ mariadb_version => $mariadb_version, ports => [3306], dns => $dns, - galera => False, + galera => false, } $cluster_nodes = lookup('mariadb_cluster_nodes', undef, undef,[]) From a3b0f5660dab00d381089ea1676c5a3ee1e73ac4 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Fri, 27 Sep 2024 13:18:09 +0200 Subject: [PATCH 153/161] Don't clash with the backup files --- manifests/mariadb.pp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/manifests/mariadb.pp b/manifests/mariadb.pp index 69fb91bbc..d82cf336a 100644 --- a/manifests/mariadb.pp +++ b/manifests/mariadb.pp @@ -65,14 +65,14 @@ content => template('sunet/mariadb/cluster-status.erb.sh'), mode => '0744', } - file { '/etc/sudoers.d/99-size-test': + file { '/etc/sudoers.d/99-cluster-size-test': ensure => file, content => "script ALL=(root) NOPASSWD: /usr/local/bin/cluster-size\n", mode => '0440', owner => 'root', group => 'root', } - file { '/etc/sudoers.d/99-status-test': + file { '/etc/sudoers.d/99-cluster-status-test': ensure => file, content => "script ALL=(root) NOPASSWD: /usr/local/bin/cluster-status\n", mode => '0440', From 4a9dcf29e0d75db411b12006a868fa4b824f0523 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Fri, 27 Sep 2024 14:09:01 +0200 Subject: [PATCH 154/161] Start doing the backup --- manifests/mariadb/backup.pp | 15 ++++++++++++++- templates/mariadb/backup/backup2baas.erb | 11 +++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) create mode 100755 templates/mariadb/backup/backup2baas.erb diff --git a/manifests/mariadb/backup.pp b/manifests/mariadb/backup.pp index dd1f37fa2..ac6f8beaa 100644 --- a/manifests/mariadb/backup.pp +++ b/manifests/mariadb/backup.pp @@ -25,13 +25,26 @@ content => template('sunet/mariadb/backup/start_replica_from_init.erb.sh'), mode => '0744', } - # XXX trigger needed file { '/opt/mariadb/scripts/do_backup.sh': ensure => present, content => template('sunet/mariadb/backup/do_backup.erb.sh'), mode => '0744', } + file { '/usr/local/bin/backup2baas': + ensure => present, + content => template('sunet/mariadb/backup/backup2baas.erb'), + mode => '0744', + } + + sunet::scriptherder::cronjob { 'backup2baas': + cmd => '/usr/local/bin/backup2baas', + hour => '6', + minute => '0', + ok_criteria => ['exit_status=0', 'max_age=24h'], + warn_criteria => ['exit_status=1'], + } + file { '/usr/local/bin/check_replication': ensure => present, content => template('sunet/mariadb/backup/check_replication.erb'), diff --git a/templates/mariadb/backup/backup2baas.erb b/templates/mariadb/backup/backup2baas.erb new file mode 100755 index 000000000..69a4dc757 --- /dev/null +++ b/templates/mariadb/backup/backup2baas.erb @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -e + +docker exec mariadb-db-1 /scripts/do_backup.sh + +BACKUPDIR=/opt/mariadb/backups +find "${BACKUPDIR}" -type f -mtime +31 -exec rm -f {} \; +find "${BACKUPDIR}" -empty -type d -delete + +/usr/bin/dsmc backup From 107d28fefd02bb5ad6e70659ce48e02717f640c8 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Fri, 27 Sep 2024 14:20:24 +0200 Subject: [PATCH 155/161] Don't run at the same time as binlog purge --- manifests/mariadb/backup.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/mariadb/backup.pp b/manifests/mariadb/backup.pp index ac6f8beaa..61849c2d5 100644 --- a/manifests/mariadb/backup.pp +++ b/manifests/mariadb/backup.pp @@ -40,7 +40,7 @@ sunet::scriptherder::cronjob { 'backup2baas': cmd => '/usr/local/bin/backup2baas', hour => '6', - minute => '0', + minute => '10', ok_criteria => ['exit_status=0', 'max_age=24h'], warn_criteria => ['exit_status=1'], } From 5083cbc6c79981103f3157c06a055ded48ccd882 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Mon, 7 Oct 2024 09:27:43 +0200 Subject: [PATCH 156/161] Make baas2 optional --- manifests/mariadb/backup.pp | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/manifests/mariadb/backup.pp b/manifests/mariadb/backup.pp index 61849c2d5..2e06f9243 100644 --- a/manifests/mariadb/backup.pp +++ b/manifests/mariadb/backup.pp @@ -2,6 +2,7 @@ class sunet::mariadb::backup( String $mariadb_version=latest, Array[String] $dns = [], + Boolean $backup_to_baas = true, Boolean $nrpe = true, ) { @@ -31,18 +32,20 @@ mode => '0744', } - file { '/usr/local/bin/backup2baas': - ensure => present, - content => template('sunet/mariadb/backup/backup2baas.erb'), - mode => '0744', - } + if $backup_to_baas { + file { '/usr/local/bin/backup2baas': + ensure => present, + content => template('sunet/mariadb/backup/backup2baas.erb'), + mode => '0744', + } - sunet::scriptherder::cronjob { 'backup2baas': - cmd => '/usr/local/bin/backup2baas', - hour => '6', - minute => '10', - ok_criteria => ['exit_status=0', 'max_age=24h'], - warn_criteria => ['exit_status=1'], + sunet::scriptherder::cronjob { 'backup2baas': + cmd => '/usr/local/bin/backup2baas', + hour => '6', + minute => '10', + ok_criteria => ['exit_status=0', 'max_age=24h'], + warn_criteria => ['exit_status=1'], + } } file { '/usr/local/bin/check_replication': From 1cb90fb27108b85889127e16cf88b82290de75be Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Mon, 7 Oct 2024 10:12:39 +0200 Subject: [PATCH 157/161] Better explain what script does Also less risk for name collitions --- manifests/mariadb.pp | 8 ++++---- manifests/mariadb/backup.pp | 14 +++++++------- .../{status-test.erb => replication-status.erb} | 0 3 files changed, 11 insertions(+), 11 deletions(-) rename templates/mariadb/backup/{status-test.erb => replication-status.erb} (100%) diff --git a/manifests/mariadb.pp b/manifests/mariadb.pp index d82cf336a..dd3e509d4 100644 --- a/manifests/mariadb.pp +++ b/manifests/mariadb.pp @@ -55,26 +55,26 @@ ok_criteria => ['exit_status=0','max_age=2d'], warn_criteria => ['exit_status=1','max_age=3d'], } - file { '/usr/local/bin/cluster-size': + file { '/usr/local/bin/mariadb-galera-size': ensure => present, content => template('sunet/mariadb/cluster-size.erb.sh'), mode => '0744', } - file { '/usr/local/bin/cluster-status': + file { '/usr/local/bin/mariadb-galera-status': ensure => present, content => template('sunet/mariadb/cluster-status.erb.sh'), mode => '0744', } file { '/etc/sudoers.d/99-cluster-size-test': ensure => file, - content => "script ALL=(root) NOPASSWD: /usr/local/bin/cluster-size\n", + content => "script ALL=(root) NOPASSWD: /usr/local/bin/mariadb-galera-size\n", mode => '0440', owner => 'root', group => 'root', } file { '/etc/sudoers.d/99-cluster-status-test': ensure => file, - content => "script ALL=(root) NOPASSWD: /usr/local/bin/cluster-status\n", + content => "script ALL=(root) NOPASSWD: /usr/local/bin/mariadb-galera-status\n", mode => '0440', owner => 'root', group => 'root', diff --git a/manifests/mariadb/backup.pp b/manifests/mariadb/backup.pp index 2e06f9243..c8ecf67fa 100644 --- a/manifests/mariadb/backup.pp +++ b/manifests/mariadb/backup.pp @@ -48,19 +48,19 @@ } } - file { '/usr/local/bin/check_replication': + file { '/usr/lib/nagios/plugins/check_mariadb-replication': ensure => present, content => template('sunet/mariadb/backup/check_replication.erb'), mode => '0744', } - file { '/usr/local/bin/status-test': + file { '/usr/local/bin/mariadb-replication-status': ensure => present, - content => template('sunet/mariadb/backup/status-test.erb'), + content => template('sunet/mariadb/backup/replication-status.erb'), mode => '0744', } - file { '/etc/sudoers.d/99-status-test': + file { '/etc/sudoers.d/99-mariadb-replication-test': ensure => file, - content => "script ALL=(root) NOPASSWD: /usr/local/bin/status-test\n", + content => "script ALL=(root) NOPASSWD: /usr/local/bin/mariadb-replication-status", mode => '0440', owner => 'root', group => 'root', @@ -70,10 +70,10 @@ sunet::sudoer {'nagios_run_replication_command': user_name => 'nagios', collection => 'nrpe_replication_check', - command_line => '/usr/local/bin/check_replication' + command_line => '/usr/lib/nagios/plugins/check_mariadb-replication' } sunet::nagios::nrpe_command {'check_async_replication': - command_line => '/usr/bin/sudo /usr/local/bin/check_replication' + command_line => '/usr/bin/sudo /usr/lib/nagios/plugins/check_mariadb-replication' } } diff --git a/templates/mariadb/backup/status-test.erb b/templates/mariadb/backup/replication-status.erb similarity index 100% rename from templates/mariadb/backup/status-test.erb rename to templates/mariadb/backup/replication-status.erb From b6ff4a5f3c602ba9db66fbd41e6ad3ea784bb3ef Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Mon, 7 Oct 2024 10:26:29 +0200 Subject: [PATCH 158/161] Less duplicated data --- templates/mariadb/backup/check_replication.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/mariadb/backup/check_replication.erb b/templates/mariadb/backup/check_replication.erb index 035dc4a6d..d8ca60ff2 100755 --- a/templates/mariadb/backup/check_replication.erb +++ b/templates/mariadb/backup/check_replication.erb @@ -1,6 +1,6 @@ #!/bin/bash -result="$(docker exec mariadb-db-1 mysql -p<%= @mariadb_root_password %> -BN -e 'show status like "slave_running"')" +result="$(/usr/local/bin/mariadb-replication-status)" if [[ "${result}" == "Slave_running ON" ]]; then echo "OK: Replica running" exit 0 From 0fb7530183e8cf959ad63931e36452b8e0e5481d Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Mon, 7 Oct 2024 12:07:24 +0200 Subject: [PATCH 159/161] Simplifiy integer calculation --- lib/puppet/functions/ipv4_to_int.rb | 12 ++++++++++++ manifests/mariadb.pp | 2 ++ templates/mariadb/my.cnf.erb | 3 +-- 3 files changed, 15 insertions(+), 2 deletions(-) create mode 100644 lib/puppet/functions/ipv4_to_int.rb diff --git a/lib/puppet/functions/ipv4_to_int.rb b/lib/puppet/functions/ipv4_to_int.rb new file mode 100644 index 000000000..055d0f0c6 --- /dev/null +++ b/lib/puppet/functions/ipv4_to_int.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + + +# Convert a single IPv4 address to an int + +require 'ipaddr' + +Puppet::Functions.create_function(:ipv4_to_int) do + def ipv4_to_int(*arguments) + IPAddr.new(arguments[0]).to_i + end +end diff --git a/manifests/mariadb.pp b/manifests/mariadb.pp index dd3e509d4..77426cbf3 100644 --- a/manifests/mariadb.pp +++ b/manifests/mariadb.pp @@ -93,6 +93,8 @@ content => template('sunet/mariadb/credentials.cnf.erb'), mode => '0744', } + + $server_id = ipv4_to_int($facts['networking']['ip']) file { "${mariadb_dir}/conf/my.cnf": ensure => present, content => template('sunet/mariadb/my.cnf.erb'), diff --git a/templates/mariadb/my.cnf.erb b/templates/mariadb/my.cnf.erb index 30c0664e6..be2d4015d 100644 --- a/templates/mariadb/my.cnf.erb +++ b/templates/mariadb/my.cnf.erb @@ -18,8 +18,7 @@ gtid_ignore_duplicates = ON gtid_strict_mode = ON log_bin = binlog log_slave_updates = ON -# Use IP adress as decimal in order to create an unique server id -server_id = <%= @facts['networking']['ip'].split(".").map(&:to_i).pack('CCCC').unpack('N')[0] %> +server_id = <%= @server_id %> # Default hostname base relay_log is no good in containers relay_log = 'relay-log' From c55791b2e1bff9cb103c350ab6d0724356aac456 Mon Sep 17 00:00:00 2001 From: Maria Haider Date: Fri, 4 Oct 2024 00:12:50 +0200 Subject: [PATCH 160/161] stop docker adding ipv6 rules --- templates/dockerhost/daemon2.json.erb | 1 + 1 file changed, 1 insertion(+) diff --git a/templates/dockerhost/daemon2.json.erb b/templates/dockerhost/daemon2.json.erb index 41fb419a2..f35f57ecc 100644 --- a/templates/dockerhost/daemon2.json.erb +++ b/templates/dockerhost/daemon2.json.erb @@ -5,6 +5,7 @@ "ip-forward": false, <% end -%> "iptables": false, + "ip6tables": false, "ip-masq": false, "default-address-pools": [ { From 9a0a5d7537b64539bf5efd6a02599a78f66ed56f Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Tue, 8 Oct 2024 09:48:28 +0200 Subject: [PATCH 161/161] Add network support --- manifests/invent/receiver.pp | 2 +- templates/invent/receiver/docker-compose.yml.erb | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/manifests/invent/receiver.pp b/manifests/invent/receiver.pp index 9d78a5034..9120f0d8b 100644 --- a/manifests/invent/receiver.pp +++ b/manifests/invent/receiver.pp @@ -5,7 +5,7 @@ String $vhost = 'invent.sunet.se' ){ $admin_password = lookup('invent_admin_password', undef, undef, undef) - $endpoints = ['hosts', 'images'] + $endpoints = ['hosts', 'images', 'network'] $nginx_dirs = [ 'acme', 'certs','conf','dhparam','html','vhost' ] $ni_host_key = lookup ('ni_host_key') $ni_host_ip = lookup ('ni_host_ip') diff --git a/templates/invent/receiver/docker-compose.yml.erb b/templates/invent/receiver/docker-compose.yml.erb index a6b35e09f..9ece9d2c6 100644 --- a/templates/invent/receiver/docker-compose.yml.erb +++ b/templates/invent/receiver/docker-compose.yml.erb @@ -55,8 +55,12 @@ services: stdin_open: true restart: always environment: + - INVENT_ADMIN_PASSWORD=<%= @admin_password %> + - INVENT_DB_DIR=/app/db + - INVENT_HOST_DIR=/app/hosts + - INVENT_IMAGE_DIR=/app/images + - INVENT_NETWORK_DIR=/app/network + - LETSENCRYPT_HOST=<%= @vhost %> - VIRTUAL_HOST=<%= @vhost %> - VIRTUAL_PATH=/ - VIRTUAL_PORT=8000 - - LETSENCRYPT_HOST=<%= @vhost %> - - INVENT_ADMIN_PASSWORD=<%= @admin_password %>