diff --git a/lib/puppet/functions/ipv4_to_int.rb b/lib/puppet/functions/ipv4_to_int.rb new file mode 100644 index 000000000..055d0f0c6 --- /dev/null +++ b/lib/puppet/functions/ipv4_to_int.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + + +# Convert a single IPv4 address to an int + +require 'ipaddr' + +Puppet::Functions.create_function(:ipv4_to_int) do + def ipv4_to_int(*arguments) + IPAddr.new(arguments[0]).to_i + end +end diff --git a/manifests/mariadb.pp b/manifests/mariadb.pp index b96f305ed..77426cbf3 100644 --- a/manifests/mariadb.pp +++ b/manifests/mariadb.pp @@ -4,8 +4,10 @@ Integer $bootstrap=0, Array[Integer] $ports = [3306, 4444, 4567, 4568], Array[String] $dns = [], + Boolean $galera = true, ) { + $mariadb_root_password = lookup('mariadb_root_password', undef, undef,'NOT_SET_IN_HIERA') $mariadb_user = lookup('mariadb_user', undef, undef,undef) $mariadb_user_password = lookup('mariadb_user_password', undef, undef,undef) @@ -14,7 +16,6 @@ $clients = lookup('mariadb_clients', undef, undef,['127.0.0.1']) $cluster_nodes = lookup('mariadb_cluster_nodes', undef, undef,[]) $mariadb_dir = '/opt/mariadb' - $server_id = 1000 + Integer($facts['networking']['hostname'][-1]) # Hack to not clash with docker_compose which tries to create the same directory exec {'mariadb_dir_create': @@ -54,26 +55,26 @@ ok_criteria => ['exit_status=0','max_age=2d'], warn_criteria => ['exit_status=1','max_age=3d'], } - file { '/usr/local/bin/cluster-size': + file { '/usr/local/bin/mariadb-galera-size': ensure => present, content => template('sunet/mariadb/cluster-size.erb.sh'), mode => '0744', } - file { '/usr/local/bin/cluster-status': + file { '/usr/local/bin/mariadb-galera-status': ensure => present, content => template('sunet/mariadb/cluster-status.erb.sh'), mode => '0744', } - file { '/etc/sudoers.d/99-size-test': + file { '/etc/sudoers.d/99-cluster-size-test': ensure => file, - content => "script ALL=(root) NOPASSWD: /usr/local/bin/cluster-size\n", + content => "script ALL=(root) NOPASSWD: /usr/local/bin/mariadb-galera-size\n", mode => '0440', owner => 'root', group => 'root', } - file { '/etc/sudoers.d/99-status-test': + file { '/etc/sudoers.d/99-cluster-status-test': ensure => file, - content => "script ALL=(root) NOPASSWD: /usr/local/bin/cluster-status\n", + content => "script ALL=(root) NOPASSWD: /usr/local/bin/mariadb-galera-status\n", mode => '0440', owner => 'root', group => 'root', @@ -92,6 +93,8 @@ content => template('sunet/mariadb/credentials.cnf.erb'), mode => '0744', } + + $server_id = ipv4_to_int($facts['networking']['ip']) file { "${mariadb_dir}/conf/my.cnf": ensure => present, content => template('sunet/mariadb/my.cnf.erb'), diff --git a/manifests/mariadb/backup.pp b/manifests/mariadb/backup.pp new file mode 100644 index 000000000..c8ecf67fa --- /dev/null +++ b/manifests/mariadb/backup.pp @@ -0,0 +1,80 @@ +# This is a asyncronous replica of the Maria DB Cluster for SUNET +class sunet::mariadb::backup( + String $mariadb_version=latest, + Array[String] $dns = [], + Boolean $backup_to_baas = true, + Boolean $nrpe = true, +) { + + sunet::mariadb { 'sunet_mariadb_simple': + mariadb_version => $mariadb_version, + ports => [3306], + dns => $dns, + galera => false, + } + + $cluster_nodes = lookup('mariadb_cluster_nodes', undef, undef,[]) + $replicate_from = $cluster_nodes[0] + + # Secrets from local.eyaml + $mariadb_root_password = safe_hiera('mariadb_root_password') + $mariadb_backup_password = safe_hiera('mariadb_root_password') + $mariadb_user_password = safe_hiera('mariadb_user_password') + + file { '/opt/mariadb/scripts/start_replica_from_init.sh': + ensure => present, + content => template('sunet/mariadb/backup/start_replica_from_init.erb.sh'), + mode => '0744', + } + file { '/opt/mariadb/scripts/do_backup.sh': + ensure => present, + content => template('sunet/mariadb/backup/do_backup.erb.sh'), + mode => '0744', + } + + if $backup_to_baas { + file { '/usr/local/bin/backup2baas': + ensure => present, + content => template('sunet/mariadb/backup/backup2baas.erb'), + mode => '0744', + } + + sunet::scriptherder::cronjob { 'backup2baas': + cmd => '/usr/local/bin/backup2baas', + hour => '6', + minute => '10', + ok_criteria => ['exit_status=0', 'max_age=24h'], + warn_criteria => ['exit_status=1'], + } + } + + file { '/usr/lib/nagios/plugins/check_mariadb-replication': + ensure => present, + content => template('sunet/mariadb/backup/check_replication.erb'), + mode => '0744', + } + file { '/usr/local/bin/mariadb-replication-status': + ensure => present, + content => template('sunet/mariadb/backup/replication-status.erb'), + mode => '0744', + } + file { '/etc/sudoers.d/99-mariadb-replication-test': + ensure => file, + content => "script ALL=(root) NOPASSWD: /usr/local/bin/mariadb-replication-status", + mode => '0440', + owner => 'root', + group => 'root', + } + + if $nrpe { + sunet::sudoer {'nagios_run_replication_command': + user_name => 'nagios', + collection => 'nrpe_replication_check', + command_line => '/usr/lib/nagios/plugins/check_mariadb-replication' + } + sunet::nagios::nrpe_command {'check_async_replication': + command_line => '/usr/bin/sudo /usr/lib/nagios/plugins/check_mariadb-replication' + } + } + +} diff --git a/templates/mariadb/backup/backup2baas.erb b/templates/mariadb/backup/backup2baas.erb new file mode 100755 index 000000000..69a4dc757 --- /dev/null +++ b/templates/mariadb/backup/backup2baas.erb @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -e + +docker exec mariadb-db-1 /scripts/do_backup.sh + +BACKUPDIR=/opt/mariadb/backups +find "${BACKUPDIR}" -type f -mtime +31 -exec rm -f {} \; +find "${BACKUPDIR}" -empty -type d -delete + +/usr/bin/dsmc backup diff --git a/templates/mariadb/backup/check_replication.erb b/templates/mariadb/backup/check_replication.erb new file mode 100755 index 000000000..d8ca60ff2 --- /dev/null +++ b/templates/mariadb/backup/check_replication.erb @@ -0,0 +1,10 @@ +#!/bin/bash + +result="$(/usr/local/bin/mariadb-replication-status)" +if [[ "${result}" == "Slave_running ON" ]]; then + echo "OK: Replica running" + exit 0 +else + echo "CRITICAL: Replica not running" + exit 2 +fi diff --git a/templates/mariadb/backup/do_backup.erb.sh b/templates/mariadb/backup/do_backup.erb.sh new file mode 100644 index 000000000..67a77d04a --- /dev/null +++ b/templates/mariadb/backup/do_backup.erb.sh @@ -0,0 +1,12 @@ +#!/bin/bash +stream_name="mariadb-stream-$(date +%Y-%m-%dT%H.%M.%S).gz" +dump_name="mariadb-dump-$(date +%Y-%m-%dT%H.%M.%S).sql.gz" +backup_dir="/backups/$(date +%Y/%m/%d)" +mkdir -p "${backup_dir}" + +buopts="--slave-info --safe-slave-backup" +dumpopts="--dump-slave" +mysql -p"${MYSQL_ROOT_PASSWORD}" -e "stop slave" +mariadb-backup --backup ${buopts} -u root -p"${MYSQL_ROOT_PASSWORD}" --stream=xbstream | gzip >"${backup_dir}/${stream_name}" +mysqldump --all-databases --single-transaction ${dumpopts} -u root -p${MYSQL_ROOT_PASSWORD} | gzip >"${backup_dir}/${dump_name}" +mysql -p${MYSQL_ROOT_PASSWORD} -e "start slave" diff --git a/templates/mariadb/backup/replication-status.erb b/templates/mariadb/backup/replication-status.erb new file mode 100644 index 000000000..8094c9eff --- /dev/null +++ b/templates/mariadb/backup/replication-status.erb @@ -0,0 +1,3 @@ +#!/bin/bash + +docker exec mariadb-db-1 mysql -u root -p'<%= @mariadb_root_password %>' -N -B -e "show status like 'Slave_running'" diff --git a/templates/mariadb/backup/start_replica_from_init.erb.sh b/templates/mariadb/backup/start_replica_from_init.erb.sh new file mode 100644 index 000000000..3f6242c34 --- /dev/null +++ b/templates/mariadb/backup/start_replica_from_init.erb.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +mysql="mysql -u root -p${MYSQL_ROOT_PASSWORD}" +init_file='/backups/init.sql.gz' +if [[ -f ${init_file} ]]; then + ${mysql} -e "STOP SLAVE;RESET SLAVE;" + master_command=$(zgrep 'CHANGE MASTER TO MASTER_LOG_FILE' ${init_file} | sed -e 's/^-- //' -e 's/;$//') + master_command="${master_command}, MASTER_HOST='<%= @replicate_from %>', MASTER_USER='backup'" + master_command="${master_command}, MASTER_PASSWORD='<%= @mariadb_backup_password%>', MASTER_SSL=1" + master_command="${master_command}, MASTER_CONNECT_RETRY=20" + zcat ${init_file} | ${mysql} + ${mysql} -e "${master_command}" + ${mysql} -e "START SLAVE" + sleep 3s + ${mysql} -e "SHOW SLAVE STATUS\G" +fi + +exit 0 diff --git a/templates/mariadb/docker-compose_mariadb.yml.erb b/templates/mariadb/docker-compose_mariadb.yml.erb index d92245767..22d635dc8 100644 --- a/templates/mariadb/docker-compose_mariadb.yml.erb +++ b/templates/mariadb/docker-compose_mariadb.yml.erb @@ -11,7 +11,12 @@ services: - /opt/mariadb/datadir:/var/lib/mysql - /opt/mariadb/init:/docker-entrypoint-initdb.d - /opt/mariadb/scripts:/scripts +<%- if @backup -%> + - /opt/mariadb_backup/start_replica_from_init.sh:/start_replica_from_init.sh +<% end -%> +<%- if @galera -%> network_mode: host +<% end -%> <%- if !@dns.empty? -%> dns: <% @dns.each do |resolver| -%> @@ -29,8 +34,10 @@ services: <%- if @mariadb_database -%> - MYSQL_DATABASE=<%= @mariadb_database %> <%- end -%> +<%- if @galera -%> - BOOTSTRAP=<%= @bootstrap %> - FORCE_BOOTSTRAP=0 command: "--wsrep_cluster_address=gcomm://<%= @cluster_nodes.join(',') %>" tty: true +<%- end -%> diff --git a/templates/mariadb/my.cnf.erb b/templates/mariadb/my.cnf.erb index 47556ed19..be2d4015d 100644 --- a/templates/mariadb/my.cnf.erb +++ b/templates/mariadb/my.cnf.erb @@ -19,6 +19,8 @@ gtid_strict_mode = ON log_bin = binlog log_slave_updates = ON server_id = <%= @server_id %> +# Default hostname base relay_log is no good in containers +relay_log = 'relay-log' # Innodb innodb_autoinc_lock_mode = 2 @@ -34,6 +36,7 @@ innodb_rollback_on_timeout = 1 innodb_write_io_threads = 4 # CPU dependent transaction_isolation = 'READ-COMMITTED' +<% if @galera -%> # Galera wsrep_cluster_name = "Sunet_MariaDB_Cluster" wsrep_gtid_domain_id = 1000 # same on all Galera nodes in the same segment @@ -45,3 +48,4 @@ wsrep_provider_options = "gcache.size=2G;gmcast.segment=0" # gmcast.seg wsrep_slave_threads = 4 # CPU dependent wsrep_sst_method = mariabackup wsrep_sync_wait = 1 +<% end -%>