Skip to content

Commit

Permalink
Merge pull request #120 from jordiprats/master
Browse files Browse the repository at this point in the history 
improved pgbouncer support
  • Loading branch information
@jordiprats
jordiprats authored Jun 17, 2020
2 parents 2c9455c + 7efba06 commit 0ec419e
Show file tree
Hide file tree
Showing 16 changed files with 344 additions and 38 deletions.
4 changes: 4 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
# CHANGELOG

## 0.5.2

* improved pgbouncer support with better control over the settings

## 0.5.1

* fix raspbian bug
Expand Down
64 changes: 64 additions & 0 deletions examples/pgbouncer_session_demo.pp
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
# [root@ip-172-31-33-72 puppet-masterless]# ./localpuppetmaster.sh -d /tmp/postgres -r https://github.com/jordiprats/eyp-postgresql -s /tmp/postgres/modules/postgresql/examples/pgbouncer_standalone_demo.pp
#
# Checking Puppetfile syntax:
# Syntax OK
# Cleanup postgresql module
# Notice: Preparing to uninstall 'eyp-postgresql' ...
# Removed 'eyp-postgresql' (v0.5.1) from /tmp/postgres/modules
# Installing puppet module using a Puppetfile
# Installing dependencies
# Dependencies installed
# Warning: The string '3787.804688' was automatically coerced to the numerical value 3787.804688 (file: /tmp/postgres/modules/postgresql/manifests/init.pp, line: 33, column: 62)
# Warning: The string '3787.804688' was automatically coerced to the numerical value 3787.804688 (file: /tmp/postgres/modules/postgresql/manifests/init.pp, line: 34, column: 70)
# Warning: The string '4096' was automatically coerced to the numerical value 4096 (file: /tmp/postgres/modules/postgresql/manifests/init.pp, line: 34, column: 110)
# Warning: The string '3787.804688' was automatically coerced to the numerical value 3787.804688 (file: /tmp/postgres/modules/postgresql/manifests/init.pp, line: 107, column: 77)
# Warning: The string '946.951172' was automatically coerced to the numerical value 946.951172 (file: /tmp/postgres/modules/postgresql/manifests/init.pp, line: 118, column: 77)
# Notice: Compiled catalog for ip-172-31-33-72.eu-west-1.compute.internal in environment production in 0.37 seconds
# Notice: Applied catalog in 0.73 seconds
# [root@ip-172-31-33-72 puppet-masterless]# psql -U demo -p 6432 -h 127.0.0.1 -d demo
# Password for user demo:
# psql (11.8)
# Type "help" for help.
#
# demo=>


postgresql::role { 'demo':
password => 'demopass',
}

postgresql::db { 'demo':
owner => 'demo',
pgbouncer_tag => 'demopgbouncer',
}

postgresql::hba_rule { 'postgres trust localhost':
user => 'postgres',
database => 'all',
address => '127.0.0.1/32',
auth_method => 'trust',
order => 0,
}

postgresql::hba_rule { 'all':
user => 'all',
database => 'all',
address => "127.0.0.1/32",
}

class { 'postgresql':
wal_level => 'hot_standby',
max_wal_senders => '3',
checkpoint_segments => '8',
wal_keep_segments => '8',
version => '11',
add_nagios_checks => false,
add_hba_default_localhost_rules => false,
}

class { 'postgresql::pgbouncer':
pool_mode => 'session',
realize_dbs_tag => 'demopgbouncer',
set_pgbouncer_password => 'pgbouncer',
enable_auth_query => true,
}
65 changes: 65 additions & 0 deletions examples/pgbouncer_transaction_demo.pp
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
# [root@ip-172-31-33-72 puppet-masterless]# ./localpuppetmaster.sh -d /tmp/postgres -r https://github.com/jordiprats/eyp-postgresql -s /tmp/postgres/modules/postgresql/examples/pgbouncer_standalone_demo.pp
#
# Checking Puppetfile syntax:
# Syntax OK
# Cleanup postgresql module
# Notice: Preparing to uninstall 'eyp-postgresql' ...
# Removed 'eyp-postgresql' (v0.5.1) from /tmp/postgres/modules
# Installing puppet module using a Puppetfile
# Installing dependencies
# Dependencies installed
# Warning: The string '3787.804688' was automatically coerced to the numerical value 3787.804688 (file: /tmp/postgres/modules/postgresql/manifests/init.pp, line: 33, column: 62)
# Warning: The string '3787.804688' was automatically coerced to the numerical value 3787.804688 (file: /tmp/postgres/modules/postgresql/manifests/init.pp, line: 34, column: 70)
# Warning: The string '4096' was automatically coerced to the numerical value 4096 (file: /tmp/postgres/modules/postgresql/manifests/init.pp, line: 34, column: 110)
# Warning: The string '3787.804688' was automatically coerced to the numerical value 3787.804688 (file: /tmp/postgres/modules/postgresql/manifests/init.pp, line: 107, column: 77)
# Warning: The string '946.951172' was automatically coerced to the numerical value 946.951172 (file: /tmp/postgres/modules/postgresql/manifests/init.pp, line: 118, column: 77)
# Notice: Compiled catalog for ip-172-31-33-72.eu-west-1.compute.internal in environment production in 0.37 seconds
# Notice: Applied catalog in 0.73 seconds
# [root@ip-172-31-33-72 puppet-masterless]# psql -U demo -p 6432 -h 127.0.0.1 -d demo
# Password for user demo:
# psql (11.8)
# Type "help" for help.
#
# demo=>


postgresql::role { 'demo':
password => 'demopass',
}

postgresql::db { 'demo':
owner => 'demo',
pgbouncer_tag => 'demopgbouncer',
}

postgresql::hba_rule { 'postgres trust localhost':
user => 'postgres',
database => 'all',
address => '127.0.0.1/32',
auth_method => 'trust',
order => 0,
}

postgresql::hba_rule { 'all':
user => 'all',
database => 'all',
address => "127.0.0.1/32",
}

class { 'postgresql':
wal_level => 'hot_standby',
max_wal_senders => '3',
checkpoint_segments => '8',
wal_keep_segments => '8',
version => '11',
add_nagios_checks => false,
add_hba_default_localhost_rules => false,
}

class { 'postgresql::pgbouncer':
pool_mode => 'transaction',
default_pool_size => 20,
realize_dbs_tag => 'demopgbouncer',
set_pgbouncer_password => 'pgbouncer',
enable_auth_query => true,
}
36 changes: 35 additions & 1 deletion lib/puppet/provider/postgresql_psql/postgresql_psql.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,15 +10,40 @@ def run_unless_sql_command(sql)
run_sql_command('SELECT COUNT(*) FROM (' << sql << ') count')
end

def run_sql_file(sql)
if resource[:search_path]
sql = "set search_path to #{Array(resource[:search_path]).join(',')}; #{sql}"
end

command = [resource[:psql_path]]
command.push("-h", resource[:host]) if resource[:host]
command.push("-d", resource[:db]) if resource[:db]
command.push("-p", resource[:port]) if resource[:port]
command.push("-t")
command.push("-f", sql)

environment = get_environment

if resource[:cwd]
Dir.chdir resource[:cwd] do
run_command(command, resource[:psql_user], resource[:psql_group], environment)
end
else
run_command(command, resource[:psql_user], resource[:psql_group], environment)
end
end

def run_sql_command(sql)
if resource[:search_path]
sql = "set search_path to #{Array(resource[:search_path]).join(',')}; #{sql}"
end

command = [resource[:psql_path]]
command.push("-h", resource[:host]) if resource[:host]
command.push("-d", resource[:db]) if resource[:db]
command.push("-p", resource[:port]) if resource[:port]
command.push("-t", "-c", '"' + sql.gsub('"', '\"') + '"')
command.push("-t")
command.push("-c", '"' + sql.gsub('"', '\"') + '"')

environment = get_environment

Expand All @@ -31,6 +56,15 @@ def run_sql_command(sql)
end
end

def run_sql(sql)
# TODO: wrapper per executar, si el primer caracter es un / es un file
if sql[0] == '/'
run_sql_file(sql)
else
run_sql_command(sql)
end
end

private

def get_environment
Expand Down
6 changes: 5 additions & 1 deletion lib/puppet/type/postgresql_psql.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ def retrieve
end

def sync
output, status = provider.run_sql_command(value)
output, status = provider.run_sql(value)
self.fail("Error executing SQL; psql returned #{status}: '#{output}'") unless status == 0
end
end
Expand Down Expand Up @@ -74,6 +74,10 @@ def matches(value)
desc "The name of the database to execute the SQL command against, this overrides any PGDATABASE value in connect_settings"
end

newparam(:host) do
desc "The name of the host to execute the SQL command against"
end

newparam(:port) do
desc "The port of the database server to execute the SQL command against, this overrides any PGPORT value in connect_settings."
end
Expand Down
21 changes: 19 additions & 2 deletions manifests/db.pp
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,11 @@
define postgresql::db (
$owner,
$dbname = $name,
$port = $postgresql::port,
$dbname = $name,
$port = $postgresql::port,
$pgbouncer_tag = undef,
$pgbouncer_addr = '127.0.0.1',
$pgbouncer_auth_user = 'pgbouncer',
$pgbouncer_enable_get_auth = true,
) {

Postgresql_psql {
Expand All @@ -24,4 +28,17 @@
{
Postgresql::Role[$owner] -> Postgresql_psql["ALTER DATABASE ${dbname} OWNER TO ${owner}"]
}

if($pgbouncer_tag!=undef)
{
@postgresql::pgbouncer::database { "pgbouncer-${dbname}-${pgbouncer_addr}-${pgbouncer_tag}":
host => $pgbouncer_addr,
auth_user => $pgbouncer_auth_user,
port => $port,
database => $dbname,
remote_database => $dbname,
tag => $pgbouncer_tag,
enable_get_auth => $pgbouncer_enable_get_auth,
}
}
}
13 changes: 10 additions & 3 deletions manifests/init.pp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,9 +127,12 @@
$hot_standby = false,
$max_standby_archive_delay = '30s',
$max_standby_streaming_delay = '30s',
$add_nagios_checks = true,
$ensure_nagios_checks = 'present',
$basedir_nagios_checks = '/usr/local/bin',
$add_nrpe_sudos_nagios_checks = true,
$add_hba_default_local_rules = true,
$add_hba_default_localhost_rules = true,
$default_local_authmethod = 'trust',
$pause_replica = undef,
) inherits postgresql::params {
Expand Down Expand Up @@ -232,9 +235,13 @@
}
}

class { '::postgresql::checks':
ensure => $ensure_nagios_checks,
basedir => $basedir_nagios_checks,
if($add_nagios_checks)
{
class { '::postgresql::checks':
ensure => $ensure_nagios_checks,
basedir => $basedir_nagios_checks,
add_nrpe_sudos => $add_nrpe_sudos_nagios_checks,
}
}

class { '::postgresql::install': }
Expand Down
36 changes: 24 additions & 12 deletions manifests/pgbouncer.pp
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,28 @@
class postgresql::pgbouncer (
$manage_package = true,
$package_ensure = 'installed',
$manage_service = true,
$manage_docker_service = true,
$service_ensure = 'running',
$service_enable = true,
$auth_type = 'md5',
$listen_addr = '127.0.0.1',
$listen_port = '6432',
$logfile = '/var/log/pgbouncer/pgbouncer.log',
$pool_mode = 'session',
) inherits postgresql::pgbouncer::params {
$manage_package = true,
$package_ensure = 'installed',
$manage_service = true,
$manage_docker_service = true,
$service_ensure = 'running',
$service_enable = true,
$auth_type = 'md5',
$auth_user = undef,
$enable_auth_query = false,
$auth_query = 'SELECT * FROM pgbouncer.get_auth($1)',
$listen_addr = '127.0.0.1',
$listen_port = '6432',
$logfile = '/var/log/pgbouncer/pgbouncer.log',
$pool_mode = 'session',
$realize_dbs_tag = undef,
$realize_users_tag = undef,
$set_pgbouncer_password = undef,
$dbhost_pgbouncer = '127.0.0.1',
$src_ip_pgbouncer = '127.0.0.1',
$verbose = '0',
$server_fast_close = false,
$max_client_conn = 200,
$default_pool_size = 100,
) inherits postgresql::params {

class { '::postgresql::pgbouncer::install': } ->
class { '::postgresql::pgbouncer::config': } ~>
Expand Down
50 changes: 45 additions & 5 deletions manifests/pgbouncer/config.pp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,50 @@
content => template("${module_name}/pgbouncer/databases-header.erb"),
}

concat { '/etc/pgbouncer/userlist.txt':
ensure => 'present',
owner => 'root',
group => 'root',
mode => '0644',
concat { '/etc/pgbouncer/userlist.txt':
ensure => 'present',
owner => 'root',
group => 'root',
mode => '0644',
}

if($postgresql::pgbouncer::realize_dbs_tag!=undef)
{
Postgresql::Pgbouncer::Database <| tag == $postgresql::pgbouncer::realize_dbs_tag |>
}

if($postgresql::pgbouncer::realize_users_tag!=undef)
{
Postgresql::Pgbouncer::Username <| tag == $postgresql::pgbouncer::realize_dbs_tag |>
}

if($postgresql::pgbouncer::set_pgbouncer_password!=undef)
{
postgresql::role { 'pgbouncer':
password => $postgresql::pgbouncer::set_pgbouncer_password,
db_host => $postgresql::pgbouncer::dbhost_pgbouncer,
}

$password_hash_md5=md5("${postgresql::pgbouncer::set_pgbouncer_password}pgbouncer")
$password_hash_sql="md5${password_hash_md5}"

postgresql::pgbouncer::username { 'pgbouncer':
password_md5 => $password_hash_sql,
}

postgresql::hba_rule { 'pgbouncer':
user => 'pgbouncer',
database => 'all',
address => "${postgresql::pgbouncer::src_ip_pgbouncer}/32",
}

#user_authentication-sql.erb
file { '/etc/pgbouncer/.user_authentication.sql':
ensure => 'present',
owner => 'root',
group => 'root',
mode => '0644',
content => template("${module_name}/pgbouncer/user_authentication-sql.erb"),
}
}
}
Loading

0 comments on commit 0ec419e

Please sign in to comment.