-
Notifications
You must be signed in to change notification settings - Fork 1
/
fabfile.py
284 lines (245 loc) · 9.7 KB
/
fabfile.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
import os
import tempfile
import yaml
from fabric.api import env, execute, get, hide, lcd, local, put, require, run, settings, sudo, task
from fabric.colors import red
from fabric.contrib import files, project
from fabric.contrib.console import confirm
from fabric.utils import abort
PROJECT_ROOT = os.path.dirname(__file__)
CONF_ROOT = os.path.join(PROJECT_ROOT, 'conf')
VALID_ROLES = (
'salt-master',
'web',
'worker',
'balancer',
'db-master',
'queue',
'cache',
'sms-gateway',
)
# FIXME: Once the master has been setup this should be set to IP of the master
# This assumes a single master for both staging and production
env.master = '107.170.161.35'
@task
def staging():
env.environment = 'staging'
@task
def production():
env.environment = 'production'
@task
def vagrant():
env.environment = 'local_vagrant'
env.master = '33.33.33.10'
env.user = 'vagrant'
vagrant_version = local('vagrant -v', capture=True).split()[-1]
env.key_filename = '/opt/vagrant/embedded/gems/gems/vagrant-%s/keys/vagrant' % vagrant_version
@task
def setup_master():
"""Provision master with salt-master."""
with settings(warn_only=True):
with hide('running', 'stdout', 'stderr'):
installed = run('which salt')
if not installed:
sudo('apt-get update -qq -y')
sudo('apt-get install python-software-properties -qq -y')
sudo('add-apt-repository ppa:saltstack/salt -y')
sudo('apt-get update -qq')
sudo('apt-get install salt-master -qq -y')
# make sure git is installed for gitfs
with settings(warn_only=True):
with hide('running', 'stdout', 'stderr'):
installed = run('which git')
if not installed:
sudo('apt-get install python-pip git-core -qq -y')
sudo('pip install -q -U GitPython')
put(local_path='conf/master.conf', remote_path="/etc/salt/master", use_sudo=True)
sudo('service salt-master restart')
@task
def sync():
"""Rysnc local states and pillar data to the master."""
# Check for missing local secrets so that they don't get deleted
# project.rsync_project fails if host is not set
with settings(host=env.master, host_string=env.master):
if not have_secrets():
get_secrets()
else:
# Check for differences in the secrets files
for environment in ['staging', 'production']:
remote_file = os.path.join('/srv/pillar/', environment, 'secrets.sls')
with lcd(os.path.join(CONF_ROOT, 'pillar', environment)):
if files.exists(remote_file):
get(remote_file, 'secrets.sls.remote')
else:
local('touch secrets.sls.remote')
with settings(warn_only=True):
result = local('diff -u secrets.sls.remote secrets.sls')
msg = "Above changes will be made to secrets.sls. Continue?"
if result.failed and not confirm(red(msg)):
abort("Aborted. File have been copied to secrets.sls.remote. "
"Resolve conflicts, then retry.")
else:
local("rm secrets.sls.remote")
salt_root = CONF_ROOT if CONF_ROOT.endswith('/') else CONF_ROOT + '/'
project.rsync_project(local_dir=salt_root, remote_dir='/tmp/salt', delete=True)
sudo('rm -rf /srv/salt /srv/pillar')
sudo('mv /tmp/salt/* /srv/')
sudo('rm -rf /tmp/salt/')
def have_secrets():
"""Check if the local secret files exist for all environments."""
found = True
for environment in ['staging', 'production']:
local_file = os.path.join(CONF_ROOT, 'pillar', environment, 'secrets.sls')
found = found and os.path.exists(local_file)
return found
@task
def get_secrets():
"""Grab the latest secrets file from the master."""
with settings(host=env.master, host_string=env.master):
for environment in ['staging', 'production']:
local_file = os.path.join(CONF_ROOT, 'pillar', environment, 'secrets.sls')
if os.path.exists(local_file):
local('cp {0} {0}.bak'.format(local_file))
remote_file = os.path.join('/srv/pillar/', environment, 'secrets.sls')
get(remote_file, local_file)
@task
def setup_minion(*roles):
"""Setup a minion server with a set of roles."""
require('environment')
for r in roles:
if r not in VALID_ROLES:
abort('%s is not a valid server role for this project.' % r)
# install salt minion if it's not there already
with settings(warn_only=True):
with hide('running', 'stdout', 'stderr'):
installed = run('which salt-call')
if not installed:
# install salt-minion from PPA
sudo('apt-get update -qq -y')
sudo('apt-get install python-software-properties -qq -y')
sudo('add-apt-repository ppa:saltstack/salt -y')
sudo('apt-get update -qq')
sudo('apt-get install salt-minion -qq -y')
config = {
'master': 'localhost' if env.master == env.host else env.master,
'output': 'mixed',
'grains': {
'environment': env.environment,
'roles': list(roles),
},
'mine_functions': {
'network.interfaces': []
},
}
_, path = tempfile.mkstemp()
with open(path, 'w') as f:
yaml.dump(config, f, default_flow_style=False)
put(local_path=path, remote_path="/etc/salt/minion", use_sudo=True)
sudo('service salt-minion restart')
# queries server for its fully qualified domain name to get minion id
key_name = run('python -c "import socket; print socket.getfqdn()"')
execute(accept_key, key_name)
@task
def add_role(name):
"""Add a role to an exising minion configuration."""
if name not in VALID_ROLES:
abort('%s is not a valid server role for this project.' % name)
_, path = tempfile.mkstemp()
get("/etc/salt/minion", path)
with open(path, 'r') as f:
config = yaml.safe_load(f)
grains = config.get('grains', {})
roles = grains.get('roles', [])
if name not in roles:
roles.append(name)
else:
abort('Server is already configured with the %s role.' % name)
grains['roles'] = roles
config['grains'] = grains
with open(path, 'w') as f:
yaml.dump(config, f, default_flow_style=False)
put(local_path=path, remote_path="/etc/salt/minion", use_sudo=True)
sudo('service salt-minion restart')
@task
def remove_role(name):
"""Remove a role from an exising minion configuration."""
if name not in VALID_ROLES:
abort('%s is not a valid server role for this project.' % name)
_, path = tempfile.mkstemp()
get("/etc/salt/minion", path)
with open(path, 'r') as f:
config = yaml.safe_load(f)
grains = config.get('grains', {})
roles = grains.get('roles', [])
if name not in roles:
abort('Server not configured for the %s role.' % name)
else:
roles.remove(name)
grains['roles'] = roles
config['grains'] = grains
with open(path, 'w') as f:
yaml.dump(config, f, default_flow_style=False)
put(local_path=path, remote_path="/etc/salt/minion", use_sudo=True)
sudo('service salt-minion restart')
@task
def salt(cmd, target="'*'"):
"""Run arbitrary salt commands."""
with settings(warn_only=True, host_string=env.master):
sudo("salt {0} {1}".format(target, cmd))
@task
def highstate(target="'*'"):
"""Run highstate on master."""
with settings(host_string=env.master):
print("This can take a long time without output, be patient")
salt('state.highstate', target)
@task
def accept_key(name):
"""Accept minion key on master."""
with settings(host_string=env.master):
sudo('salt-key --accept={0} -y'.format(name))
sudo('salt-key -L')
@task
def delete_key(name):
"""Delete specific key on master."""
with settings(host_string=env.master):
sudo('salt-key -L')
sudo('salt-key --delete={0} -y'.format(name))
sudo('salt-key -L')
@task
def deploy():
"""Deploy to a given environment by pushing the latest states and executing the highstate."""
require('environment')
with settings(host_string=env.master):
sync()
target = "-G 'environment:{0}'".format(env.environment)
salt('saltutil.sync_all', target)
highstate(target)
@task
def copy_prod_db_to_staging():
""" quick hack to copy prod db to staging server """
import yaml
config = yaml.load(open(os.path.join(os.path.dirname(__file__), 'conf',
'pillar', 'staging', 'secrets.sls')))
dbpass = config['secrets']['DB_PASSWORD']
local('ssh myvoice-staging.caktusgroup.com '
'"sudo -u postgres dropdb myvoice_staging"')
local('ssh myvoice-staging.caktusgroup.com '
'"sudo -u postgres createdb -E UTF8 -O myvoice_staging myvoice_staging"')
local('ssh myvoice-staging.caktusgroup.com '
'"sudo -u postgres psql myvoice_staging -c \'CREATE EXTENSION postgis;\'"')
local('ssh -C myvoicenigeria.com '
'"sudo -u postgres pg_dump -Ox myvoice_production" | '
'ssh -C myvoice-staging.caktusgroup.com '
'"PGPASSWORD={dbpass} psql -U myvoice_staging myvoice_staging"'.format(dbpass=dbpass))
@task
def download_prod_db(filename):
"""
Download a copy of the prod db to the named file and save to the named file.
Sample usage:
fab download_prod_db:prod.sql
dropdb myvoice
createdb -E UTF8 myvoice
psql myvoice < prod.sql
"""
local('ssh -C myvoicenigeria.com '
'"sudo -u postgres pg_dump -Ox myvoice_production" > {}'.format(filename))