-
Notifications
You must be signed in to change notification settings - Fork 1
/
master.cfg
330 lines (270 loc) · 19.3 KB
/
master.cfg
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
# -*- python -*-
# ex: set syntax=python:
# Buildbot master configuration.
#
# After changing anything in this file, it should be copied to the
# Buildbot master directory, typically /var/lib/buildbot/masters/
# on the buildbot host and the buildbot restarted, using e.g.
# "sudo buildbot restart /var/lib/buildbot/masters".
######################################################################
####### PROJECT CONFIGURATION
# The email address here must be reachable by the buildmaster. (see "Mail Notifications" in the README)
NOTIFICATION_EMAIL = "topographica-cvs@lists.sourceforge.net"
GITURL = "git://github.com/ioam/topographica.git"
c = BuildmasterConfig = {} # the dictionary data structure read by buildmaster; c for convenience
# Project description
c['projectName'] = "Topographica"
c['projectURL'] = "http://www.topographica.org/"
c['buildbotURL'] = "http://buildbot.topographica.org/"
import os
#from buildbot.changes.gitpoller import GitPoller
from buildbot.status.html import WebStatus
from buildbot.status.web.authz import Authz
from buildbot.status.web.auth import BasicAuth
from buildbot.status import mail
# Git repository (see "Change Sources: GitPoller" in the Buildbot manual)
#c['change_source']= GitPoller(repourl=GITURL,pollinterval=300)
# Connection
c['slavePortnum'] = "tcp:9989" # 9989:interface=127.0.0.1 for local only
# Restricting public availability of actions (see "Web Interface" in the README)
users = [('buildbot', 'ciroux')]
authz = Authz(auth=BasicAuth(users),
forceBuild='auth', # only authenticated users
stopBuild = 'auth',
gracefulShutdown='auth',
cancelPendingBuild='auth',
pingBuilder=True, # but anyone can do this
)
# Status: web interface and mail notifications (see README plus "Status Targets" in the manual)
c['status'] = []
c['status'].append(WebStatus(8010, authz=authz, public_html="/var/www/buildbot", change_hook_dialects={'github':True}))
c['status'].append(mail.MailNotifier(fromaddr="buildbot@topographica.org",
mode=('failing'),
extraRecipients=[NOTIFICATION_EMAIL],
sendToInterestedUsers=False))
######## NAMES AND PATHS #######################################################
# The following are names and paths defined in a single place instead of copying them many times later,
# to ensure better pluggability (e.g. easily changing a slave's name) and minimise the chance of errors.
# Slave names. Definitions follow later.
MAIN_SLAVE = "doozy"
DICE_SLAVE = "mckellar"
WINXP_SLAVE = "VM_WinXP"
WIN732_SLAVE = "VM_Win7_32"
SLAVE_PASSWORD = "PASSWD"
# Files publicly available via the Buildbot website are kept here
MASTER_WEB = "/var/www/buildbot/"
# Buildbot home dirs for the various machines. Often there's a dedicated buildbot user. The main slave is currently
# on doozy (same place as the master, but they still communicate over TCP as if the slave were elsewhere)
MAIN_SLAVE_HOME = "/var/lib/buildbot/"
DICE_SLAVE_HOME = "/disk/scratch/topo-buildbot/"
SLAVE_COVERAGE_DIR = "htmlcov" # This is how nose-cov (see topo/tests/unit/README) names the directory containing coverage reports
SLAVE_DOC_DIR = "doc" # This is where compiled docs appear
# upload coverage reports and docs to the public dir on the master:
MASTER_COVERAGE_HTML = MASTER_WEB + "coverage"
####### COMMANDS #####################################################
# Use this space to define commands that are reused or repeated many
# times (but don't create unnecessary variables for commands that only
# appear once or twice!
# Commands are a bit different on Linux and Windows. E.g. Windows
# requires typing "python topographica" while Linux requires "./" with
# the script name. See topo/tests/README and topo/tests/unit/README
# to find out about the different types of tests.
CMD_UNIX_UNIT = ["xvfb-run", "-a", "./topographica", "-t", "unit"]
CMD_UNIX_ALL = ["xvfb-run", "-a", "./topographica", "-t", "exhaustive"]
CMD_UNIX_SPEED = ["xvfb-run", "-a", "./topographica", "-t", "speed"]
CMD_WIN_UNIT = ["python", "topographica", "-t", "unit"]
CMD_WIN_ALL = ["python", "topographica", "-t", "exhaustive"]
CMD_WIN_SPEED = ["python", "topographica", "-t", "speed"]
CMD_UNIT_COVERAGE = ["xvfb-run", "-a", "./topographica", "-t", "coverage"]
CMD_MAKE_DOC = ["make", "-C", "doc", "html-doc"]
CMD_MAKE_IPYNB = ["make", "-C", "doc", "ipynb-doc"]
####### SLAVES #######################################################
# Defining slaves using the previously listed names and
# passwords. When starting a slave, the name and password must already
# be defined here.
from buildbot import locks
from buildbot.buildslave import BuildSlave
c['slaves' ] = [
BuildSlave(MAIN_SLAVE, SLAVE_PASSWORD, notify_on_missing=NOTIFICATION_EMAIL),
BuildSlave(DICE_SLAVE, SLAVE_PASSWORD, notify_on_missing=NOTIFICATION_EMAIL),
# No "notify_on_missing" enabled for VMs because they are shut down after every use and would generate unneeded mail traffic.
BuildSlave(WINXP_SLAVE, SLAVE_PASSWORD),
BuildSlave(WIN732_SLAVE, SLAVE_PASSWORD)
]
# Set a lock if more than one build will be taking place on a given machine.
MAIN_SLAVE_LOCK = locks.SlaveLock(MAIN_SLAVE, maxCount=1)
####### FACTORIES ####################################################
# See "Build Factories" and "Build Steps" in the Buildbot manual. Essentially, a factory is a collection of buildsteps to be performed.
# These are later attached to builders.
import datetime
from buildbot.process.factory import BuildFactory
from buildbot.steps.transfer import DirectoryUpload
from buildbot.steps.shell import ShellCommand
from buildbot.steps.master import MasterShellCommand
from buildbot.steps.source.git import Git
from buildbot.steps.slave import CopyDirectory
# BACKUPS: clone repository and copy it to a dir named after today's date.
# Since this is a one-time configuration, the buildbot needs to be restarted daily (see "Crontab" in the README) to get today's date
# and use new dir name instead of the one defined during the previous restart.
date = datetime.datetime.now()
backups_factory = BuildFactory()
# All builds start with the source checkout step. Upon each build, the previous copy of the repo is deleted (see logs) and a fresh copy is obtained.
backups_factory.addStep(Git(repourl = GITURL, mode = 'full', method = 'clobber', submodules = True, haltOnFailure = True, retry = (10,2)))
# CopyDirectory not available until next release after 0.8.7p1; library on doozy is modified! See Work_on_CI.rst
backups_factory.addStep(CopyDirectory(src="build", dest=date.strftime("%Y-%m-%d")))
# Locations to push the built docs from
MASTER_DOC = '/home/topo-buildbot/docbuilds/'
TOPO_DOC = os.path.join(MASTER_DOC, 'topodoc')
PARAM_DOC = os.path.join(MASTER_DOC, 'paramdoc')
DATAVIEWS_DOC = os.path.join(MASTER_DOC, 'dataviewsdoc')
IMAGEN_DOC = os.path.join(MASTER_DOC, 'imagendoc')
FEATUREMAPPER_DOC = os.path.join(MASTER_DOC, 'featuremapperdoc')
LANCET_DOC = os.path.join(MASTER_DOC, 'lancetdoc')
# GIT REPOSITORIES
GITURL = "git://github.com/ioam/topographica.git"
PARAM_GIT = "git://github.com/ioam/param.git"
DATAVIEWS_GIT = "git://github.com/ioam/dataviews.git"
IMAGEN_GIT = "git://github.com/ioam/imagen.git"
FEATUREMAPPER_GIT = "git://github.com/ioam/featuremapper.git"
LANCET_GIT = "git://github.com/ioam/lancet.git"
BUILD_DIR = '/var/lib/buildbot/slaves/topographica_docs/build/'
PARAM_DIR = '/var/lib/buildbot/slaves/param_docs/build/'
DATAVIEWS_DIR = '/var/lib/buildbot/slaves/dataviews_docs/build/'
IMAGEN_DIR = '/var/lib/buildbot/slaves/imagen_docs/build/'
FEATUREMAPPER_DIR = '/var/lib/buildbot/slaves/featuremapper_docs/build/'
LANCET_DIR = '/var/lib/buildbot/slaves/lancet_docs/build/'
REF_DIR = 'doc/reference_data'
doc_config = {'topographica': {'build_dir': BUILD_DIR, 'push_dir': TOPO_DOC, 'git': GITURL, 'unit': CMD_UNIX_UNIT, 'notebooks': True, 'submodules': {}},
'param': {'build_dir': PARAM_DIR, 'push_dir': PARAM_DOC, 'git': PARAM_GIT, 'unit': ['nosetests'], 'notebooks': False, 'submodules': {}},
'dataviews': {'build_dir': DATAVIEWS_DIR, 'push_dir': DATAVIEWS_DOC, 'git': DATAVIEWS_GIT, 'unit': ['nosetests'], 'notebooks': True, 'submodules': {'param': PARAM_GIT}},
'imagen': {'build_dir': IMAGEN_DIR, 'push_dir': IMAGEN_DOC, 'git': IMAGEN_GIT, 'unit': None, 'notebooks': True, 'submodules':{'param': PARAM_GIT, 'dataviews': DATAVIEWS_GIT}},
'featuremapper': {'build_dir': FEATUREMAPPER_DIR, 'push_dir': FEATUREMAPPER_DOC, 'git': FEATUREMAPPER_GIT, 'unit': None, 'notebooks': True, 'submodules':{'param': PARAM_GIT, 'dataviews': DATAVIEWS_GIT, 'imagen': IMAGEN_GIT}},
'lancet': {'build_dir': LANCET_DIR, 'push_dir': LANCET_DOC, 'git': LANCET_GIT, 'unit': None, 'notebooks': False, 'submodules': {'param': PARAM_GIT}}}
docbuilder_env={'MPLCONFIGDIR': "/home/topo-buildbot/.config/matplotlib/", 'XDG_CONFIG_HOME': '/home/topo-buildbot/.config/', 'PAGER': 'more'}
docbuild_factories = {}
for project, config in doc_config.items():
docs_factory = BuildFactory()
#haltOnFailure should be enabled for critical steps (such as source checkout) if the build cannot continue should they fail
docs_factory.addStep(Git(repourl=config['git'], mode='full', method='clobber', submodules=False, haltOnFailure=True, retry=(10,2)))
for submodule, submodule_git in config['submodules'].items():
docs_factory.addStep(ShellCommand(command=["git", "submodule", "add", submodule_git, submodule], workdir=config['build_dir'], haltOnFailure=True))
docs_factory.addStep(ShellCommand(command=["git", "submodule", "update", "--init"], workdir=config['build_dir'], haltOnFailure = True))
if config['notebooks']:
docs_factory.addStep(ShellCommand(command=["git", "remote", "set-url", "origin", "git@github.com:ioam/ioam.github.com.git"], workdir=os.path.join(config['build_dir'], REF_DIR), haltOnFailure = True))
if config['unit']:
# Unit tests. Always enable flunkOnFailure for test steps because that will make the build fail if tests fail and will generate a notification.
docs_factory.addStep(ShellCommand(command=config['unit'], description = "unit tests", flunkOnFailure = True))
# Building documentation. Ask someone how that works exactly; I am not entirely clear.
docs_factory.addStep(ShellCommand(command=CMD_MAKE_DOC, description="Build %s docs" % project, haltOnFailure = True, flunkOnFailure = True, env=docbuilder_env))
# Delete previously generated docs on the master:
docs_factory.addStep(ShellCommand(command=["rm", "-rf", config['push_dir']]))
# Pull the "gh-pages" branch from GitHub
docs_factory.addStep(Git(repourl = config['git'], mode = 'full', method = 'clobber', submodules = False, haltOnFailure = True, branch = 'gh-pages', retry = (10,2), workdir = config['push_dir']))
# # The above does not honour the "branch" argument, force the gh-pages branch
docs_factory.addStep(ShellCommand(command=["git", "checkout", "gh-pages"], workdir=config['push_dir']))
# Remove all the old files from the gh-pages branch
#docs_factory.addStep(ShellCommand(command=["git", "rm", "-r", "*"], workdir=TOPO_DOC))
# Upload new docs to the public dir on the master:
docs_factory.addStep(ShellCommand(command="""cp -r %s %s""" % (os.path.join(config['build_dir'], 'doc/_build/html/*'), config['push_dir']), haltOnFailure = True))
# Commit and push the changes back to GH Pages
docs_factory.addStep(ShellCommand(command=["git", "add", "*"], workdir = config['push_dir']))
docs_factory.addStep(ShellCommand(command=["git", "commit", "-am", "Automatic commit "+date.strftime("%Y-%m-%d %H:%M:%S")], workdir = config['push_dir']))
docs_factory.addStep(ShellCommand(command=["git", "remote", "set-url", "origin", "git@github.com:ioam/%s.git" % project], workdir = config['push_dir']))
docs_factory.addStep(ShellCommand(command=["git", "push", "origin", "gh-pages"], workdir = config['push_dir']))
# Build the projects notebooks
if config['notebooks']:
docs_factory.addStep(ShellCommand(command=CMD_MAKE_IPYNB, description="Build %s IPYNB" % project, haltOnFailure = True, flunkOnFailure = True, env=docbuilder_env))
docbuild_factories[project] = docs_factory
# COVERAGE: produce reports for code coverage (see topo/tests/unit/README)
coverage_factory = BuildFactory()
coverage_factory.addStep(Git(repourl = GITURL, mode = 'full', method = 'clobber', submodules = True, haltOnFailure = True, retry = (10,2)))
# coverage-enabled unit tests:
coverage_factory.addStep(ShellCommand(command=CMD_UNIT_COVERAGE, description="unit tests", haltOnFailure = True, flunkOnFailure=True))
# same procedure as for docs to upload the new reports:
coverage_factory.addStep(MasterShellCommand(command=["rm", "-rf", MASTER_COVERAGE_HTML]))
coverage_factory.addStep(DirectoryUpload(slavesrc = SLAVE_COVERAGE_DIR, masterdest = MASTER_COVERAGE_HTML))
coverage_factory.addStep(MasterShellCommand(command=["chmod", "-R", "o=rx", MASTER_COVERAGE_HTML]))
# FULL BUILD: build all of Topographica's external dependencies and perform the full suite of tests.
full_build_factory = BuildFactory()
full_build_factory.addStep(Git(repourl = GITURL, mode = 'full', method = 'clobber', submodules = True, haltOnFailure = True, retry = (10,2)))
# perform code analysis:
#full_build_factory.addStep(ShellCommand(command=["make", "lint-base"], description = "lint-base", flunkOnFailure = True))
full_build_factory.addStep(ShellCommand(command=["./topographica", "-t", "flakes"], description = "pyflakes", flunkOnFailure = True))
# run all tests:
full_build_factory.addStep(ShellCommand(command=CMD_UNIX_UNIT, description = "unit tests", flunkOnFailure = True))
full_build_factory.addStep(ShellCommand(command=CMD_UNIX_ALL, description = "system tests", flunkOnFailure = True))
full_build_factory.addStep(ShellCommand(command=CMD_UNIX_SPEED, description = "speed tests", flunkOnFailure = True))
# PLATFORM SLAVES: they only need to perform a source checkout and run the tests.
dice_factory = BuildFactory()
dice_factory.addStep(Git(repourl = GITURL, mode = "full", method = "clobber", submodules = True, haltOnFailure = True, retry = (10,2)))
dice_factory.addStep(ShellCommand(command=CMD_UNIX_UNIT, description="unit tests", flunkOnFailure=True))
dice_factory.addStep(ShellCommand(command=CMD_UNIX_ALL, description="system tests", flunkOnFailure=True))
dice_factory.addStep(ShellCommand(command=CMD_UNIX_SPEED, description="speed tests", flunkOnFailure=True))
windows_factory = BuildFactory()
windows_factory.addStep(Git(repourl = GITURL, mode = "full", method = "clobber", submodules = True, haltOnFailure = True, retry = (10,2)))
windows_factory.addStep(ShellCommand(command=CMD_WIN_UNIT, description="unit tests", flunkOnFailure=True))
windows_factory.addStep(ShellCommand(command=CMD_WIN_ALL, description="system tests", flunkOnFailure=True))
windows_factory.addStep(ShellCommand(command=CMD_WIN_SPEED, description="speed tests", flunkOnFailure=True))
####### BUILDERS #####################################################
# Builders (see "Builder Configuration" in the manual) run tasks on the slaves, using the previously defined factories.
# The latter can be reused across multiple builders.
from buildbot.config import BuilderConfig
c['builders'] = [
# You'll need the name of the slave to which the builder is attached, the factory it uses, and any locks that are enabled.
# You can modify the build environment using the 'env' option by stating e.g. where HOME is and what is available in the PATH.
BuilderConfig(name = "full_build", slavename = MAIN_SLAVE, factory = full_build_factory,
locks = [MAIN_SLAVE_LOCK.access('counting')], env = {'HOME': MAIN_SLAVE_HOME + "full_build", 'PATH': "/usr/local/bin:/usr/bin:/bin"}),
# Here, different builders on the main slave use different homes to avoid potential conflicts with files generated
# by Topographica.
BuilderConfig(name = "coverage", slavename = MAIN_SLAVE, factory = coverage_factory,
locks = [MAIN_SLAVE_LOCK.access('counting')], env = {'HOME': MAIN_SLAVE_HOME + "coverage", 'PATH': "/usr/local/bin:/usr/bin:/bin"}),
BuilderConfig(name = "backups", slavename = MAIN_SLAVE, factory = backups_factory,
locks = [MAIN_SLAVE_LOCK.access('counting')], env = {'HOME': MAIN_SLAVE_HOME + "backups", 'PATH': "/usr/local/bin:/usr/bin:/bin"}),
BuilderConfig(name="DICE_alltests", slavename=DICE_SLAVE, factory=dice_factory),
BuilderConfig(name="WinXP_alltests", slavename=WINXP_SLAVE, factory=windows_factory),
BuilderConfig(name="Win7_32_alltests", slavename=WIN732_SLAVE, factory=windows_factory)
]
for project, factory in docbuild_factories.items():
c['builders'].append(BuilderConfig(name = "%s_docs" % project, slavename = MAIN_SLAVE, factory = factory,
locks = [MAIN_SLAVE_LOCK.access('counting')], env = {'HOME': MAIN_SLAVE_HOME + "docs", 'PATH': "/usr/local/bin:/usr/bin:/bin"}))
####### SCHEDULERS ###################################################
# Only nightly schedulers are used; further options are available but were deemed unnecessary for the time being. See Buildbot manual for more.
from buildbot.scheduler import Nightly
from buildbot.schedulers.forcesched import ForceScheduler
from buildbot.scheduler import Scheduler
# Since all builders need the Force Scheduler, a failsafe to ensure that no builder has been left out.
ALL_BUILDERS = []
for builder in c['builders']:
ALL_BUILDERS.append(builder.name)
c['schedulers'] = []
c['schedulers'].append(Nightly(name="nightly", builderNames=["full_build", "DICE_alltests"], branch="master", hour=02, minute=00))
c['schedulers'].append(Nightly(name="monthly-backup", builderNames=["backups"], branch="master", hour=06, minute=00, dayOfMonth=1))
c['schedulers'].append(Nightly(name="docs-nightly", builderNames=["coverage", "topographica_docs", "param_docs", "dataviews_docs", "imagen_docs", "featuremapper_docs", "lancet_docs"],
branch="master", hour=03, minute=00))
c['schedulers'].append(Nightly(name="WinXP-nightly", builderNames=["WinXP_alltests"], branch="master", hour=04, minute=00))
c['schedulers'].append(Nightly(name="Win7-nightly", builderNames=["Win7_32_alltests"], branch="master", hour=05, minute=00))
# Force scheduler is required to enable "Force build" on all builders
c['schedulers'].append(ForceScheduler(name="force", builderNames=ALL_BUILDERS))
# Commit hook scheduler, triggering a doc build on commits to topographica/docs
# Currently disabled because Notebook builds are to expensive to run on every commit
#from buildbot.schedulers.filter import ChangeFilter
# CB: (I guess imports are scattered around because this file is in sections?)
# import re
# def _doc_change(change):
# # detect if change included something in doc/
# for file_ in change.files:
# if re.match('doc/',file_):
# return True
# return False
# # Hook to rebuild docs
# c['schedulers'].append(Scheduler(
# name="docs-hook",
# #branch="master",
# builderNames=["docs"],
# #repository="https://github.com/ioam/topographica",
# change_filter=ChangeFilter(
# repository="https://github.com/ioam/topographica",
# branch="master"
# ),
# fileIsImportant = _doc_change
# ))