From 611281489ef646ac7bbfca70342b1b6ad498c743 Mon Sep 17 00:00:00 2001 From: FooBarQuaxx Date: Wed, 25 Apr 2018 18:52:17 +0200 Subject: [PATCH] added basic testing for docker --- .gitignore | 4 + easybuild/main.py | 14 +-- easybuild/tools/containers/__init__.py | 6 +- easybuild/tools/containers/docker.py | 24 +++--- test/framework/containers.py | 113 ++++++++++++++++++++++++- 5 files changed, 136 insertions(+), 25 deletions(-) diff --git a/.gitignore b/.gitignore index 4b67ea8dfa..31e6aff1ea 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,7 @@ build/ dist/ *egg-info/ *.swp + +Dockerfile.* +Singularity.* +test-reports/ diff --git a/easybuild/main.py b/easybuild/main.py index dc395637c1..9141ebea00 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -389,6 +389,13 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): if try_to_generate and build_specs and not generated_ecs: easyconfigs = tweak(easyconfigs, build_specs, modtool, targetdirs=tweaked_ecs_paths) + # create a container + if options.containerize: + _log.info("Creating %s container" % options.container_type) + containerize(easyconfigs, options.container_type) + cleanup(logfile, eb_tmpdir, testing) + sys.exit(0) + forced = options.force or options.rebuild dry_run_mode = options.dry_run or options.dry_run_short @@ -415,13 +422,6 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): print_msg("No easyconfigs left to be built.", log=_log, silent=testing) ordered_ecs = [] - # create a container - if options.containerize: - _log.info("Creating %s container" % options.container_type) - containerize(ordered_ecs, eb_go) - cleanup(logfile, eb_tmpdir, testing) - sys.exit(0) - # creating/updating PRs if new_update_preview_pr: if options.new_pr: diff --git a/easybuild/tools/containers/__init__.py b/easybuild/tools/containers/__init__.py index f4a01824a5..b3e1d2d77d 100644 --- a/easybuild/tools/containers/__init__.py +++ b/easybuild/tools/containers/__init__.py @@ -1,6 +1,5 @@ from vsc.utils import fancylogger -from easybuild.tools.config import build_option from easybuild.tools.config import CONT_TYPE_SINGULARITY, CONT_TYPE_DOCKER from easybuild.tools.build_log import EasyBuildError from .singularity import singularity as singularity_containerize @@ -9,16 +8,15 @@ _log = fancylogger.getLogger('tools.containers') # pylint: disable=C0103 -def containerize(easyconfigs, eb_go=None): +def containerize(easyconfigs, container_type): """ Generate container recipe + (optionally) image """ _log.experimental("support for generating container recipes and images (--containerize/-C)") - container_type = build_option('container_type') if container_type == CONT_TYPE_SINGULARITY: singularity_containerize(easyconfigs) elif container_type == CONT_TYPE_DOCKER: - docker_containerize(easyconfigs, eb_go) + docker_containerize(easyconfigs) else: raise EasyBuildError("Unknown container type specified: %s", container_type) diff --git a/easybuild/tools/containers/docker.py b/easybuild/tools/containers/docker.py index 31e8859da2..7c35820e33 100644 --- a/easybuild/tools/containers/docker.py +++ b/easybuild/tools/containers/docker.py @@ -107,14 +107,14 @@ def check_docker_containerize(): docker_path = which('docker') if not docker_path: raise EasyBuildError("docker executable not found.") - _log.debug("found docker executable '%s'" % docker_path) + print_msg("docker tool found at %s" % docker_path) sudo_path = which('sudo') if not sudo_path: raise EasyBuildError("sudo not found.") try: - run_cmd("sudo docker --version") + run_cmd("sudo docker --version", trace=False, force_in_dry_run=True) except Exception: raise EasyBuildError("Error getting docker version") @@ -130,7 +130,7 @@ def _det_os_deps(easyconfigs): return res -def generate_dockerfile(easyconfigs, container_base, eb_go): +def generate_dockerfile(easyconfigs, container_base): os_deps = _det_os_deps(easyconfigs) module_naming_scheme = ActiveMNS() @@ -141,9 +141,7 @@ def generate_dockerfile(easyconfigs, container_base, eb_go): mod_names = [e['ec'].full_mod_name for e in easyconfigs] - eb_opts = [eb_opt for eb_opt in eb_go.generate_cmd_line() - if not eb_opt.startswith('--container') and eb_opt not in ['--ignore-osdeps', '--experimental']] - eb_opts.extend(eb_go.args) + eb_opts = [os.path.basename(ec['spec']) for ec in easyconfigs] tmpl = _DOCKER_TMPLS[container_base] content = tmpl % { @@ -159,7 +157,7 @@ def generate_dockerfile(easyconfigs, container_base, eb_go): if img_name: file_label = os.path.splitext(img_name)[0] else: - file_label = mod_names[-1].replace('/', '-') + file_label = mod_names[0].replace('/', '-') dockerfile = os.path.join(cont_path, 'Dockerfile.%s' % file_label) if os.path.exists(dockerfile): @@ -169,6 +167,7 @@ def generate_dockerfile(easyconfigs, container_base, eb_go): raise EasyBuildError("Dockerfile at %s already exists, not overwriting it without --force", dockerfile) write_file(dockerfile, content) + print_msg("Dockerfile file created at %s" % dockerfile, log=_log) return dockerfile @@ -180,20 +179,23 @@ def build_docker_image(easyconfigs, dockerfile): module_name = module_naming_scheme.det_full_module_name(ec) tempdir = tempfile.mkdtemp(prefix='easybuild-docker') - container_name = build_option('container_image_name') or "%s:latest" % module_name + container_name = build_option('container_image_name') or "%s:latest" % module_name.replace('/', '-') docker_cmd = ' '.join(['sudo', 'docker', 'build', '-f', dockerfile, '-t', container_name, '.']) - run_cmd(docker_cmd, path=tempdir) + + print_msg("Running '%s', you may need to enter your 'sudo' password..." % docker_cmd) + run_cmd(docker_cmd, path=tempdir, stream_output=True) + print_msg("Docker image created at %s" % container_name, log=_log) shutil.rmtree(tempdir) -def docker_containerize(easyconfigs, eb_go): +def docker_containerize(easyconfigs): check_docker_containerize() # Generate dockerfile container_base = build_option('container_base') or DEFAULT_DOCKER_BASE_IMAGE - dockerfile = generate_dockerfile(easyconfigs, container_base, eb_go) + dockerfile = generate_dockerfile(easyconfigs, container_base) # Build image if requested if build_option('container_build_image'): diff --git a/test/framework/containers.py b/test/framework/containers.py index 006105867f..655351aaf8 100644 --- a/test/framework/containers.py +++ b/test/framework/containers.py @@ -59,6 +59,12 @@ fi """ +MOCKED_DOCKER = """\ +echo "docker was called with arguments: $@" +echo "$@" +echo $# +""" + class ContainersTest(EnhancedTestCase): """Tests for containers support""" @@ -89,11 +95,11 @@ def test_parse_container_base(self): expected.update({'arg2': 'bar'}) self.assertEqual(parse_container_base('%s:foo:bar' % agent), expected) - def run_main(self, args): + def run_main(self, args, raise_error=False): """Helper function to run main with arguments specified in 'args' and return stdout/stderr.""" self.mock_stdout(True) self.mock_stderr(True) - self.eb_main(args, raise_error=True, verbose=True, do_build=True) + self.eb_main(args, raise_error=raise_error, verbose=True, do_build=True) stdout = self.get_stdout().strip() stderr = self.get_stderr().strip() self.mock_stdout(False) @@ -120,6 +126,7 @@ def test_end2end_singularity_recipe(self): args = [ toy_ec, '--containerize', + '--container-type=singularity', '--experimental', ] @@ -138,7 +145,7 @@ def test_end2end_singularity_recipe(self): remove_file(os.path.join(self.test_prefix, 'containers', 'Singularity.toy-0.0')) args.append("--container-base=shub:test123") - self.run_main(args) + self.run_main(args, raise_error=True) # existing definition file is not overwritten without use of --force error_pattern = "Container recipe at .* already exists, not overwriting it without --force" @@ -204,6 +211,7 @@ def test_end2end_singularity_image(self): toy_ec, '-C', # equivalent with --containerize '--experimental', + '--container-type=singularity', '--container-base=localimage:%s' % test_img, '--container-build-image', ] @@ -282,6 +290,105 @@ def test_end2end_singularity_image(self): stdout, stderr = self.run_main(args) self.assertFalse(stderr) regexs[-3] = "^== Running 'sudo\s*SINGULARITY_TMPDIR=%s \S*/singularity build .*" % self.test_prefix + + def test_end2end_dockerfile(self): + test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') + toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') + + containerpath = os.path.join(self.test_prefix, 'containers') + os.environ['EASYBUILD_CONTAINERPATH'] = containerpath + # --containerpath must be an existing directory (this is done to avoid misconfiguration) + mkdir(containerpath) + + base_args = [ + toy_ec, + '--containerize', + '--container-type=docker', + '--experimental', + ] + + error_pattern = "Unsupported container base image 'not-supported'" + self.assertErrorRegex(EasyBuildError, + error_pattern, + self.eb_main, + base_args + ['--container-base=not-supported'], + raise_error=True) + + for cont_base in ['ubuntu:16.04', 'centos:7']: + stdout, stderr = self.run_main(base_args + ['--container-base=%s' % cont_base]) + self.assertFalse(stderr) + regexs = ["^== Dockerfile file created at %s/containers/Dockerfile.toy-0.0" % self.test_prefix] + self.check_regexs(regexs, stdout) + remove_file(os.path.join(self.test_prefix, 'containers', 'Dockerfile.toy-0.0')) + + self.run_main(base_args + ['--container-base=centos:7'], raise_error=True) + + error_pattern = "Dockerfile at .* already exists, not overwriting it without --force" + self.assertErrorRegex(EasyBuildError, + error_pattern, + self.eb_main, + base_args + ['--container-base=centos:7'], + raise_error=True) + + remove_file(os.path.join(self.test_prefix, 'containers', 'Dockerfile.toy-0.0')) + + base_args.insert(1, os.path.join(test_ecs, 'g', 'GCC', 'GCC-4.9.2.eb')) + self.run_main(base_args, raise_error=True) + def_file = read_file(os.path.join(self.test_prefix, 'containers', 'Dockerfile.toy-0.0')) + regexs = [ + "FROM ubuntu:16.04", + "eb toy-0.0.eb GCC-4.9.2.eb", + "module load toy/0.0 GCC/4.9.2", + ] + self.check_regexs(regexs, def_file) + remove_file(os.path.join(self.test_prefix, 'containers', 'Dockerfile.toy-0.0')) + + def test_end2end_docker_image(self): + + topdir = os.path.dirname(os.path.abspath(__file__)) + toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') + + containerpath = os.path.join(self.test_prefix, 'containers') + os.environ['EASYBUILD_CONTAINERPATH'] = containerpath + # --containerpath must be an existing directory (this is done to avoid misconfiguration) + mkdir(containerpath) + + args = [ + toy_ec, + '-C', # equivalent with --containerize + '--experimental', + '--container-type=docker', + '--container-build-image', + ] + + if not which('docker'): + error_pattern = "docker executable not found." + self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, raise_error=True) + + # install mocked versions of 'sudo' and 'docker' commands + docker = os.path.join(self.test_prefix, 'bin', 'docker') + write_file(docker, MOCKED_DOCKER) + adjust_permissions(docker, stat.S_IXUSR, add=True) + + sudo = os.path.join(self.test_prefix, 'bin', 'sudo') + write_file(sudo, '#!/bin/bash\necho "running command \'$@\' with sudo..."\neval "$@"\n') + adjust_permissions(sudo, stat.S_IXUSR, add=True) + + os.environ['PATH'] = '%s:%s' % (os.path.join(self.test_prefix, 'bin'), os.getenv('PATH')) + + stdout, stderr = self.run_main(args) + self.assertFalse(stderr) + regexs = [ + "^== docker tool found at %s/bin/docker" % self.test_prefix, + "^== Dockerfile file created at %s/containers/Dockerfile\.toy-0.0" % self.test_prefix, + "^== Running 'sudo docker build -f .* -t .* \.', you may need to enter your 'sudo' password...", + "^== Docker image created at toy-0.0:latest", + ] + self.check_regexs(regexs, stdout) + + args.extend(['--force', '--extended-dry-run']) + stdout, stderr = self.run_main(args) + self.assertFalse(stderr) self.check_regexs(regexs, stdout)