Add build staging integration test from POC #899

Based on the proof of concept from PR #899, add Python scripts to stage
files for building.

An integration test for this functionality can be found in
`tests/stage-test` and can be run as `make stage-test` or manually as
`./tests/stage-test/test_staging.sh`.

Additionally add flake8 configuration to tox.ini for linting the added
code. Currently, some of the existing code is also flagged by flake8, an
area for future improvement. flake8 linting can be run with `make lint`.

Signed-off-by: Blaine Gardner <blaine.gardner@suse.com>

.gitignore

@@ -8,3 +8,9 @@ ceph.osd.keyring
 ceph.rgw.keyring
 client-admin-key
 ceph-releases/devel/*
+
+__pycache__
+*.pyc
+*.pyo
+*.log
+staging/

Makefile

@@ -0,0 +1,22 @@
+# Copyright (c) 2017 SUSE LLC
+
+# ==============================================================================
+# Test targets
+.PHONY: lint test.staging
+
+lint:
+	flake8
+
+test.staging:
+	DEBUG=1 tests/stage-test/test_staging.sh
+
+# ==============================================================================
+# Help
+
+.PHONY: help
+help:
+	@echo 'Usage: make <OPTIONS> ... <TARGETS>'
+	@echo ''
+	@echo 'Targets:'
+	@echo '    lint:              Lint the source code.'
+	@echo '    test.staging:      Perform staging integration test.'

README.md

@@ -82,3 +82,28 @@ A recorded video on how to deploy your Ceph cluster entirely in Docker container
 ## With Ansible
 
 [![Demo Running Ceph in Docker containers with Ansible](http://img.youtube.com/vi/DQYZU1VsqXc/0.jpg)](http://youtu.be/DQYZU1VsqXc "Demo Running Ceph in Docker containers with Ansible")
+
+# Project structure and staging
+
+## Staging override priority order
+More specific files will override (overwrite) less specific files when staging. Note here that
+`FILE` may be a file or a directory containing further files.
+
+```
+# Most specific
+ceph-releases/<ceph release>/<os distro>/<os release>/{daemon-base,daemon}/FILE
+ceph-releases/<ceph release>/<os distro>/<os release>/FILE
+ceph-releases/<ceph release>/<os distro>/{daemon-base,daemon}/FILE
+ceph-releases/<ceph release>/<os distro>/FILE
+ceph-releases/<ceph release>/{daemon-base,daemon}/FILE
+ceph-releases/<ceph release>/FILE
+ceph-releases/ALL/<os distro>/<os release>/{daemon-base,daemon}/FILE
+ceph-releases/ALL/<os distro>/<os release>/FILE
+ceph-releases/ALL/<os distro>/{daemon-base,daemon}/FILE
+ceph-releases/ALL/<os distro>/FILE
+ceph-releases/ALL/{daemon-base,daemon}/FILE
+ceph-releases/ALL/FILE
+src/{daemon-base,daemon}/FILE
+src/FILE
+# Least specific
+```

stage.py

@@ -0,0 +1,96 @@
+#!/usr/bin/env python3
+# Copyright (c) 2017 SUSE LLC
+
+import os
+import logging
+import shutil
+import sys
+import time
+
+from stagelib.envglobals import (printGlobal, CEPH_VERSION, OS_NAME, OS_VERSION,
+                                 IMAGES_TO_BUILD, STAGING_DIR)
+from stagelib.filetools import (list_files, mkdir_if_dne, copy_files, recursive_copy_dir,
+                                IOOSErrorGracefulFail)
+from stagelib.replace import do_variable_replace
+from stagelib.blacklist import get_blacklist
+
+
+# Set default values for tunables (primarily only interesting for testing)
+CORE_FILES_DIR = "src"
+CEPH_RELEASES_DIR = "ceph-releases"
+BLACKLIST_FILE = "flavor-blacklist.txt"
+LOG_FILE = os.path.join(STAGING_DIR, "stage.log")
+
+# Start with empty staging dir so there are no previous artifacts
+try:
+    if os.path.isdir(STAGING_DIR):
+        shutil.rmtree(STAGING_DIR)
+    os.makedirs(STAGING_DIR, mode=0o755)
+except (OSError, IOError) as o:
+    IOOSErrorGracefulFail(o,
+                          'Could not delete and recreate staging dir: {}'.format(STAGING_DIR))
+
+loglevel = logging.INFO
+# If DEBUG env var is set to anything (including empty string) except '0', log debug text
+if 'DEBUG' in os.environ and not os.environ['DEBUG'] == '0':
+    loglevel = logging.DEBUG
+logging.basicConfig(filename=LOG_FILE, level=loglevel,
+                    format='%(levelname)5s:  %(message)s')
+logger = logging.getLogger(__name__)
+
+# Build dependency on python3 for `replace.py`. Looking to py2.7 deprecation in 2020.
+if sys.version_info[0] < 3:
+    print('This must be run with Python 3+')
+    sys.exit(1)
+
+
+def main(CORE_FILES_DIR, CEPH_RELEASES_DIR, BLACKLIST_FILE):
+    logger.info('\n\n\n')  # Make it easier to determine where new runs start
+    logger.info('Start time: {}'.format(time.ctime()))
+
+    print('')
+    printGlobal('CEPH_VERSION')
+    printGlobal('OS_NAME')
+    printGlobal('OS_VERSION')
+    printGlobal('BASEOS_REG')
+    printGlobal('BASEOS_REPO')
+    printGlobal('BASEOS_TAG')
+    printGlobal('ARCH')
+    printGlobal('IMAGES_TO_BUILD')
+    printGlobal('STAGING_DIR')
+    print('')
+
+    # Search from least specfic to most specific
+    path_search_order = [
+        "{}".format(CORE_FILES_DIR),
+        os.path.join(CEPH_RELEASES_DIR, 'ALL'),
+        os.path.join(CEPH_RELEASES_DIR, 'ALL', OS_NAME),
+        os.path.join(CEPH_RELEASES_DIR, 'ALL', OS_NAME, OS_VERSION),
+        os.path.join(CEPH_RELEASES_DIR, CEPH_VERSION),
+        os.path.join(CEPH_RELEASES_DIR, CEPH_VERSION, OS_NAME),
+        os.path.join(CEPH_RELEASES_DIR, CEPH_VERSION, OS_NAME, OS_VERSION),
+    ]
+    logger.debug('Path search order: {}'.format(path_search_order))
+
+    blacklist = get_blacklist(BLACKLIST_FILE)
+    logger.debug('Blacklist: {}'.format(blacklist))
+
+    for image in IMAGES_TO_BUILD:
+        logger.info('')
+        logger.info('{}/'.format(image))
+        logger.info('    Copying files')
+        for src_path in path_search_order:
+            if not os.path.isdir(src_path):
+                continue
+            src_files = list_files(src_path)
+            # e.g., IMAGES_TO_BUILD = ['daemon-base', 'daemon']
+            staging_path = os.path.join(STAGING_DIR, image)
+            mkdir_if_dne(staging_path, mode=0o755)
+            copy_files(src_files, src_path, staging_path, blacklist)
+            recursive_copy_dir(src_path=os.path.join(src_path, image), dst_path=staging_path,
+                               blacklist=blacklist)
+        do_variable_replace(replace_root_dir=os.path.join(STAGING_DIR, image))
+
+
+if __name__ == "__main__":
+    main(CORE_FILES_DIR, CEPH_RELEASES_DIR, BLACKLIST_FILE)

stagelib/blacklist.py

@@ -0,0 +1,67 @@
+# Copyright (c) 2017 SUSE LLC
+
+import logging
+import os
+
+logger = logging.getLogger(__name__)
+
+
+def _parse_condition(rawcondition):
+    parts = rawcondition.split('=')
+    if not len(parts) == 2:
+        raise Exception('Blacklist condition is not properly formatted: {}'.format(rawcondition))
+    varname, varvalues = parts[0], parts[1].split(',')
+    return (varname, varvalues)
+
+
+# Return true if a system environment variable matches any of a list of raw conditions.
+# ['VAR=t1,t2', 'XYZ=qop'] && env VAR==t2 --> True
+# ['VAR=t1,t2', 'XYZ=qop'] && env VAR==t3 --> False
+def _environment_matches(raw_conditions):
+    for rawcondition in raw_conditions:
+        varname, varvalues = _parse_condition(rawcondition)
+        if os.environ[varname] in varvalues:
+            return True
+    return False
+
+
+# Validate that path is either a file or directory, and return dirs w/ trailing '/'
+def _get_blacklisted_item(blacklist_path):
+    if os.path.isfile(blacklist_path):
+        return blacklist_path
+    if os.path.isdir(blacklist_path):
+        return os.path.join(blacklist_path, '')  # make sure dirs end in '/'
+    raise Exception('Blacklist path is not a file or directory: {}'.format(blacklist_path))
+
+
+# Parse a blacklist file line. If the line conditions are met, return the blacklisted location.
+# Return nothing if there is no matching blacklisted location.
+def _parse_line(line):
+    line = line.strip()
+    if len(line) == 0 or line[0] == '#':
+        return []  # Empty line or comment line
+    splitline = line.split(' ')
+    if len(splitline) < 2:
+        raise Exception('Blacklist line improperly formatted:\n{}'.format(line))
+    if _environment_matches(raw_conditions=splitline[1:]):
+        logger.info('    Blacklist line matches environment: {}'.format(line))
+        return [_get_blacklisted_item(splitline[0])]
+    return []
+
+
+def get_blacklist(blacklist_filename):
+    """
+    Returns a list of files that are part of the current blacklist from the given file.
+    Blacklist file format is expcted to be:
+      <path to be blacklisted> <ENV_VARIABLE>=<value>
+    If a current environment <ENV_VARIABLE> is equal to <value>, then there are files to be
+    blacklisted. If the <path to be blacklisted> is a file, a list containing only that filename
+    is returned. If the <path to be blacklisted> is a directory, a list of all files in the
+    directory recursively is returned.
+    """
+    logger.info('Parsing blacklist file: {}'.format(blacklist_filename))
+    blacklist = []
+    with open(blacklist_filename) as blacklist_file:
+        for line in blacklist_file.readlines():
+            blacklist += _parse_line(line)
+    return blacklist

stagelib/envglobals.py

@@ -0,0 +1,45 @@
+# Copyright (c) 2017 SUSE LLC
+
+import logging
+import os
+import sys
+
+try:
+    CEPH_VERSION = os.environ['CEPH_VERSION']
+    OS_NAME = os.environ['OS_NAME']
+    OS_VERSION = os.environ['OS_VERSION']
+    BASEOS_REG = os.environ['BASEOS_REG']
+    BASEOS_REPO = os.environ['BASEOS_REPO']
+    BASEOS_TAG = os.environ['BASEOS_TAG']
+    ARCH = os.environ['ARCH']
+    IMAGES_TO_BUILD = os.environ['IMAGES_TO_BUILD'].split(' ')
+    STAGING_DIR = os.environ['STAGING_DIR']
+except KeyError as k:
+    unset_var = k.args[0]
+    errtext = """
+Expected environment variable '{}' to be set.
+Required environment variables:
+ - CEPH_VERSION - Ceph named version being built (e.g., luminous, mimic)
+ - ARCH - Architecture of binaries being built (e.g., amd64, arm32, arm64)
+ - OS_NAME - OS name as used by the ceph-container project (e.g., ubuntu, opensuse)
+ - OS_VERSION - OS version as used by the ceph-container project (e.g., 16.04, 42.3 respectively)
+ - BASEOS_REG - Registry for the container base image (e.g., _ (default reg), arm32v7, arm64v8)
+                There is a relation between binaries built (ARCH) and this value
+ - BASEOS_REPO - Repository for the container base image (e.g., ubuntu, opensuse, alpine)
+ - BASEOS_TAG - Tagged version of the BASEOS_REPO container (e.g., 16.04, 42.3, 3.6 respectively)
+ - IMAGES_TO_BUILD - Container images to be built (usually should be 'dockerfile daemon')
+ - STAGING_DIR - Dir into which files will be staged
+                 This dir will be overwritten if it already exists
+
+"""
+    sys.stderr.write(errtext.format(unset_var))
+    sys.exit(1)
+
+logger = logging.getLogger(__name__)
+
+
+def printGlobal(varname):
+    """Print the name and value of a global variable to stdout and to the log"""
+    varvalstr = '  {:<16}: {}'.format(varname, globals()[varname])
+    print(varvalstr)
+    logger.info(varvalstr)

stagelib/filetools.py

@@ -0,0 +1,96 @@
+# Copyright (c) 2017 SUSE LLC
+
+import logging
+import os
+import shutil
+import sys
+
+COPY_LOGTEXT = '        {:<80}  -> {}'
+PARENTHETICAL_LOGTEXT = '        {:<80}     {}'
+
+
+def IOOSErrorGracefulFail(io_os_error, message):
+    """
+    Given an IOError or OSError exception, print the message to stdout, and then print relevant
+    stats about the exception to stdout before exiting with error code 1.
+    """
+    o = io_os_error
+    sys.stderr.write('{}\n'.format(message))
+    # errno and strerror are common to IOError and OSError
+    sys.stderr.write('Error [{}]: {}\n'.format(o.errno, o.strerror))
+    sys.exit(1)
+
+
+def save_text_to_file(text, file_path):
+    """Save text to a file at the file path. Will overwrite an existing file at the same path."""
+    try:
+        with open(file_path, 'w') as f:
+            f.write(text)
+    except (OSError, IOError) as o:
+        IOOSErrorGracefulFail(o, "Could not write text to file: {}".format(file_path))
+
+
+# List only files in dir
+def list_files(path):
+    """ List all files in the path non-recursively. Do not list dirs."""
+    return [f for f in os.listdir(path)
+            if os.path.isfile(os.path.join(path, f))]
+
+
+def mkdir_if_dne(path, mode=0o755):
+    """Make a directory if it does not exist"""
+    if not os.path.isdir(path):
+        try:
+            os.mkdir(path, mode)
+        except (OSError, IOError) as o:
+            IOOSErrorGracefulFail(o, "Could not create directory: {}".format(path))
+
+
+# Copy file from src to dst
+def _copy_file(file_path, dst_path):
+    try:
+        shutil.copy2(file_path, dst_path)
+    except (OSError, IOError) as o:
+        IOOSErrorGracefulFail(o, "Could not copy file {} to {}".format(file_path, dst_path))
+
+
+def copy_files(filenames, src_path, dst_path, blacklist):
+    """
+    Copy a list of filenames from src to dst. Will overwrite existing files.
+    If any files are in the blacklist, they will not be copied.
+    If the src directory is in the blacklist, the dest path will not be created, and the files will
+      not be copied or processed.
+    """
+    # Adding a trailing "/" if needed to improve output coherency
+    dst_path = os.path.join(dst_path, '')
+    if os.path.join(src_path, '') in blacklist:
+        logging.info(PARENTHETICAL_LOGTEXT.format(src_path, '[DIR BLACKLISTED]'))
+        return
+    mkdir_if_dne(dst_path)
+    for f in filenames:
+        file_path = os.path.join(src_path, f)
+        if file_path in blacklist:
+            logging.info(PARENTHETICAL_LOGTEXT.format(file_path, '[FILE BLACKLISTED]'))
+            continue
+        logging.info(COPY_LOGTEXT.format(file_path, dst_path))
+        _copy_file(file_path, dst_path)
+
+
+def recursive_copy_dir(src_path, dst_path, blacklist=[]):
+    """
+    Copy all files in the src directory recursively to dst. Will overwrite existing files.
+    If any files encountered are in the blacklist, they will not be copied.
+    If any directories encountered are in the blacklist, the corresponding dest path will not be
+      created, and files/subdirs within the blacklisted dir will not be copied.
+    """
+    if not os.path.isdir(src_path):
+        return
+    for dirname, subdirs, files in os.walk(src_path, topdown=True):
+        # Remove src_path (and '/' immediately following) from our dirname
+        if os.path.join(dirname, '') in blacklist:
+            logging.info(PARENTHETICAL_LOGTEXT.format(dirname, '[DIR BLACKLISTED]'))
+            subdirs[:] = []
+            continue
+        dst_path_offset = dirname[len(src_path)+1:]
+        copy_files(filenames=files, src_path=dirname,
+                   dst_path=os.path.join(dst_path, dst_path_offset), blacklist=blacklist)