dependencies.py

import os
import tarfile
import zipfile
import re
import urllib
import requests
import platform
import subprocess
import json
import shutil
import io
import tempfile
from glob import glob
from default_platform import default_platform
import deps_cross_checker
import aws

# Master table of dependency types.

# A dependency definition can specify 'type' to inherit definitions from one of these.
# String values can depend on other string values from the dependency. For example,
# if 'name' is defined as 'Example' then '${name}.exe' will expand to 'Example.exe'.
# It does not matter which order the values are defined.
# String values can also depend on boolean values. For example, the string
# '${test-value?yes-result:no-result}' will get the value of the string named
# 'yes-result' if 'test-value' is a true boolean value, and the string named
# 'no-result' if 'test-value' is a false boolean value.
# Finally, string values can also depend on a lookup table defined as a JSON object.
# For example, given these definitions:
# {
#     "servertable":{
#         "Windows":"windows.openhome.org",
#         "Linux":"linux.openhome.org",
#         "*":"openhome.org"
#     },
#     "server":"${servertable[$system]}"
# }
# If 'system' is defined as 'Windows', then 'server' will be defined as
# 'windows.openhome.org'. The '*' entry is the default: if a lookup fails the default
# will be used instead.

# The principle string values that must be defined are 'archive-path' to point to the
# .tar.gz file with the dependency's binaries, 'dest' to specify where to untar it,
# and 'configure-args' to specify the list of arguments to pass to waf.

# In order for source control fetching to work, the string 'source-git' should point
# to the git repo and 'tag' should identify the git tag that corresponds to the
# fetched binaries.

DEPENDENCY_TYPES = {
    # Label a dependency with the 'ignore' type to prevent it being considered at all.
    # Can be useful to include comments. (Json has no comment syntax.)
    'ignore': {
        'ignore': True     # This causes the entire dependency entry to be ignored. Useful for comments.
    },

    # Openhome dependencies generally have an associated git repo to allow us to
    # fetch source code. They also have a different directory layout to accomodate
    # the large number of versions created by CI builds.
    #
    # An openhome dependency, at minimum, must define:
    #     name
    #     version
    #
    # Commonly overridden:
    #     archive-suffix
    #     platform-specific
    #     configure-args
    'openhome': {
        'archive-extension': '.tar.gz',
        'archive-prefix': '',
        'archive-suffix': '',
        'binary-repo': 'http://builds.openhome.org/releases/artifacts',
        'archive-directory': '${binary-repo}/${name}/',
        'archive-filename': '${archive-prefix}${name}-${version}-${archive-platform}${archive-suffix}${archive-extension}',
        'remote-archive-path': '${archive-directory}${archive-filename}',
        'use-local-archive': False,
        'archive-path': '${use-local-archive?local-archive-path:remote-archive-path}',
        'source-path': '${linn-git-user}@core.linn.co.uk:/home/git',
        'repo-name': '${name}',
        'source-git': '${source-path}/${repo-name}.git',
        'tag': '${repo-name}_${version}',
        'any-platform': 'AnyPlatform',
        'platform-specific': True,
        'host-platform': default_platform(),
        'archive-platform': '${platform-specific?platform:any-platform}',
        'dest': 'dependencies/${archive-platform}/',
        'configure-args': []
    },

    # Internal dependencies are named and structured in a similar manner
    # to those of type 'openhome', but are considered private, and held
    # in a non-public location
    #
    # Must define, at minimum:
    #       name
    #       version
    #
    # Commonly overridden:
    #       archive-suffix

    'internal': {
        'binary-repo': 'http://core.linn.co.uk/~artifacts/artifacts',
        'source-git': None,
        'any-platform': 'AnyPlatform',
        'platform-specific': True,
        'archive-suffix': '',
        'archive-filename': '${name}-${version}-${platform}${archive-suffix}.tar.gz',
        'archive-platform': '${platform-specific?platform:any-platform}',
        'archive-path': '${binary-repo}/${name}/${archive-filename}',
        'host-platform': default_platform(),
        'dest': 'dependencies/${archive-platform}/',
        'configure-args': []
    },

    # External dependencies generally don't have a git repo, and even if they do,
    # it won't conform to our conventions.
    #
    # An external dependency, at minimum, must define:
    #     name
    #     archive-filename
    #
    # Commonly overriden:
    #     platform-specific
    #     configure-args
    'external': {
        'binary-repo': 'http://builds.openhome.org/releases/artifacts',
        'source-git': None,
        'any-platform': 'AnyPlatform',
        'platform-specific': True,
        'archive-platform': '${platform-specific?platform:any-platform}',
        'archive-path': '${binary-repo}/${archive-platform}/${archive-filename}',
        'host-platform': default_platform(),
        'dest': 'dependencies/${archive-platform}/',
        'configure-args': []
    },
}

AWS_BUCKET = {'private': 'linn-artifacts-private',
              'public':  'linn-artifacts-public'}
AWS_URL    = {'private': 's3-eu-west-1.amazonaws.com/linn-artifacts-private',
              'public':  's3-eu-west-1.amazonaws.com/linn-artifacts-public'}


class FileFetcher(object):

    def __init__(self):
        pass

    def fetch(self, path):
        if path.startswith("file:") or path.startswith("smb:"):
            return self.fetch_file_url(path)
        elif path.startswith("s3:"):
            return self.fetch_aws(path)
        elif re.match("[^\W\d]{2,8}:", path):
            urlpath = path
            if DEPENDENCY_TYPES['internal']['binary-repo'] in path:
                awspath = 's3://' + AWS_BUCKET['private'] + '/' + path.replace(DEPENDENCY_TYPES['internal']['binary-repo'] + '/', '')
                urlpath = 'https://' + AWS_URL['private'] + '/' + path.replace(DEPENDENCY_TYPES['internal']['binary-repo'] + '/', '')
            elif DEPENDENCY_TYPES['openhome']['binary-repo'] in path:
                awspath = 's3://' + AWS_BUCKET['public'] + '/artifacts/' + path.replace(DEPENDENCY_TYPES['openhome']['binary-repo'] + '/', '')
                urlpath = 'https://' + AWS_URL['public'] + '/artifacts/' + path.replace(DEPENDENCY_TYPES['openhome']['binary-repo'] + '/', '')
            if ('awspath' in vars()):
                rc = self.fetch_aws(awspath)
                if rc:
                    return rc
            return self.fetch_url(urlpath)
        return self.fetch_local(path)

    @staticmethod
    def fetch_aws(awspath):
        print('  from AWS %s' % awspath)
        temppath = tempfile.mktemp( suffix='.tmp' )
        try:
            aws.copy(awspath, temppath)
            return temppath
        except:
            return None

    @staticmethod
    def fetch_local(path):
        print( '  from %s' % path)
        return path

    @staticmethod
    def fetch_file_url(url):
        print( '  from %s' % url)
        smb = False
        if url.startswith("smb://"):
            url = url[6:]
            smb = True
        elif url.startswith("file://"):
            url = url[7:]
        path = urllib.url2pathname(url).replace(os.path.sep, "/")
        if path[0] == '/':
            if path[1] == '/':
                # file:////hostname/path/file.ext
                # Bad remote path.
                remote = True
                legacy = True
                final_path = path.replace("/", os.path.sep)
            else:
                # file:///path/file.ext
                # Good local path.
                remote = False
                legacy = False
                if smb:
                    raise Exception("Bad smb:// path")
                final_path = path[1:].replace("/", os.path.sep)
        else:
            if path[0].isalpha() and path[1] == ':':
                # file:///x:/foo/bar/baz
                # Good absolute local path.
                remote = False
                legacy = False
                final_path = path.replace('/', os.path.sep)
            else:
                # file://hostname/path/file.ext
                # Good remote path.
                remote = True
                legacy = False
                final_path = "\\\\" + path.replace("/", os.path.sep)
        if smb and (legacy or not remote):
            raise Exception("Bad smb:// path. Use 'smb://hostname/path/to/file.ext'")
        if (smb or remote) and not platform.platform().startswith("Windows"):
            raise Exception("SMB file access not supported on non-Windows platforms.")
        return final_path

    @staticmethod
    def fetch_url(url):
        print("  from '%s'" % url)
        handle, temppath = tempfile.mkstemp(suffix='.tmp')
        remotefile = requests.get(url=url, headers={'Accept-Encoding': 'identity'})
        localfile = os.fdopen(handle, 'wb')
        if remotefile.status_code == 200:
            localfile.write(remotefile.content)
        localfile.close()
        return temppath


class EnvironmentExpander(object):
    # template_regex matches
    template_regex = re.compile(r"""
        (?x)                                # Enable whitespace and comments
        (?P<dollar>\$\$)|                   # Match $$
        (?P<word>\$[a-zA-Z_][a-zA-Z_0-9]*)| # Match $word
        (?P<parens>\$\{[^}]*\})             # Match ${any-thing}
        """)
    # Matches foo[bar]
    index_regex = re.compile(r"""
        (?x)         # Enable whitespace and comments
        ^            # Match only at start of string
        ([^][]*)     # Match table name (no brackets allowed)
        \[           # Match one open bracket: [
        ([^][]*)     # Match key (no brackets allowed)
        \]           # Match one close bracket: ]
        $
        """)

    def __init__(self, env_dict):
        self.env_dict = env_dict
        self.cache = {}
        self.expandset = set()

    def __getitem__(self, key):
        return self.expand(key)

    def getraw(self, key):
        return self.env_dict[key]

    def __contains__(self, key):
        return key in self.env_dict

    def keys(self):
        return self.env_dict.keys()

    def values(self):
        return [self.expand(key) for key in self.keys()]

    def items(self):
        return [(key, self.expand(key)) for key in self.keys()]

    def expand(self, key):
        if key in self.cache:
            return self.cache[key]
        if key in self.expandset:
            raise ValueError("Recursive expansion for key:", key)
        self.expandset.add(key)
        result = self._expand(key)
        self.cache[key] = result
        self.expandset.remove(key)
        return result

    def _expand(self, key):
        if key not in self.env_dict:
            raise KeyError("Key undefined:", key)
        value = self.env_dict[key]
        return self._expandvalue(value)

    def _expandvalue(self, value):
        if isinstance(value, ("".__class__, u"".__class__)):
            return self.expandstring(value)
            # return self.template_regex.sub(self.replacematch, value)
        elif isinstance(value, (list, tuple)):
            return [self._expandvalue(x) for x in value]
        elif isinstance(value, dict):
            return dict((k, self._expandvalue(v)) for (k, v) in value.items())
        return value

    def expandstring(self, value):
        firstmatch = self.template_regex.match(value)
        if firstmatch is not None and firstmatch.group(0) == value and value != "$$":
            # Special case: The entire string is a single expansion. In this case,
            # we allow the expansion to be *anything* (bool, int, list...),
            # not just a string.
            return self.replacematch(firstmatch)
        return self.template_regex.sub(self.replacematch, value)

    def replacematch(self, match):
        if match.group('dollar'):
            return '$'
        key = None
        if match.group('word'):
            key = match.group('word')[1:]
        if match.group('parens'):
            key = match.group('parens')[2:-1]
        assert key is not None
        key = key.strip()
        if '[' in key:
            return self.expandlookup(key)
        if '?' in key:
            return self.expandconditional(key)
        return self.expand(key)

    def expandlookup(self, key):
        match = self.index_regex.match(key)
        if match is None:
            raise ValueError('lookup must be of form ${table[key]}')
        tablename = match.group(1).strip()
        keyname = match.group(2).strip()
        table = self.expand(tablename)
        if keyname.startswith('$'):
            key = self.expand(keyname[1:])
        else:
            key = keyname
        if not isinstance(table, dict):
            raise ValueError("lookup table must expand to a JSON object (got {0!r} instead)".format(table))
        if not isinstance(key, ("".__class__, u"".__class__)):
            raise ValueError("lookup index must expand to a JSON string (got {0!r} instead)".format(key))
        if key not in table:
            if '*' in table:
                return table['*']
            raise KeyError("Key not in table, and no default '*' entry found: key={0!r}\ntable={1!r}".format(key, table))
        return table[key]

    def expandconditional(self, key):
        if '?' not in key:
            raise ValueError('conditional must be of form ${condition?result:alternative}')
        condition, rest = key.split('?', 1)
        if ':' not in rest:
            raise ValueError('conditional must be of form ${condition?result:alternative}')
        primary, alternative = rest.split(':', 1)
        condition, primary, alternative = [x.strip() for x in [condition, primary, alternative]]
        try:
            conditionvalue = self.expand(condition)
        except KeyError:
            conditionvalue = False
        if self.is_trueish(conditionvalue):
            return self.expand(primary)
        return self.expand(alternative)

    @staticmethod
    def is_trueish(value):
        if hasattr(value, "upper"):
            value = value.upper()
        return value in [1, "1", "YES", "Y", "TRUE", "ON", True]


class Dependency(object):

    def __init__(self, name, environment, fetcher, has_overrides=False):
        self.expander = EnvironmentExpander(environment)
        self.has_overrides = has_overrides
        self.fetcher = fetcher

    def fetch(self):
        remote_path = self.expander.expand('archive-path')
        local_path = os.path.abspath(self.expander.expand('dest'))
        fetched_path = None

        print("\nFetching '%s'" % self.name)
        try:
            fetched_path = self.fetcher.fetch(remote_path)
            statinfo = os.stat(fetched_path)
            if not statinfo.st_size:
                os.unlink(fetched_path)
                print("  **** WARNING - failed to fetch %s ****" % os.path.basename(remote_path))
                return False
        except IOError:
            print("  **** FAILED ****")
            return False

        try:
            os.makedirs(local_path)
        except OSError:
            # We get an error if the directory exists, which we are happy to
            # ignore. If something worse went wrong, we will find out very
            # soon when we try to extract the files.
            pass

        print("  unpacking to '%s'" % (local_path,))
        if os.path.splitext(remote_path)[1].upper() in ['.ZIP', '.NUPKG', '.JAR']:
            self.unzip(fetched_path, local_path)
        else:
            self.untar(fetched_path, local_path)

        if fetched_path:
            if fetched_path != remote_path:
                os.unlink(fetched_path)
        print("OK")
        return True

    @property
    def name(self):
        return self['name']

    def __getitem__(self, key):
        return self.expander.expand(key)

    def __contains__(self, key):
        return key in self.expander

    def items(self):
        return self.expander.items()

    def checkout(self):
        name = self['name']
        sourcegit = self['source-git']
        if sourcegit is None:
            print('No git repo defined for {0}'.format(name))
            return False
        print("Fetching source for '%s'\n  into '%s'" % (name, os.path.abspath('../' + name)))
        tag = self['tag']
        try:
            if not os.path.exists('../' + name):
                print('  git clone {0} {1}'.format(sourcegit, name))
                subprocess.check_call(['git', 'clone', sourcegit, name], cwd='..', shell=False)
            elif not os.path.isdir('../' + name):
                print('Cannot checkout {0}, because directory ../{0} already exists'.format(name))
                return False
            else:
                print('  git fetch origin')
                subprocess.check_call(['git', 'fetch', 'origin'], cwd='../' + name, shell=False)
            print("  git checkout {0}".format(tag))
            subprocess.check_call(['git', 'checkout', tag], cwd='../' + name, shell=False)
        except subprocess.CalledProcessError as cpe:
            print(str(cpe))
            return False
        return True

    @staticmethod
    def untar(source, dest):
        tf = tarfile.open(source, 'r')
        for f in tf:
            try:
                tf.extract(f.name, path=dest)
            except IOError:
                os.unlink( os.path.join(dest, f.name ))
                tf.extract(f.name, path=dest)
        tf.close()

    @staticmethod
    def unzip(source, dest):
        zf = zipfile.ZipFile(source, mode='r')
        zf.extractall(path=dest)
        zf.close()

    def expand_remote_path(self):
        return self.expander.expand('archive-path')

    def expand_local_path(self):
        return self.expander.expand('dest')

    def expand_configure_args(self):
        return self.expander.expand('configure-args')


class DependencyCollection(object):

    def __init__(self, env):
        fetcher = FileFetcher()
        self.base_env = env
        self.dependency_types = DEPENDENCY_TYPES
        self.dependencies = {}
        self.fetcher = fetcher

    def create_dependency(self, dependency_definition, overrides={}):
        defn = dependency_definition
        env = {}
        env.update(self.base_env)
        if 'type' in defn:
            dep_type = defn['type']
            env.update(self.dependency_types[dep_type])
        else:
            # default to an 'external' dependency type if none specified
            dep_type = 'external'
            env.update(self.dependency_types[dep_type])
        env.update(defn)
        env.update(overrides)
        if 'name' not in env:
            raise ValueError('Dependency definition contains no name')
        name = env['name']
        new_dependency = Dependency(name, env, self.fetcher, has_overrides=len(overrides) > 0)
        if 'ignore' in new_dependency and new_dependency['ignore']:
            return
        self.dependencies[name] = new_dependency

    def __contains__(self, key):
        return key in self.dependencies

    def __getitem__(self, key):
        return self.dependencies[key]

    def items(self):
        return self.dependencies.items()

    def _filter(self, subset=None):
        if subset is None:
            return self.dependencies.values()
        missing_dependencies = [name for name in subset if name not in self.dependencies]
        if len(missing_dependencies) > 0:
            raise Exception("No entries in dependency file named: " + ", ".join(missing_dependencies) + ".")
        return [self.dependencies[name] for name in subset]

    def get_args(self, subset=None):
        dependencies = self._filter(subset)
        configure_args = sum((d.expand_configure_args() for d in dependencies), [])
        return configure_args

    def fetch(self, subset=None):
        dependencies = self._filter(subset)
        failed_dependencies = []
        filename = self.fetched_deps_filename(dependencies)
        fetched_deps = self.load_fetched_deps(filename)
        for d in dependencies:
            do_fetch = True
            name = ''
            path = ''
            dest = ''
            if 'name' in d.expander:
                name = d.expander.expand('name')
            if 'archive-path' in d.expander:
                path = d.expander.expand('archive-path')
            if 'dest' in d.expander:
                dest = d.expander.expand('dest')
            lookup = dest.rstrip( '/' ) + '/' + name
            version = os.path.basename(path)
            if lookup in fetched_deps:
                if fetched_deps[lookup] == version:
                    print("Skipping fetch of %s as unchanged (%s)" % (name, version))
                    do_fetch = False
            if do_fetch:
                if not d.fetch():
                    failed_dependencies.append(d.name)
                else:
                    fetched_deps[lookup] = version
        if filename:
            self.save_fetched_deps(filename, fetched_deps)
        if failed_dependencies:
            print("Failed to fetch some dependencies: " + ' '.join(failed_dependencies))
            return False
        return True

    @staticmethod
    def fetched_deps_filename(deps):
        filename = None
        for d in deps:
            if 'dest' in d.expander:
                filename = os.path.join(d.expander.expand('dest').split('/')[0], 'loadedDeps.json')
                break
        return filename

    def load_fetched_deps(self, filename):
        loaded_deps = {}
        if filename and os.path.isfile(filename):
            try:
                f = open(filename, 'rt')
                loaded_deps = json.load(f)
                f.close()
            except:
                print("Error with current fetched dependency file: %s" % filename)
        return loaded_deps

    @staticmethod
    def save_fetched_deps(filename, deps):
        f = open(filename, 'wt')
        json.dump(deps, f)
        f.close()

    def checkout(self, subset=None):
        dependencies = self._filter(subset)
        failed_dependencies = []
        for d in dependencies:
            if not d.checkout():
                failed_dependencies.append(d.name)
        if failed_dependencies:
            print("Failed to check out some dependencies: " + ' '.join(failed_dependencies))
            return False
        return True


def read_json_dependencies(dependencyfile, overridefile, env):
    collection = DependencyCollection(env)
    dependencies = json.load(dependencyfile)
    overrides = json.load(overridefile)
    overrides_by_name = dict((dep['name'], dep) for dep in overrides)
    for d in dependencies:
        name = d['name']
        override = overrides_by_name.get(name, {})
        collection.create_dependency(d, override)
    return collection


def read_json_dependencies_from_filename(dependencies_filename, overrides_filename, env):
    try:
        dependencyfile = open(dependencies_filename, "r")
        with open(dependencies_filename) as dependencyfile:
            if overrides_filename is not None and os.path.isfile(overrides_filename):
                with open(overrides_filename) as overridesfile:
                    return read_json_dependencies(dependencyfile, overridesfile, env)
            else:
                return read_json_dependencies(dependencyfile, io.StringIO(u'[]'), env)
    except (OSError, IOError) as e:
        if e.errno != 2:
            raise
        return DependencyCollection(env)


def cli(args):
    if platform.system() != "Windows":
        args = ["mono", "--runtime=v4.0.30319"] + args
    subprocess.check_call(args, shell=False)


def clean_dirs(dir):
    """Remove the specified directory tree - don't remove anything if it would fail"""
    if os.path.isdir( dir ):
        locked = []
        for dirName, subdirList, fileList in os.walk(dir):
            for fileName in fileList:
                filePath = os.path.join(dirName, fileName)
                try:
                    if not os.path.islink( filePath ):
                        openAtt = 'r'
                        if platform.system().lower() == 'windows':
                            openAtt = 'a'
                        f = open(filePath, openAtt)
                        f.close()
                except:
                    locked.append(filePath)
        if locked:
            for f in locked:
                print('Locked file:- ', f)
            raise Exception('Failed to clean dependencies\n')
        else:
            shutil.rmtree(dir)


def fetch_dependencies(dependency_names=None, platform=None, env=None, fetch=True, clean=True, source=False, list_details=False, local_overrides=True, verbose=False):
    '''
    Fetch all the dependencies defined in projectdata/dependencies.json and in
    projectdata/packages.config.
    platform:
        Name of target platform. E.g. 'Windows-x86', 'Linux-x64', 'Mac-x64'...
    env:
        Extra variables referenced by the dependencies file.
    fetch:
        True to fetch the listed dependencies, False to skip.
    clean:
        True to clean out directories before fetching, False to skip.
    source:
        True to fetch source for the listed dependencies, False to skip.
    '''
    if env is None:
        env = {}

    if platform is not None:
        env['platform'] = None
        fName = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'platforms.txt')
        f = open(fName, 'rt')
        supported = f.readlines()
        f.close()
        for entry in supported:
            if platform in entry:
                env['platform'] = platform
        if not env['platform']:
            raise Exception('Platform not supported (%s) - see %s for list of supported platforms' % (platform, fName))
    if 'platform' not in env:
        platform = env['platform'] = default_platform()
    if '-' in platform:
        env['system'], env['architecture'] = platform.split('-', 2)
    if platform is None:
        raise Exception('Platform not specified and unable to guess.')

    if clean and not list_details:
        try:
            os.unlink('dependencies/loadedDeps.json')
        except:
            pass
        clean_dirs('dependencies')

    overrides_filename = '../dependency_overrides.json' if local_overrides else None
    dependencies = read_json_dependencies_from_filename('projectdata/dependencies.json', overrides_filename, env=env)
    if list_details:
        for name, dependency in dependencies.items():
            print("Dependency '{0}':".format(name))
            print("    fetches from:     {0!r}".format(dependency['archive-path']))
            print("    unpacks to:       {0!r}".format(dependency['dest']))
            print("    local override:   {0}".format("YES (see '../dependency_overrides.json')" if dependency.has_overrides else 'no'))
            if verbose:
                print("    all keys:")
                for key, value in sorted(dependency.items()):
                    print("        {0} = {1!r}".format(key, value))
            print("")
    else:
        if fetch:
            if not dependencies.fetch(dependency_names):
                raise Exception("Failed to load requested dependencies")

        if source:
            dependencies.checkout(dependency_names)

    # Finally perform cross-check of (major.minor) dependency versions to ensure that these are in sync
    # across this (current) repo and all its pulled-in dependencies. Done as totally seperate operation
    # to isolate from the main fetcher code to assist with any future maintenance
    if not clean:
        xcheck = deps_cross_checker.DepsCrossChecker( platform )
        result = xcheck.execute()
        if result != 0:
            raise Exception( 'Failed: dependency cross-checker detected problem(s)' )

    return dependencies