diff --git a/.gitignore b/.gitignore index 9ce81fb..4517115 100644 --- a/.gitignore +++ b/.gitignore @@ -105,3 +105,4 @@ ENV/ # mypy .mypy_cache/ +.vscode/ diff --git a/rever/activities/custom_forge.xsh b/rever/activities/custom_forge.xsh new file mode 100644 index 0000000..6557f5c --- /dev/null +++ b/rever/activities/custom_forge.xsh @@ -0,0 +1,261 @@ +"""Activity for updating a custom (private) conda-forge feedstocks.""" +import os +import re +import sys + +from lazyasd import lazyobject +from xonsh.tools import print_color + +from rever import vcsutils +from rever import github +from rever.activity import Activity +from rever.tools import eval_version, indir, hash_url, replace_in_file + + +@lazyobject +def github3(): + import github3 as gh3 + return gh3 + + +def feedstock_url(feedstock, feedstock_org, protocol='ssh'): + """Returns the URL for a custom conda-forge feedstock.""" + if feedstock is None: + feedstock = $PROJECT + '-feedstock' + elif feedstock.startswith('http://github.com/'): + return feedstock + elif feedstock.startswith('https://github.com/'): + return feedstock + elif feedstock.startswith('git@github.com:'): + return feedstock + protocol = protocol.lower() + if protocol == 'http': + url = 'http://github.com/{}/'.format(feedstock_org) + feedstock + '.git' + elif protocol == 'https': + url = 'https://github.com/{}/'.format(feedstock_org) + feedstock + '.git' + elif protocol == 'ssh': + url = 'git@github.com:{}/'.format(feedstock_org) + feedstock + '.git' + else: + msg = 'Unrecognized github protocol {0!r}, must be ssh, http, or https.' + raise ValueError(msg.format(protocol)) + return url + + +def feedstock_repo(feedstock): + """Gets the name of the feedstock repository.""" + if feedstock is None: + repo = $PROJECT + '-feedstock' + else: + repo = feedstock + repo = repo.rsplit('/', 1)[-1] + if repo.endswith('.git'): + repo = repo[:-4] + return repo + + +def fork_url(feedstock_url, username, feedstock_org): + """Creates the URL of the user's fork.""" + beg, end = feedstock_url.rsplit('/', 1) + beg = beg.replace(feedstock_org, '') # chop off `feedstock_org` + url = beg + username + '/' + end + return url + + +DEFAULT_PATTERNS = ( + # filename, pattern, new + # set the version + ('meta.yaml', r' version:\s*[A-Za-z0-9._-]+', ' version: "$VERSION"'), + ('meta.yaml', '{% set version = ".*" %}', '{% set version = "$VERSION" %}'), + # reset the build number to 0 + ('meta.yaml', ' number:.*', ' number: 0'), + ) + + +class CustomForge(Activity): + """Updates custom (private) conda-forge feedstocks. + + The behaviour of this activity may be adjusted through the following + environment variables: + + :$CUSTOM_FORGE_FEEDSTOCK: str or None, feedstock name or URL, + default ``$PROJECT-feedstock``. + :$CUSTOM_FORGE_PROTOCOL: str, one of ``'ssh'``, ``'http'``, or ``'https'`` + that specifies how the activity should interact with github when + cloning, pulling, or pushing to the feedstock repo. Note that + ``'ssh'`` requires you to have an SSH key registered with github. + The default is ``'ssh'``. + :$CUSTOM_FORGE_SOURCE_URL: str, the URL that the recipe will use to + download the source code. This is needed so that we may update the + hash of the downloaded file. This string is evaluated with the current + environment. Default + ``'https://github.com/$GITHUB_ORG/$GITHUB_REPO/archive/$VERSION.tar.gz'``. + :$CUSTOM_FORGE_HASH_TYPE: str, the type of hash that the recipe uses, eg + ``'md5'`` or ``'sha256'``. Default ``'sha256'``. + :$CUSTOM_FORGE_PATTERNS: list or 3-tuples of str, this is list of + (filename, pattern-regex, replacement) tuples that is evaluated + inside of the recipe directory. This is similar to the version bump + pattern structure. Both the pattern-regex str and the replacement str + will have environment variables expanded. The following environment + variables are added for this evaluation: + + * ``$SOURCE_URL``: the fully expanded source code URL. + * ``$HASH_TYPE``: the hash type used to hash ``$SOURCE_URL``. + * ``$HASH``: the hexdigest of ``$SOURCE_URL``. + + The default patterns match most standard recipes. + :$CUSTOM_FORGE_PULL_REQUEST: bool, whether the activity should open + a pull request to the upstream conda-forge feestock, default True. + :$CUSTOM_FORGE_RERENDER: bool, whether the activity should rerender the + feedstock using conda-smithy, default True. + :$CUSTOM_FORGE_FEEDSTOCK_ORG: str, specify the feedstock organization. Must be set. + :$CUSTOM_FORGE_FORK: bool, whether the activity should create a new fork of + the feedstock if it doesn't exist already, default True. + :$CUSTOM_FORGE_FORK_ORG: str, the org to fork the recipe to or which holds + the fork, if ``''`` use the registered gh username, defaults to ``''`` + :$CUSTOM_FORGE_USE_GIT_URL: bool, whether or not to use `git_url` in the recipe source + url, default True. + + Other environment variables that affect the behavior are: + + :$GITHUB_CREDFILE: the credential file to use. This should NOT be + set in the rever.xsh file + :$GITHUB_ORG: the github organization that the project belongs to. + :$GITHUB_REPO: the github repository of the project. + :$TAG_TEMPLATE: str, the template string used to tag the version, by default + this is '$VERSION'. Used to download project source. + :$PROJECT: the name of the project being released. + :$REVER_CONFIG_DIR: the user's config directory for rever, which + is where the GitHub credential files are stored by default. + + """ + + # def __init__(self, *, deps=frozenset(('tag', 'push_tag'))): + def __init__(self, *, deps=frozenset()): + requires = {"imports": {"github3.exceptions": "github3.py"}, + "commands": {"conda": "conda", "conda-smithy": "conda-smithy"}} + + super().__init__(name='custom_forge', deps=deps, func=self._func, + desc="Updates a custom (private) conda-forge feedstocks", + requires=requires, check=self.check_func) + + def _func(self, feedstock=None, protocol='ssh', source_url=None, + hash_type='sha256', patterns=DEFAULT_PATTERNS, + pull_request=False, rerender=True, feedstock_org=None, + fork=False, fork_org='', use_git_url=True): + + if feedstock_org is None: + raise ValueError("CUSTOM_FORGE_FEEDSTOCK_ORG must be set.") + + if source_url is None: + version_tag = ${...}.get('TAG_TEMPLATE', $VERSION) + release_fn = $GITHUB_REPO + '-' + version_tag + '.tar.gz' + if release_fn in os.listdir($REVER_DIR): + source_url=('https://github.com/$GITHUB_ORG/$GITHUB_REPO' + '/releases/download/{}/{}'.format( + version_tag, release_fn)) + else: + source_url = ('https://github.com/$GITHUB_ORG/$GITHUB_REPO/' + 'archive/{}.tar.gz'.format(version_tag)) + + # first, let's grab the feedstock locally + gh, username = github.login(return_username=True) + upstream = feedstock_url(feedstock, feedstock_org, protocol=protocol) + + # Allow to push in `master` in `upstream` if `fork` and `pull_request` are set to False. + if not fork and not pull_request: + origin = upstream + else: + origin = fork_url(upstream, username, feedstock_org) + feedstock_reponame = feedstock_repo(feedstock) + + if pull_request or fork: + repo = gh.repository(feedstock_org, feedstock_reponame) + + # Check if fork exists + if fork: + try: + fork_repo = gh.repository(fork_org or username, + feedstock_reponame) + except github3.exceptions.NotFoundError: + fork_repo = None + if fork_repo is None or (hasattr(fork_repo, 'is_null') and + fork_repo.is_null()): + print("Fork doesn't exist creating feedstock fork...", + file=sys.stderr) + if fork_org: + repo.create_fork(fork_org) + else: + repo.create_fork() + + feedstock_dir = os.path.join($REVER_DIR, $PROJECT + '-feedstock') + recipe_dir = os.path.join(feedstock_dir, 'recipe') + + if not os.path.isdir(feedstock_dir): + p = ![git clone @(origin) @(feedstock_dir)] + if p.rtn != 0: + msg = 'Could not clone ' + origin + msg += '. Do you have a personal fork of the feedstock?' + raise RuntimeError(msg) + + with indir(feedstock_dir): + # make sure feedstock is up-to-date with origin + git checkout master + git pull @(origin) master + # make sure feedstock is up-to-date with upstream + git pull @(upstream) master + + if fork or pull_request: + # make and modify version branch + with ${...}.swap(RAISE_SUBPROC_ERROR=False): + git checkout -b $VERSION master or git checkout $VERSION + + # now, update the feedstock to the new version + if not use_git_url: + source_url = eval_version(source_url) + hash = hash_url(source_url) + with indir(recipe_dir), ${...}.swap(HASH_TYPE=hash_type, HASH=hash, + SOURCE_URL=source_url): + for f, p, n in patterns: + p = eval_version(p) + n = eval_version(n) + replace_in_file(p, n, f) + else: + with indir(recipe_dir): + for f, p, n in patterns: + p = eval_version(p) + n = eval_version(n) + replace_in_file(p, n, f) + + with indir(feedstock_dir), ${...}.swap(RAISE_SUBPROC_ERROR=False): + git commit -am @("updated v" + $VERSION) + if rerender: + print_color('{YELLOW}Rerendering the feedstock{NO_COLOR}', + file=sys.stderr) + conda smithy regenerate -c auto + + # lastly make a PR for the feedstock or directly push + if not pull_request: + with indir(feedstock_dir), ${...}.swap(RAISE_SUBPROC_ERROR=False): + if fork: + git push --set-upstream @(origin) $VERSION + else: + git push @(origin) master + return + + print('Creating conda-forge feedstock pull request...', file=sys.stderr) + title = $PROJECT + ' v' + $VERSION + head = username + ':' + $VERSION + body = ('Merge only after success.\n\n' + 'This pull request was auto-generated by ' + '[rever](https://regro.github.io/rever-docs/)') + pr = repo.create_pull(title, 'master', head, body=body) + if pr is None: + print_color('{RED}Failed to create pull request!{NO_COLOR}') + else: + print_color('{GREEN}Pull request created at ' + pr.html_url + \ + '{NO_COLOR}') + + def check_func(self): + """Checks that we can rerender and login""" + rerender = ![conda-smithy regenerate --check] + return rerender and github.can_login() diff --git a/rever/environ.xsh b/rever/environ.xsh index 70656b7..5852e72 100644 --- a/rever/environ.xsh +++ b/rever/environ.xsh @@ -48,6 +48,7 @@ def default_dag(env): from rever.activities.changelog import Changelog from rever.activities.check import Check from rever.activities.conda_forge import CondaForge + from rever.activities.custom_forge import CustomForge from rever.activities.docker import DockerBuild, DockerPush from rever.activities.ghpages import GHPages from rever.activities.ghrelease import GHRelease @@ -66,6 +67,7 @@ def default_dag(env): 'changelog': Changelog(), 'check': Check(), 'conda_forge': CondaForge(), + 'custom_forge': CustomForge(), 'docker_build': DockerBuild(), 'docker_push': DockerPush(), 'ghpages': GHPages(),