zashi-pipeline.py

#!/usr/bin/env python3

import networkx as nx
from str2bool import str2bool as strtobool

import itertools
import os
import re
from textwrap import wrap
from urllib.parse import urlparse

from helpers import github, repos as repositories, zenhub

GITHUB_TOKEN = os.environ.get('GITHUB_TOKEN')
ZENHUB_TOKEN = os.environ.get('ZENHUB_TOKEN')

# Repository groups we look for releases in. Each of these groups corresponds to
# a column in the pipeline table; in some cases the releases for that column may
# be spread across several repositories.
RUST = (
    repositories.LIBRUSTZCASH,
    repositories.ZIP32,
)
ANDROID_SDK = (repositories.ZCASH_ANDROID_WALLET_SDK,)
SWIFT_SDK = (repositories.ZCASH_SWIFT_WALLET_SDK,)
ZASHI_ANDROID = (repositories.ZASHI_ANDROID,)
ZASHI_IOS = (repositories.ZASHI_IOS,)

REPOS = github.CORE_REPOS + github.WALLET_REPOS

RELEASE_MATRIX = {
    RUST: [ANDROID_SDK, SWIFT_SDK],
    ANDROID_SDK: [ZASHI_ANDROID],
    SWIFT_SDK: [ZASHI_IOS],
    ZASHI_ANDROID: [],
    ZASHI_IOS: []
}


class Release:
    def __init__(self, repo_group, child):
        self.repo_group = repo_group

        # Extract version number from title
        self.version = None
        if repo_group == RUST:
            version = re.search(r'zcash_[^ ]+ \d+(\.\d+)+', child.title)
            if version:
                self.version = version.group()
                self.version_ints = tuple(int(x) for x in self.version.split(' ')[1].split('.'))
        if self.version is None:
            self.version = re.search(r'\d+(\.\d+)+', child.title).group()
            self.version_ints = tuple(int(x) for x in self.version.split('.'))

        self.is_closed = child.state == 'closed'
        self.url = child.url

    def __repr__(self):
        return self.version

    def __eq__(self, other):
        return (self.repo_group, self.version) == (other.repo_group, other.version)

    def __hash__(self):
        return hash((self.repo_group, self.version))

    def __lt__(self, other):
        return self.version_ints < other.version_ints


def build_release(row, repo_group):
    child = row.get(repo_group)
    if child is None:
        return None
    else:
        return Release(repo_group, child)


class ReleasePipeline:
    def __init__(self, row):
        self.rust = build_release(row, RUST)
        self.android_sdk = build_release(row, ANDROID_SDK)
        self.swift_sdk = build_release(row, SWIFT_SDK)
        self.zashi_android = build_release(row, ZASHI_ANDROID)
        self.zashi_ios = build_release(row, ZASHI_IOS)

    def __repr__(self):
        return '%s | %s | %s | %s | %s' % self.columns()

    def __eq__(self, other):
        return self.columns() == other.columns()

    def __hash__(self):
        return hash(self.columns())

    def __lt__(self, other):
        return self.columns() < other.columns()

    def columns(self):
        return (
            self.rust,
            self.android_sdk,
            self.swift_sdk,
            self.zashi_android,
            self.zashi_ios,
        )

def build_release_matrix_from(dg, issue, repo_group):
    acc = []
    for child in dg.neighbors(issue):
        if child.repo in repo_group and 'C-release' in child.labels:
            # Fetch the rows that each child's downstreams need rendered.
            child_deps = [
                build_release_matrix_from(dg, child, dep_repo)
                for dep_repo in RELEASE_MATRIX.get(repo_group)
            ]

            # Merge the rows from each downstream repo together.
            child_releases = [
                {k: v for d in prod for k, v in d.items()}
                for prod in itertools.product(*child_deps)
            ]

            if len(child_releases) > 0:
                for rec in child_releases:
                    rec[repo_group] = child
            else:
                child_releases = [{repo_group: child}]

            acc.extend(child_releases)
        else:
            acc.extend(build_release_matrix_from(dg, child, repo_group))

    return acc

def main():
    gapi = github.api(GITHUB_TOKEN)
    zapi = zenhub.api(ZENHUB_TOKEN)

    print('Fetching tracked issues')
    tracked_issues = github.download_issues_with_labels(gapi, ['C-tracked-bug', 'C-tracked-feature'], REPOS)

    # The repos we care about are now:
    # - Any repo containing a tracked issue.
    # - The wallet repos where releases occur.
    repos = set([repo for (repo, _) in tracked_issues] + [
        repositories.LIBRUSTZCASH,
        repositories.ZCASH_ANDROID_WALLET_SDK,
        repositories.ZCASH_LIGHT_CLIENT_FFI,
        repositories.ZCASH_SWIFT_WALLET_SDK,
        repositories.ZASHI_ANDROID,
        repositories.ZASHI_IOS,
    ])
    workspaces = {
        workspace_id: [repo for repo in repos if repo in repos]
        for (workspace_id, _) in zenhub.WORKSPACE_SETS.items()
    }

    # Build the full dependency graph from ZenHub's per-workspace API.
    print('Fetching graph')
    dg = nx.compose_all([
        zenhub.get_dependency_graph(zapi, workspace_id, repos)
        for (workspace_id, repos) in workspaces.items()
        if len(repos) > 0
    ])

    print('Rendering deployment pipeline')

    # Ensure that the tracked issues all exist in the graph. This is a no-op for
    # issues that are already present.
    start_at = set([issue for issue in tracked_issues])
    for i in start_at:
        dg.add_node(i)

    # Replace the graph with the subgraph that only includes the tracked
    # issues and their descendants.
    descendants = [nx.descendants(dg, n) for n in start_at]
    dg = nx.subgraph(dg, start_at.union(*descendants))

    # Fetch the issues within the graph.
    mapping = github.download_issues(gapi, dg.nodes, repos)

    # Relabel the graph
    dg = nx.relabel_nodes(dg, mapping)

    # Filter out unknown issues
    unknown = [n for n in dg if n.repo not in repos]
    if len(unknown) > 0:
        dg.remove_nodes_from(unknown)

    # Apply property annotations
    for (source, sink) in dg.edges:
        attrs = dg.edges[source, sink]
        attrs['is_open'] = 0 if source.state == 'closed' else 1

    # Render the HTML version!
    html_header = '''<!DOCTYPE html>
<html>
  <head>
    <title>Zashi Pipeline</title>

    <style>
      body {
        color: #1f2328;
        font-family: -apple-system,BlinkMacSystemFont,"Segoe UI","Noto Sans",Helvetica,Arial,sans-serif,"Apple Color Emoji","Segoe UI Emoji";
        font-size: 14px;
        line-height: 1.5;
        word-wrap: break-word;
      }
      a {
        color: #0969da;
      }
      table {
        border-collapse: collapse;
        width: 100%;
        width: max-content;
        max-width: 100%;
        overflow: auto;
      }
      table tr {
        border-top: 1px solid #d1d9e0b3;
      }
      table th, table td {
        border: 1px solid #d1d9e0b3;
        padding: 6px 13px;
        text-align: center;
      }

      @media (prefers-color-scheme: dark) {
        body {
          background-color: #121212;
          color: #f0f6fc;
        }
        a {
          color: #4493f8;
        }
        table tr {
            border-top: 1px solid #3d444db3;
        }
        table th, table td {
            border: 1px solid #3d444db3;
        }
      }
    </style>
  </head>
  <body>
    <h1>Zashi Pipeline</h1>
    <p>🐞 = bug, 💡 = feature, ✅ = implemented / released, 🛑 = unfinished, 📥 = unassigned / DAG needs updating</p>
    <table>
      <thead>
        <tr>
          <th>Type</th>
          <th>Issue</th>
          <th>Rust crate</th>
          <th>Android SDK</th>
          <th>Swift SDK</th>
          <th>Zashi Android</th>
          <th>Zashi iOS</th>
        </tr>
      </thead>
      <tbody>
'''
    html_footer = '''
      </tbody>
    </table>
  </body>
</html>
'''
    with open('public/zashi-pipeline.html', 'w') as f:
        f.write(html_header)

        for issue in tracked_issues.values():
            rows = [ReleasePipeline(row) for row in build_release_matrix_from(dg, issue, RUST)]

            # If we traversed the entire graph and there are no releases downstream of the
            # issue, show this as an empty row.
            if len(rows) == 0:
                rows = [ReleasePipeline({})]

            # Deduplicate rows
            rows = list(dict.fromkeys(rows))

            # At this point we have a row for each path through the DAG from a tracked bug
            # or feature through a unique set of releases. However, as individual releases
            # within a repo have graph edges between them (e.g. Zashi Android 1.2.3 blocks
            # on Zashi Android 1.2.2), this produces additional rows that we don't care
            # about:
            #
            # - The Zashi repos don't follow SemVer, and are also the ends of the paths;
            #   all we care about here is the earliest Zashi Android and Zashi iOS release
            #   that includes the bug or feature.
            # - The Android and Swift SDKs don't appear to follow SemVer (even though e.g.
            #   Swift Package Manager assumes it), so we also don't care about tracking
            #   whether a bug or feature makes it into both a point release and the next
            #   non-point release.
            # - The Rust crates do follow SemVer; if a bug or feature is deployed in a
            #   point release as a backwards-compatible change, we want to ensure that it
            #   also gets merged back into `main` so that we don't accidentally omit it
            #   from the next breaking crate release. However, we don't need to track
            #   subsequent breaking crate releases after that, because we develop breaking
            #   releases in `main`.

            # Create a map from Rust releases to lists of rows involving that release.
            rust_map = {}
            for row in rows:
                if row.rust in rust_map:
                    rust_map[row.rust].append(row)
                else:
                    rust_map[row.rust] = [row]

            # Remove the Rust releases that we don't need to track.
            rust_releases = list(rust_map.keys())
            rust_releases.sort(reverse=True)
            next_rust_release = rust_releases.pop()
            while len(rust_releases) > 0:
                rust_release = rust_releases.pop()
                if next_rust_release.version_ints[:-1] == rust_release.version_ints[:-1]:
                    # This is a subsequent point release; ignore.
                    del rust_map[rust_release]
                else:
                    # This is the subsequent non-point release.
                    if next_rust_release.version_ints[-1] == 0:
                        # The next release is also a non-point release; ignore this one.
                        del rust_map[rust_release]

                    # Ignore all subsequent releases.
                    for rust_release in rust_releases:
                        del rust_map[rust_release]
                    break

            # For the tracked Rust releases, select the row that has the earliest version
            # number across each of the SDKs and Zashi apps.
            rows = []
            for rust_release in rust_map.values():
                rust_release.sort()
                rows.append(rust_release[0])

            for i, row in enumerate(rows):
                f.write('<tr>')

                if i == 0:
                    rowspan = ''
                    if len(rows) > 1:
                        rowspan = ' rowspan="{}"'.format(len(rows))

                    f.write('<td{}>{}</td>'.format(
                        rowspan,
                        '🐞' if 'C-tracked-bug' in issue.labels else '💡',
                    ))
                    f.write('<td{}>{} <a href="{}">{}</a></td>'.format(
                        rowspan,
                        '✅' if issue.state == 'closed' else '🛑',
                        issue.url,
                        issue.title,
                    ))

                for release in row.columns():
                    if release is None:
                        # Release not found in this repo
                        f.write('<td>📥</td>')
                    else:
                        f.write('<td>{} <a href="{}">{}</a></td>'.format(
                            '✅' if release.is_closed else '🛑',
                            release.url,
                            release.version,
                        ))
                f.write('</tr>')

        f.write(html_footer)


if __name__ == '__main__':
    if GITHUB_TOKEN and ZENHUB_TOKEN:
        main()
    else:
        print('Please set the GITHUB_TOKEN and ZENHUB_TOKEN environment variables.')