diff --git a/.github/actionlint-matcher.json b/.github/actionlint-matcher.json deleted file mode 100644 index a99709f7170..00000000000 --- a/.github/actionlint-matcher.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "problemMatcher": [ - { - "owner": "actionlint", - "pattern": [ - { - "code": 5, - "column": 3, - "file": 1, - "line": 2, - "message": 4, - "regexp": "^(?:\\x1b\\[\\d+m)?(.+?)(?:\\x1b\\[\\d+m)*:(?:\\x1b\\[\\d+m)*(\\d+)(?:\\x1b\\[\\d+m)*:(?:\\x1b\\[\\d+m)*(\\d+)(?:\\x1b\\[\\d+m)*: (?:\\x1b\\[\\d+m)*(.+?)(?:\\x1b\\[\\d+m)* \\[(.+?)\\]$" - } - ] - } - ] -} diff --git a/.github/sync.yml b/.github/sync.yml index accbec4adef..80e74da3cc1 100644 --- a/.github/sync.yml +++ b/.github/sync.yml @@ -15,6 +15,8 @@ group: dest: automations/js/src/project_automation/ - source: automations/js/src/utils/ dest: automations/js/src/utils/ + - source: automations/js/src/label_pr.mjs + dest: automations/js/src/label_pr.mjs # Synced workflows - source: .github/workflows/issue_automations.yml dest: .github/workflows/issue_automations.yml @@ -22,16 +24,10 @@ group: dest: .github/workflows/pr_automations.yml - source: .github/workflows/pr_automations_init.yml dest: .github/workflows/pr_automations_init.yml - - source: .github/workflows/label_new_pr.yml - dest: .github/workflows/label_new_pr.yml - source: .github/workflows/pr_label_check.yml dest: .github/workflows/pr_label_check.yml - source: .github/workflows/pr_ping.yml dest: .github/workflows/pr_ping.yml - - source: .github/workflows/actionlint.yml - dest: .github/workflows/actionlint.yml - - source: .github/actionlint-matcher.json - dest: .github/actionlint-matcher.json - source: .github/workflows/subscribe_to_label.yml dest: .github/workflows/subscribe_to_label.yml - source: .github/subscribe-to-label.json diff --git a/.github/workflows/actionlint.yml b/.github/workflows/actionlint.yml deleted file mode 100644 index 3309aa0ff10..00000000000 --- a/.github/workflows/actionlint.yml +++ /dev/null @@ -1,24 +0,0 @@ -name: Lint GitHub Actions workflows - -on: - pull_request: - push: - branches: - - main - -# Cancels all previous workflow runs for pull requests that have not completed. -concurrency: - # The concurrency group contains the workflow name and the branch name for pull requests - # or the commit hash for any other events. - group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }} - cancel-in-progress: true - -jobs: - actionlint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Check workflow files - uses: docker://rhysd/actionlint:latest - with: - args: -color diff --git a/.github/workflows/label_new_pr.yml b/.github/workflows/label_new_pr.yml deleted file mode 100644 index 3819104763c..00000000000 --- a/.github/workflows/label_new_pr.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: Label new PR -# ℹ️ https://github.com/WordPress/openverse/blob/main/.github/GITHUB.md#label-new-pr - -on: - pull_request_target: - types: - # Only set labels when the PR is created, this should fire `labeled` - # events that will trigger the checks defined in `pr_label_check.yml`. - - opened - -jobs: - set_label: - name: Set labels - runs-on: ubuntu-latest - steps: - - name: Trigger remote workflow - uses: felixp8/dispatch-and-wait@v0.1.0 - env: - PR_URL: ${{ github.event.pull_request.html_url }} - with: - owner: WordPress - repo: openverse - token: ${{ secrets.ACCESS_TOKEN }} # Cannot create repository dispatch using GITHUB_TOKEN - event_type: label_pr - client_payload: ${{ format('{{"pr_url":"{0}"}}', env.PR_URL) }} - wait_time: 5 # check every 5 seconds - max_time: 120 # timeout after 2 minutes - - - uses: actions/checkout@v4 - - name: Add default label in case of failure - uses: actions-ecosystem/action-add-labels@v1 - if: ${{ failure() }} - with: - labels: "🚦 status: awaiting triage" diff --git a/.github/workflows/label_pr.yml b/.github/workflows/label_pr.yml deleted file mode 100644 index eeef2d3d6a1..00000000000 --- a/.github/workflows/label_pr.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: Label PR - -on: - repository_dispatch: - types: - - label_pr - -env: - ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }} # Can't use GITHUB_TOKEN to access PRs in other repos - PR_URL: ${{ github.event.client_payload.pr_url }} - GH_LOGIN: ${{ secrets.GH_LOGIN }} - GH_PASSWORD: ${{ secrets.GH_PASSWORD }} - GH_2FA_SECRET: ${{ secrets.GH_2FA_SECRET }} - -jobs: - label_pr: - name: Label PR - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Setup CI env - uses: ./.github/actions/setup-env - with: - setup_nodejs: false # Node.js is not needed to run Python automations. - install_recipe: "automations/python/install" - - - name: Label PR - working-directory: ./automations/python - run: | - pipenv run python label_pr.py \ - --pr-url "$PR_URL" diff --git a/.github/workflows/label_sync.yml b/.github/workflows/label_sync.yml index 0b1d2b13447..d4d3710faea 100644 --- a/.github/workflows/label_sync.yml +++ b/.github/workflows/label_sync.yml @@ -6,10 +6,6 @@ on: schedule: - cron: "0 0 * * *" # at 00:00 -env: - LOGGING_LEVEL: 20 # corresponds to INFO - ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }} - jobs: sync_labels: name: Sync labels @@ -18,13 +14,10 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 - - name: Setup CI env - uses: ./.github/actions/setup-env + - name: Sync labels from monorepo to infra + uses: actions/github-script@v7 with: - setup_nodejs: false # Node.js is not needed to run Python automations. - install_recipe: "automations/python/install" - - - name: Sync standard labels - working-directory: ./automations/python - run: | - pipenv run python sync_labels.py + github-token: ${{ secrets.ACCESS_TOKEN }} + script: | + const { main } = await import('${{ github.workspace }}/automations/js/src/sync_labels.mjs') + await main(github, core) diff --git a/.github/workflows/pr_automations.yml b/.github/workflows/pr_automations.yml index 9bd165b4c6d..4270cbbb92c 100644 --- a/.github/workflows/pr_automations.yml +++ b/.github/workflows/pr_automations.yml @@ -65,10 +65,18 @@ jobs: unzip event_info.zip mv event.json /tmp/event.json + - name: Perform PR labelling + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.ACCESS_TOKEN }} + script: | + const { main } = await import('${{ github.workspace }}/automations/js/src/label_pr.mjs') + await main(github) + - name: Perform PR automations uses: actions/github-script@v7 with: github-token: ${{ secrets.ACCESS_TOKEN }} script: | const { main } = await import('${{ github.workspace }}/automations/js/src/project_automation/prs.mjs') - await main(github) + await main(github, core) diff --git a/.prettierignore b/.prettierignore index 845e1475829..12be1ee975a 100644 --- a/.prettierignore +++ b/.prettierignore @@ -19,5 +19,8 @@ api/docs/_templates/page.html # Django templates api/api/templates +# One-time scripts +utilities/migrate_issues/ + # Autogenerated pnpm-lock.yaml diff --git a/automations/js/src/label_pr.mjs b/automations/js/src/label_pr.mjs new file mode 100644 index 00000000000..ece81a1d2b2 --- /dev/null +++ b/automations/js/src/label_pr.mjs @@ -0,0 +1,119 @@ +import { readFileSync } from 'fs' +import { PullRequest } from './utils/pr.mjs' +import { IdSet } from './utils/id_set.mjs' + +const exactlyOne = ['priority', 'goal'] +const atleastOne = ['aspect'] +const atleastOneCheckOnly = ['stack'] + +/** + * Check if the list of labels covers all requirements. + * + * @param labels {import('./utils/pr.mjs').Label[]} the list of labels + * @returns {boolean} whether the list of labels covers all requirements + */ +function getIsFullyLabeled(labels) { + for (let req of exactlyOne) { + if (labels.filter((label) => label.name.includes(req)).length !== 1) { + return false + } + } + for (let req of atleastOne + atleastOneCheckOnly) { + if (labels.filter((label) => label.name.includes(req)).length < 1) { + return false + } + } + return true +} + +/** + * Apply labels to a PR based on the PR's linked issues. + * + * Note that this function does not concern itself with the management of stack + * labels as that is performed by a job in the CI + CD workflow. + * + * @param octokit {import('@octokit/rest').Octokit} the Octokit instance to use + * @param core {import('@actions/core')} GitHub Actions toolkit, for logging + */ +export const main = async (octokit, core) => { + const { eventName, eventAction, prNodeId } = JSON.parse( + readFileSync('/tmp/event.json', 'utf-8') + ) + + if ( + eventName !== 'pull_request' || + !['opened', 'edited'].includes(eventAction) + ) { + core.info( + `Event "${eventName}"/"${eventAction}" is not an event where a PR should be labelled.` + ) + return + } + + const pr = new PullRequest(octokit, core, prNodeId) + await pr.init() + + let isTriaged = false + if (pr.labels && pr.labels.some((label) => !label.name.includes('stack'))) { + // If a PR has non-stack labels, it has likely been triaged by a maintainer. + core.info('The PR already has non-stack labels.') + isTriaged = true + } + + // The logic for labelling a PR is as follows. + const finalLabels = new IdSet() + + // We start with the PRs current labels. We do not remove any labels already + // set as they could be the work of the CI labeller job or a maintainer. + pr.labels.forEach((label) => { + core.debug(`Adding label "${label.name}" from PR.`) + finalLabels.add(label) + }) + + // Then we compile all the labels of all the linked issues into a pool. This + // will be used to find the labels that satisfy the requirements. + const labelPool = pr.linkedIssues.flatMap((issue) => issue.labels) + core.debug(`Label pool: ${labelPool}`) + + // For each label that we only need one of, we check if the PR already has + // such a label. If not, we check if the label pool contains any valid labels + // and add the first one we find. + for (let rule of exactlyOne) { + if (finalLabels.items.some((label) => label.name.includes(rule))) { + core.info(`PR already has a "${rule}" label.`) + continue + } + const validLabel = labelPool.find((label) => label.name.includes(rule)) + if (validLabel) { + core.info(`Adding label "${validLabel.name}" to PR.`) + finalLabels.add(validLabel) + } + } + + // For each label that we need at least one of, we add all the valid labels + // from the label pool. Our ID set implementation will weed out duplicates. + for (let rule of atleastOne) { + const validLabels = labelPool.filter((label) => label.name.includes(rule)) + core.info(`Adding labels "${validLabels}" to PR.`) + validLabels.forEach((label) => { + finalLabels.add(label) + }) + } + + // We check if the label is fully labeled. If not, we add the appropriate + // label to get the maintainers' attention. + if (!getIsFullyLabeled(finalLabels.items)) { + let attnLabel + if (isTriaged) { + attnLabel = '🏷 status: label work required' + } else { + attnLabel = '🚦 status: awaiting triage' + } + core.info(`Pull not fully labelled so adding "${attnLabel}".`) + finalLabels.add(attnLabel) + } + + // Finally we commit all label IDs to the PR via a mutation. GitHub will only + // add the new IDs we provide, no existing label will be changed. + await pr.addLabels(Array.from(finalLabels.ids)) +} diff --git a/automations/js/src/last_week_tonight.mjs b/automations/js/src/last_week_tonight.mjs index e4b4058a883..536116de6b0 100644 --- a/automations/js/src/last_week_tonight.mjs +++ b/automations/js/src/last_week_tonight.mjs @@ -10,7 +10,7 @@ import yaml from 'js-yaml' import axios from 'axios' import { Octokit } from '@octokit/rest' -import { escapeHtml } from './html.mjs' +import { escapeHtml } from './utils/html.mjs' /* Environment variables */ diff --git a/automations/js/src/project_automation/prs.mjs b/automations/js/src/project_automation/prs.mjs index 4264e8280e6..114e851992c 100644 --- a/automations/js/src/project_automation/prs.mjs +++ b/automations/js/src/project_automation/prs.mjs @@ -34,7 +34,7 @@ async function syncReviews(pr, prBoard, prCard) { */ async function syncIssues(pr, backlogBoard, destColumn) { for (let linkedIssue of pr.linkedIssues) { - const issueCard = await backlogBoard.addCard(linkedIssue) + const issueCard = await backlogBoard.addCard(linkedIssue.id) await backlogBoard.moveCard(issueCard.id, backlogBoard.columns[destColumn]) } } @@ -43,13 +43,14 @@ async function syncIssues(pr, backlogBoard, destColumn) { * This is the entrypoint of the script. * * @param octokit {import('@octokit/rest').Octokit} the Octokit instance to use + * @param core {import('@actions/core')} GitHub Actions toolkit, for logging */ -export const main = async (octokit) => { +export const main = async (octokit, core) => { const { eventName, eventAction, prNodeId } = JSON.parse( readFileSync('/tmp/event.json', 'utf-8') ) - const pr = new PullRequest(octokit, prNodeId) + const pr = new PullRequest(octokit, core, prNodeId) await pr.init() const prBoard = await getBoard(octokit, 'PRs') diff --git a/automations/js/src/sync_labels.mjs b/automations/js/src/sync_labels.mjs new file mode 100644 index 00000000000..613f902a142 --- /dev/null +++ b/automations/js/src/sync_labels.mjs @@ -0,0 +1,97 @@ +/** + * a tag applied to an issue or label + * @typedef {{name: string, color: string, description: string}} Label + */ + +const infraRepo = { + owner: 'WordPress', + repo: 'openverse-infrastructure', +} + +const monoRepo = { + owner: 'WordPress', + repo: 'openverse', +} + +const exclusions = [/^design:/, /^migrations$/] + +/** + * Compare two labels and list the aspects that are different. + * @param a {Label} the first label to compare with the second + * @param b {Label} the second label to compare with the first + * @returns {string[]} a list of differences between the two labels + */ +const cmpLabels = (a, b) => { + const differences = [] + if (a.description !== b.description) { + differences.push('description') + } + if (a.color !== b.color) { + differences.push('color') + } + return differences +} + +/** + * Non-destructively update labels in the `WordPress/openverse-infrastructure` + * repo using the Openverse monorepo as the source of truth. + * + * - Excluded labels from the monorepo will not be synced. + * - Labels with the same name will be updated to match description and color. + * - Monorepo labels not present in the infrastructure repo will be created. + * - Any extra labels in the infrastructure repo will be unchanged. + * + * @param octokit {import('@octokit/rest').Octokit} the Octokit instance to use + * @param core {import('@actions/core')} GitHub Actions toolkit, for logging + */ +export const main = async (octokit, core) => { + // We assume that both repos have < 100 labels and do not paginate. + + /** @type {Label[]} */ + const { data: monoLabels } = await octokit.issues.listLabelsForRepo({ + ...monoRepo, + per_page: 100, + }) + + core.info(`Found ${monoLabels.length} labels in the monorepo.`) + + /** @type {Label[]} */ + const { data: infraLabels } = await octokit.issues.listLabelsForRepo({ + ...infraRepo, + per_page: 100, + }) + + core.info(`Found ${infraLabels.length} labels in the infrastructure repo.`) + + /** @type {{[p: string]: Label}} */ + const infraLabelMap = Object.fromEntries( + infraLabels.map((label) => [label.name, label]) + ) + + for (let label of monoLabels) { + if (exclusions.some((rule) => label.name.match(rule))) { + core.info(`Label "${label.name}" is excluded from sync.`) + continue + } + + const newLabel = { + ...infraRepo, + name: label.name, + description: label.description, + color: label.color, + } + + const infraLabel = infraLabelMap[label.name] + if (infraLabel) { + const diff = cmpLabels(label, infraLabel) + if (diff.length) { + const diffs = diff.join(', ') + core.info(`Label "${label.name}" differs in ${diffs}. Updating.`) + await octokit.issues.updateLabel(newLabel) + } + } else { + core.info(`Label "${label.name}" does not exist. Creating.`) + await octokit.issues.createLabel(newLabel) + } + } +} diff --git a/automations/js/src/html.mjs b/automations/js/src/utils/html.mjs similarity index 100% rename from automations/js/src/html.mjs rename to automations/js/src/utils/html.mjs diff --git a/automations/js/src/utils/id_set.mjs b/automations/js/src/utils/id_set.mjs new file mode 100644 index 00000000000..4378aca7c2c --- /dev/null +++ b/automations/js/src/utils/id_set.mjs @@ -0,0 +1,31 @@ +/** + * An ID set is a set that can uniquely store objects, something a regular `Set` + * cannot do, by using their `id` field as the unique identifier. Ironically, it + * does this by storing the IDs in a regular `Set` under the hood. + */ +export class IdSet { + constructor() { + this.items = [] + this.ids = new Set() + } + + /** + * Add an item to the set. The item must have an `id` field. + * @param item {object | {id: string}} the item to add + */ + add(item) { + if (!this.ids.has(item.id)) { + this.items.push(item) + this.ids.add(item.id) + } + } + + /** + * Check if the set contains the given item. + * @param item {object | {id: string}} the item to check + * @returns {boolean} whether the set contains the item + */ + has(item) { + return this.ids.has(item.id) + } +} diff --git a/automations/js/src/utils/pr.mjs b/automations/js/src/utils/pr.mjs index f31f3a6c8f5..bcc9b0b3cdd 100644 --- a/automations/js/src/utils/pr.mjs +++ b/automations/js/src/utils/pr.mjs @@ -5,8 +5,21 @@ * the state of a particular review left on a PR * @typedef {'APPROVED' | 'CHANGES_REQUESTED' | 'COMMENTED' | 'DISMISSED' | 'PENDING'} ReviewState * + * a tag applied to an issue or a PR + * @typedef {{id: string, name: string}} Label + * + * the linked issue of a PR + * @typedef {{id: string, labels: Label[]}} Issue + * * the additional information about the PR obtained from a GraphQL query - * @typedef {{reviewDecision: ReviewDecision, linkedIssues: string[], reviewStates: ReviewState[]}} PrDetails + * @typedef {{ + * isMerged: boolean, + * isDraft: boolean, + * reviewDecision: ReviewDecision, + * linkedIssues: Issue[], + * reviewStates: ReviewState[], + * labels: Label[], + * }} PrDetails */ export class PullRequest { @@ -15,21 +28,22 @@ export class PullRequest { * as opposed to the conventional `id` or `number` fields. * * @param octokit {import('@octokit/rest').Octokit} the Octokit instance to use + * @param core {import('@actions/core')} GitHub Actions toolkit, for logging * @param nodeId {boolean} the `node_id` of the PR for GraphQL requests */ - constructor(octokit, nodeId) { + constructor(octokit, core, nodeId) { this.octokit = octokit + this.core = core this.nodeId = nodeId } + /** + * Initialise the PR and populate fields that require API call to GitHub. + */ async init() { const prDetails = await this.getPrDetails() - this.linkedIssues = prDetails.linkedIssues - this.reviewDecision = prDetails.reviewDecision - this.reviewStates = prDetails.reviewStates - this.isDraft = prDetails.isDraft - this.isMerged = prDetails.isMerged + Object.assign(this, prDetails) } /** @@ -46,9 +60,21 @@ export class PullRequest { isDraft merged reviewDecision + labels(first: 20) { + nodes { + id + name + } + } closingIssuesReferences(first: 10) { nodes { id + labels(first: 20) { + nodes { + id + name + } + } } } reviews(first: 100) { @@ -68,11 +94,53 @@ export class PullRequest { isMerged: pr.isMerged, isDraft: pr.merged, reviewDecision: pr.reviewDecision, - linkedIssues: pr.closingIssuesReferences.nodes.map((node) => node.id), + linkedIssues: pr.closingIssuesReferences.nodes.map((node) => ({ + id: node.id, + labels: node.labels.nodes, + })), reviewStates: pr.reviews.nodes.map((node) => node.state), + labels: pr.labels.nodes, } } + /** + * Add the given list of labels to the PR, leaving all existing labels + * unaffected by this change. + * + * This call is idempotent in that once a specific label is added by this + * method, any subsequent calls, with or without that ID, will not remove the + * label. + * + * @param labelIds {string[]} the list of label IDs to add to the PR + * @returns {Promise} the final list of labels on the PR + */ + async addLabels(labelIds) { + this.core.info(`Adding labels with IDs ${labelIds} to PR.`) + const res = await this.octokit.graphql( + `mutation addLabels($labelableId: ID!, $labelIds: [ID!]!) { + addLabelsToLabelable(input: { + labelableId: $labelableId, + labelIds: $labelIds + }) { + labelable { + labels(first: 20) { + nodes { + id + name + } + } + } + } + }`, + { + labelableId: this.nodeId, + labelIds, + } + ) + this.core.debug('addLabels response:', JSON.stringify(res)) + return res.addLabelsToLabelable.labelable.labels.nodes + } + /** * Get the count of each type of PR reviews. * diff --git a/automations/js/src/utils/projects.mjs b/automations/js/src/utils/projects.mjs index 3a307d78ac2..a38d0be8947 100644 --- a/automations/js/src/utils/projects.mjs +++ b/automations/js/src/utils/projects.mjs @@ -42,8 +42,7 @@ class Project { */ async init() { const projectDetails = await this.getProjectDetails() - this.projectId = projectDetails.projectId - this.fields = projectDetails.fields + Object.assign(this, projectDetails) this.columns = this.getColumns() } diff --git a/automations/python/archive_column.py b/automations/python/archive_column.py deleted file mode 100644 index 9232fbd29ee..00000000000 --- a/automations/python/archive_column.py +++ /dev/null @@ -1,65 +0,0 @@ -import argparse -import logging - -from shared.data import get_data -from shared.github import get_client -from shared.log import configure_logger -from shared.project import get_org_project, get_project_column - - -log = logging.getLogger(__name__) - -# region argparse -parser = argparse.ArgumentParser( - description="Archive all cards in the given column of a project", -) -parser.add_argument( - "--project-number", - dest="proj_number", - metavar="project-number", - type=int, - required=True, - help="the project in which to the column to archive is located", -) -parser.add_argument( - "--column", - dest="col_name", - metavar="column", - type=str, - required=True, - help="column from which to archive the cards", -) - - -# endregion - - -def main(): - configure_logger() - - args = parser.parse_args() - - log.debug(f"Project number: {args.proj_number}") - log.debug(f"Column name: {args.col_name}") - - github_info = get_data("github.yml") - org_handle = github_info["org"] - log.info(f"Organization handle: {org_handle}") - - gh = get_client() - org = gh.get_organization(org_handle) - - proj = get_org_project(org=org, proj_number=args.proj_number) - log.info(f"Found project: {proj.name}") - column = get_project_column(proj=proj, col_name=args.col_name) - log.debug("Found column") - - cards_to_archive = list(column.get_cards(archived_state="not archived")) - log.info(f"Found {len(cards_to_archive)} cards") - for card in cards_to_archive: - card.edit(archived=True) - log.info("Archived all cards in column") - - -if __name__ == "__main__": - main() diff --git a/automations/python/issues_with_prs.py b/automations/python/issues_with_prs.py deleted file mode 100755 index 39838d1021d..00000000000 --- a/automations/python/issues_with_prs.py +++ /dev/null @@ -1,256 +0,0 @@ -#!/usr/bin/env python3 -import argparse -import logging -import sys -from typing import Literal - -import requests -from github import Github, ProjectCard, ProjectColumn -from shared.data import get_data -from shared.github import get_access_token, get_client -from shared.log import configure_logger -from shared.project import get_org_project, get_project_column - - -log = logging.getLogger(__name__) - -CLOSED = "closed" -OPEN = "open" -IssueState = Literal[CLOSED, OPEN] -# Unique identifier of repo + issue -RepoIssue = tuple[str, str] - -LINKED_PR_QUERY = """ -{ - repository(owner: "%s", name: "%s") { - pullRequests(first: 100, states:%s, - orderBy:{field:UPDATED_AT, direction:DESC}) { - nodes { - number - title - closingIssuesReferences (first: 50) { - edges { - node { - number - title - state - } - } - } - } - } - } -} -""" - - -# region argparse -parser = argparse.ArgumentParser( - description="Move issues to the correct columns in projects", -) -parser.add_argument( - "--project-number", - dest="proj_number", - metavar="project-number", - type=int, - required=True, - help="the project in which to move cards containing issues with PRs", -) -parser.add_argument( - "--source-column", - dest="source_col_name", - metavar="source-column", - type=str, - default="To do", - help="column from which to move cards containing issues with PRs", -) -parser.add_argument( - "--target-column", - dest="target_col_name", - metavar="target-column", - type=str, - default="In progress", - help="column in which to move cards containing issues with PRs", -) -parser.add_argument( - "--linked-pr-state", - dest="linked_pr_state", - metavar="linked-pr-state", - type=str, - default=OPEN, - choices=[OPEN, CLOSED], - help="filter issues by this state of their linked PRs", -) - -# endregion - - -def run_query( - query, -) -> dict: - """ - Run a GitHub GraphQL query. - Taken from https://gist.github.com/gbaman/b3137e18c739e0cf98539bf4ec4366ad - """ - log.debug(f"{query=}") - request = requests.post( - "https://api.github.com/graphql", - json={"query": query}, - headers={"Authorization": f"Bearer {get_access_token()}"}, - ) - if request.status_code == 200: - results = request.json() - log.debug(f"{results=}") - return request.json().get("data") - else: - raise Exception( - "Query failed to run by returning code of {}. {}".format( - request.status_code, query - ) - ) - - -def get_pulls_with_linked_issues( - org_handle: str, repo_name: str, state: IssueState -) -> dict[str, str]: - """ - Query all the pull requests with issues of a linked state. - - This uses the GraphQL API since the REST API doesn't support querying for linked - issues. - - :return: a dict of issue numbers to issue titles - """ - results = run_query(LINKED_PR_QUERY % (org_handle, repo_name, state.upper())) - pulls = results["repository"]["pullRequests"]["nodes"] - log.info(f"Found {len(pulls)} {state} PRs in {org_handle}/{repo_name}") - return { - issue["node"]["number"]: issue["node"]["title"] - for pull in pulls - for issue in pull["closingIssuesReferences"]["edges"] - } - - -def get_open_issues_with_prs( - gh: Github, - org_handle: str, - repo_names: list[str], - linked_pr_state: str, -) -> set[RepoIssue]: - """ - Retrieve open issues with linked PRs. - - :param gh: the GitHub client - :param org_handle: the name of the org in which to look for issues - :param repo_names: the name of the repos in which to look for issues - :param linked_pr_state: the state of the linked PRs to filter by - :return: a set of tuples containing the repo name and issue number for each issue - """ - - all_issues = set() - for repo_name in repo_names: - log.info( - f"Looking for {linked_pr_state} PRs in {org_handle}/{repo_name} " - f"with linked issues" - ) - issues = get_pulls_with_linked_issues(org_handle, repo_name, linked_pr_state) - # In the case where we're querying for closed PRs, we'll also need to consider - # open PRs - if there's an issue with both an open PR and a closed PR, we should - # not take any action on it - open_issues = {} - if linked_pr_state == CLOSED: - open_issues = get_pulls_with_linked_issues(org_handle, repo_name, OPEN) - - log.info(f"Found {len(issues)} issues") - for number, title in issues.items(): - log.info(f"• #{number: >5} | {title}") - if linked_pr_state == CLOSED and number in open_issues: - log.info(f"{' ' * 11}(skipped because there's an open PR)") - continue - all_issues.add((repo_name, number)) - - log.info( - f"Found a total of {len(all_issues)} open issues " - f"with linked {linked_pr_state} PRs" - ) - return all_issues - - -def get_issue_cards(col: ProjectColumn) -> list[tuple[ProjectCard, RepoIssue]]: - """ - Get all cards linked to issues in the given column. - - This excludes cards that either have no links (just notes) or are linked to PRs. - - :param col: the project column from which to retrieve issue cards - :return: the list of cards linked to issues in the given column - """ - - cards = col.get_cards(archived_state="not archived") - issue_cards = [] - for card in cards: - # Example URL: https://api.github.com/repos/api-playground/project-test/issues/3 - # See: https://docs.github.com/en/rest/projects/cards?apiVersion=2022-11-28 - url = card.content_url - try: - url_parts = url.split("/") - # Repo + issue - issue = (url_parts[-3], url_parts[-1]) - except Exception: - log.debug(f"Could not decode card with content_url: {url}") - continue - issue_cards.append((card, issue)) - return issue_cards - - -def main(): - configure_logger() - - args = parser.parse_args() - - log.debug(f"Project number: {args.proj_number}") - log.debug(f"Source column name: {args.source_col_name}") - log.debug(f"Target column name: {args.target_col_name}") - log.debug(f"Linked issue PR state: {args.linked_pr_state}") - - github_info = get_data("github.yml") - org_handle = github_info["org"] - log.info(f"Organization handle: {org_handle}") - repo_names = github_info["repos"].values() - log.info(f"Repository names: {', '.join(repo_names)}") - - gh = get_client() - org = gh.get_organization(org_handle) - - issues_with_prs = get_open_issues_with_prs( - gh=gh, - org_handle=org_handle, - repo_names=repo_names, - linked_pr_state=args.linked_pr_state, - ) - if len(issues_with_prs) == 0: - log.warning("Found no issues with PRs, stopping") - sys.exit() - - proj = get_org_project(org=org, proj_number=args.proj_number) - log.info(f"Found project: {proj.name}") - source_column = get_project_column(proj=proj, col_name=args.source_col_name) - log.debug("Found source column") - target_column = get_project_column(proj=proj, col_name=args.target_col_name) - log.debug("Found target column") - - issue_cards = get_issue_cards(source_column) - - cards_to_move = [] - for issue_card, issue in issue_cards: - if issue in issues_with_prs: - cards_to_move.append((issue_card, issue)) - log.info(f"Found {len(cards_to_move)} cards to move") - - for issue_card, issue in cards_to_move: - log.info(f"Moving card for issue {issue.html_url} to {target_column.name}") - issue_card.move("bottom", target_column) - - -if __name__ == "__main__": - main() diff --git a/automations/python/label_pr.py b/automations/python/label_pr.py deleted file mode 100644 index ece95ce6c61..00000000000 --- a/automations/python/label_pr.py +++ /dev/null @@ -1,241 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import argparse -import logging -import os -import re -from http.cookiejar import CookieJar - -import mechanize -import pyotp -import requests -from bs4 import BeautifulSoup -from github import Github -from github.Issue import Issue -from github.Label import Label -from github.PullRequest import PullRequest -from shared.data import get_data -from shared.github import get_client -from shared.log import configure_logger - - -log = logging.getLogger(__name__) - -# region argparse -parser = argparse.ArgumentParser(description="") -parser.add_argument( - "--pr-url", - dest="pr_url", - metavar="pr-url", - required=True, - help="the URL for the PR to label and/or check for labels", -) - - -# endregion - - -def get_owner_repo_num(html_url: str) -> tuple[str, str, int]: - """ - Get the owner and repo name and issue/PR number from the HTML URL. - - :param html_url: the URL from which to get the owner and repo name and issue/PR num - :return: the owner and repo name and the issue/PR number - :raise: ``ValueError``, if the data URL does not match the pattern - """ - - pattern = re.compile( - r"https://github.com/(?P.+)/(?P.+)/(issues|pull)/(?P\d+)" - ) - if (match := pattern.match(html_url)) is not None: - return match.group("owner"), match.group("repo"), int(match.group("num")) - raise ValueError("Could not identify owner and repo name and issue/PR number") - - -def get_issue(gh: Github, html_url: str) -> Issue: - """ - Get the ``Issue`` instance from a GitHub issue's HTML URL. - - :param gh: the GitHub client - :param html_url: the HTML URL of the issue - :return: the ``Issue`` instance corresponding to the given URL - """ - - org, repo, issue_num = get_owner_repo_num(html_url) - return gh.get_organization(org).get_repo(repo).get_issue(issue_num) - - -def get_pull_request(gh: Github, html_url: str) -> PullRequest: - """ - Get the ``PullRequest`` instance from a GitHub PR's HTML URL. - - :param gh: the GitHub client - :param html_url: the HTML URL of the PR - :return: the ``PullRequest`` instance corresponding to the given URL - """ - - org, repo, pr_num = get_owner_repo_num(html_url) - return gh.get_organization(org).get_repo(repo).get_pull(pr_num) - - -def get_authenticated_html(url: str) -> str: - """ - Retrieve HTML for GitHub webpages that require authentication to view. - - Login to the GitHub UI using the username and password, followed by 2FA. - Then navigate to the specified URL as the authenticated user and scrape the - text body. - - :param url: the URL to scrape after authenticating in GitHub - :return: the text content of the scraped page - """ - - browser = mechanize.Browser() - browser.set_cookiejar(CookieJar()) - browser.open("https://github.com/login/") - try: - browser.select_form(nr=0) # focus on the first (and only) form on the page - browser.form["login"] = os.getenv("GH_LOGIN") - browser.form["password"] = os.getenv("GH_PASSWORD") - browser.submit() - - browser.select_form(nr=0) # focus on the first (and only) form on the page - browser.form["app_otp"] = pyotp.TOTP(os.getenv("GH_2FA_SECRET")).now() - browser.submit() - except mechanize.ControlNotFoundError as err: - form_name = err.args[0].replace("no control matching name ", "").strip("'") - raise ValueError( - f"Unable to locate form input '{form_name}' on GitHub login page, " - "perhaps its name has changed?" - ) - - browser.open(url) - return browser.response().read() - - -def get_linked_issues(url: str) -> list[str]: - """ - Get the list of linked issues from the GitHub UI by parsing the HTML. - - This is a workaround because GitHub API does not provide a reliable way - to find the linked issues for a PR. - - If the page returns a 404 response, it is assumed that the page belongs to a private - repository and needs authentication. In these cases, ``get_authenticated_html`` is - used to scrape the page. - - :param url: the URL to the GitHub UI for a PR - :return: the list of HTML URLs for linked issues - """ - - res = requests.get(url) - if res.status_code == 404: - text = get_authenticated_html(url) - elif res.status_code == 200: - text = res.text - else: - return [] - - soup = BeautifulSoup(text, "html.parser") - form = soup.find("form", **{"aria-label": "Link issues"}) - if form is None: - return [] - return [a["href"] for a in form.find_all("a")] - - -def get_all_labels_of_cat(cat: str, labels: list[Label]) -> list[Label]: - """ - Get all the available labels from a category from a given list of - labels. - - :param cat: the category to which the label should belong - :param labels: the list of labels to choose from - :return: the labels matching the given category - """ - available_labels = [] - for label in labels: - if cat in label.name: - available_labels.append(label) - return available_labels - - -def get_label_of_cat(cat: str, labels: list[Label]) -> Label | None: - """ - Get the label of a particular category from the given list of labels. - - :param cat: the category to which the label should belong - :param labels: the list of labels to choose from - :return: the label matching the given category - """ - return next(iter(get_all_labels_of_cat(cat, labels)), None) - - -def main(): - configure_logger() - - args = parser.parse_args() - - log.debug(f"PR URL: {args.pr_url}") - - label_info = get_data("labels.yml") - label_groups = label_info["groups"] - required_label_categories = [ - i.get("name") for i in label_groups if i.get("is_required") - ] - # Categories where all labels should be retrieved rather than first only - categories_with_all_labels = [ - i.get("name") for i in label_groups if i.get("apply_all_available") - ] - - github_info = get_data("github.yml") - org_handle = github_info["org"] - log.info(f"Organization handle: {org_handle}") - - gh = get_client() - - pr = get_pull_request(gh, args.pr_url) - log.info(f"Found PR: {pr.title}") - - # Skip a PR if it already has labels, as long as it has labels that are NOT a stack - # label. Stack labels are applied automatically in another part of the CI/CD. - # If all of its labels are stack labels, apply the new labels too. - if pr.labels and not all(["stack" in label.name for label in pr.labels]): - log.info("PR already labelled") - return - - linked_issues = get_linked_issues(pr.html_url) - log.info(f"Found {len(linked_issues)} linked issues") - - for issue in linked_issues: - issue = get_issue(gh, issue) - labels = issue.labels - labels_to_add = [] - - for category in required_label_categories: - if category in categories_with_all_labels and ( - available_labels := get_all_labels_of_cat(category, labels) - ): - log.info(f"Found labels for category {category}: {available_labels}") - labels_to_add.extend(available_labels) - elif label := get_label_of_cat(category, labels): - log.info(f"Found label for category {category}: {label}") - labels_to_add.append(label) - - if labels_to_add: - pr.set_labels(*labels_to_add, *pr.labels) - # Only break when all labels are applied, if we're missing any - # then continue to the else to apply the awaiting triage label. - # Stack can have more than one label and on most PRs the CI/CD workflow - # will already have added a stack label by the time this step runs, - # so remove it from the total count of labels to add. - if len(labels_to_add) >= len(required_label_categories) - 1: - break - else: - log.info("Could not find properly labelled issue") - # Only add the triage label onto existing labels, don't replace them - pr.set_labels("🚦 status: awaiting triage", *pr.get_labels()) - - -if __name__ == "__main__": - main() diff --git a/automations/python/print_labels.py b/automations/python/print_labels.py deleted file mode 100644 index 2f77b2a0bca..00000000000 --- a/automations/python/print_labels.py +++ /dev/null @@ -1,8 +0,0 @@ -"""Simple utility for printing all the labels from labels.yml.""" -from shared.labels import get_labels - - -if __name__ == "__main__": - labels = get_labels() - for label in labels: - print(label) diff --git a/automations/python/shared/project.py b/automations/python/shared/project.py deleted file mode 100644 index 7d374b7c5e4..00000000000 --- a/automations/python/shared/project.py +++ /dev/null @@ -1,44 +0,0 @@ -import logging - -from github import Organization, Project, ProjectColumn - - -log = logging.getLogger(__name__) - - -def get_org_project(org: Organization, proj_number: int) -> Project: - """ - Get the project with the given number in the given organization. - - :param org: the organization in which to find the project - :param proj_number: the number of the project to find in the organization - :return: the project being searched for - :raise: ValueError if no project found with given number - """ - - log.info(f"Getting project {proj_number} in org {org.name or org.login}") - projects = org.get_projects() - project = next(proj for proj in projects if proj.number == proj_number) - if project is None: - log.error(f"No project was found with number {proj_number}.") - raise ValueError("Project not found") - return project - - -def get_project_column(proj: Project, col_name: str) -> ProjectColumn: - """ - Get the project column with the given name in the given project. - - :param proj: the project in which to find the column - :param col_name: the name of the project column to find in the project - :return: the project column being searched for - :raise: ValueError if no project column found with given name - """ - - log.info(f"Getting column {col_name} in project {proj.name}") - columns = proj.get_columns() - column = next(col for col in columns if col.name == col_name) - if column is None: - log.error(f"No column was found with name {col_name}.") - raise ValueError("Column not found") - return column diff --git a/automations/python/sync_labels.py b/automations/python/sync_labels.py deleted file mode 100644 index 395a47fc63b..00000000000 --- a/automations/python/sync_labels.py +++ /dev/null @@ -1,65 +0,0 @@ -import logging - -from github import Repository -from models.label import Label -from shared.data import get_data -from shared.github import get_client -from shared.labels import get_labels -from shared.log import configure_logger - - -log = logging.getLogger(__name__) - - -def set_labels(repo: Repository, labels: list[Label]): - """ - Set the given list of labels on the given repository. - - Missing labels will be added, but extraneous labels will - be left intact. - - :param repo: the repo in which to set the given labels - :param labels: the list of labels to define on the repo - """ - - log.info(f"Fetching existing labels from {repo.full_name}") - existing_labels = {label.name.casefold(): label for label in repo.get_labels()} - log.info(f"Found {len(existing_labels)} existing labels") - - for label in labels: - qualified_name = label.qualified_name - folded_name = qualified_name.casefold() - if folded_name not in existing_labels: - log.info(f"Creating label {qualified_name}") - repo.create_label(**label.api_arguments) - elif label != existing_labels[folded_name]: - log.info(f"Updating label {qualified_name}") - existing_label = existing_labels[folded_name] - existing_label.edit(**label.api_arguments) - else: - log.info(f"Label {qualified_name} already exists") - - -def main(): - configure_logger() - - github_info = get_data("github.yml") - org_handle = github_info["org"] - log.info(f"Organization handle: {org_handle}") - repo_names = github_info["repos"].values() - log.info(f"Repository names: {', '.join(repo_names)}") - - gh = get_client() - org = gh.get_organization(org_handle) - - labels = get_labels() - log.info(f"Synchronizing {len(labels)} standard labels") - for label in labels: - log.info(f"• {label.qualified_name}") - repos = [org.get_repo(repo_name) for repo_name in repo_names] - for repo in repos: - set_labels(repo, labels) - - -if __name__ == "__main__": - main() diff --git a/automations/js/src/migrate_issues.js b/utilities/migrate_issues/migrate_issues.js similarity index 100% rename from automations/js/src/migrate_issues.js rename to utilities/migrate_issues/migrate_issues.js