From 4fdfc32e9f7ca4af0aa6b30ca03c7deac1366ebb Mon Sep 17 00:00:00 2001 From: Melissa DeLucchi Date: Mon, 1 Dec 2025 11:13:47 -0500 Subject: [PATCH 1/3] Query for open pull requests. --- .../lsdb_interrupts/external_issues.py | 90 ++--------------- .../lsdb_interrupts/github_api.py | 96 +++++++++++++++++++ .../lsdb_interrupts/open_prs.py | 92 ++++++++++++++++++ templates/pr_list.jinja | 43 +++++++++ 4 files changed, 240 insertions(+), 81 deletions(-) create mode 100644 src/lf_workflow_dash/lsdb_interrupts/github_api.py create mode 100644 src/lf_workflow_dash/lsdb_interrupts/open_prs.py create mode 100644 templates/pr_list.jinja diff --git a/src/lf_workflow_dash/lsdb_interrupts/external_issues.py b/src/lf_workflow_dash/lsdb_interrupts/external_issues.py index 7d655a24bca..683c17fd691 100644 --- a/src/lf_workflow_dash/lsdb_interrupts/external_issues.py +++ b/src/lf_workflow_dash/lsdb_interrupts/external_issues.py @@ -4,85 +4,22 @@ organization, and writes them to HTML for human monitoring. """ -import re import sys from collections import defaultdict from datetime import datetime, timezone from typing import Dict, List, Set -import human_readable import requests from jinja2 import Environment, FileSystemLoader -GITHUB_API_BASE = "https://api.github.com" -TEAM_MEMBERS = [ - "delucchi-cmu", - "nevencaplar", - "hombit", - "smcguire-cmu", - "gitosaurus", - "dougbrn", - "olivialynn", - "camposandro", - "wilsonbb", - "mjuric", -] - - -def create_github_session(token) -> requests.Session: - """Create a requests session with GitHub authentication.""" - session = requests.Session() - session.headers.update( - { - "Authorization": f"Bearer {token}", - "Accept": "application/vnd.github+json", - } - ) - return session - - -def paginate_github_api(session: requests.Session, url: str) -> List[Dict]: - """Paginate through GitHub API responses. - - Follows the Link header for pagination as documented in: - https://docs.github.com/en/rest/guides/using-pagination-in-the-rest-api - """ - results = [] - while url: - response = session.get(url) - response.raise_for_status() - data = response.json() - - # Handle both list and dict responses - if isinstance(data, list): - results.extend(data) - else: - results.append(data) - - # Check for pagination using Link header (RFC 8288) - # Format: ; rel="next", <...>; rel="last" - link_header = response.headers.get("Link", "") - url = None - if link_header: - # Match URLs within angle brackets that have rel="next" - # Pattern: ; rel="next" - match = re.search(r'<([^>]+)>;\s*rel="next"', link_header) - if match: - url = match.group(1) - - return results - - -def get_org_repos(org: str, token: str) -> List[str]: - """Get all repos in the org that are related to HATS or LSDB""" - print("Fetching org repositories...") - session = create_github_session(token) - url = f"{GITHUB_API_BASE}/orgs/{org}/repos?per_page=100" - repos_data = paginate_github_api(session, url) - repos = [repo["name"] for repo in repos_data] - repos = [repo for repo in repos if "hats" in repo or "lsdb" in repo] - print(f"Found {len(repos)} repositories.") - return repos +from lf_workflow_dash.lsdb_interrupts.github_api import ( + GITHUB_API_BASE, + TEAM_MEMBERS, + create_github_session, + get_humanized_updated_at, + get_lsdb_repos, + paginate_github_api, +) def get_open_issues(org: str, repos: List[str], org_members: Set[str], token: str) -> List[Dict]: @@ -142,15 +79,6 @@ def get_open_issues(org: str, repos: List[str], org_members: Set[str], token: st return all_issues -def get_humanized_updated_at(iso_time: str, now: datetime) -> str: - """Convenience method to get a human readable duration, like '2 days ago'.""" - try: - dt = datetime.strptime(iso_time, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc) - return human_readable.date_time(dt, when=now) - except Exception: # pylint: disable=broad-except - return iso_time - - def write_html_issues(external_issues: List[Dict], html_file: str): """Fill in the jinja template, using the external interest issues found""" print(f"Writing HTML output to {html_file} ...") @@ -177,7 +105,7 @@ def write_html_issues(external_issues: List[Dict], html_file: str): def main(token, out_file): """Convenience method to do the work.""" - repos = get_org_repos("astronomy-commons", token) + repos = get_lsdb_repos("astronomy-commons", token) external_issues = get_open_issues("astronomy-commons", repos, TEAM_MEMBERS, token) # Sort by most recent activity external_issues.sort(key=lambda x: x["updatedAt"], reverse=True) diff --git a/src/lf_workflow_dash/lsdb_interrupts/github_api.py b/src/lf_workflow_dash/lsdb_interrupts/github_api.py new file mode 100644 index 00000000000..5fb41cc2d4d --- /dev/null +++ b/src/lf_workflow_dash/lsdb_interrupts/github_api.py @@ -0,0 +1,96 @@ +import re +from datetime import datetime, timezone +from typing import Dict, List + +import human_readable +import requests + +GITHUB_API_BASE = "https://api.github.com" +TEAM_MEMBERS = [ + "delucchi-cmu", + "nevencaplar", + "hombit", + "smcguire-cmu", + "gitosaurus", + "dougbrn", + "olivialynn", + "camposandro", + "wilsonbb", + "mjuric", +] + + +def create_github_session(token) -> requests.Session: + """Create a requests session with GitHub authentication.""" + session = requests.Session() + session.headers.update( + { + "Authorization": f"Bearer {token}", + "Accept": "application/vnd.github+json", + } + ) + return session + + +def paginate_github_api(session: requests.Session, url: str) -> List[Dict]: + """Paginate through GitHub API responses. + + Follows the Link header for pagination as documented in: + https://docs.github.com/en/rest/guides/using-pagination-in-the-rest-api + """ + results = [] + while url: + response = session.get(url) + response.raise_for_status() + data = response.json() + + # Handle both list and dict responses + if isinstance(data, list): + results.extend(data) + else: + results.append(data) + + # Check for pagination using Link header (RFC 8288) + # Format: ; rel="next", <...>; rel="last" + link_header = response.headers.get("Link", "") + url = None + if link_header: + # Match URLs within angle brackets that have rel="next" + # Pattern: ; rel="next" + match = re.search(r'<([^>]+)>;\s*rel="next"', link_header) + if match: + url = match.group(1) + + return results + + +def get_org_repos(org: str, token: str) -> List[str]: + """Get all repos in the org""" + print("Fetching org repositories...") + session = create_github_session(token) + url = f"{GITHUB_API_BASE}/orgs/{org}/repos?per_page=100" + repos_data = paginate_github_api(session, url) + repos = [repo["name"] for repo in repos_data if not repo["archived"]] + print(f"Found {len(repos)} repositories.") + return repos + + +def get_lsdb_repos(token: str) -> List[str]: + """Get all repos that are related to HATS or LSDB""" + print("Fetching org repositories...") + session = create_github_session(token) + url = f"{GITHUB_API_BASE}/orgs/astronomy-commons/repos?per_page=100" + repos_data = paginate_github_api(session, url) + repos = [repo["name"] for repo in repos_data] + repos = [repo for repo in repos if "hats" in repo or "lsdb" in repo] + print(f"Found {len(repos)} repositories.") + return repos + + +def get_humanized_updated_at(iso_time: str, now: datetime) -> str: + """Convenience method to get a human readable duration, like '2 days ago'.""" + try: + dt = datetime.strptime(iso_time, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc) + return human_readable.date_time(dt, when=now) + except Exception: # pylint: disable=broad-except + return iso_time diff --git a/src/lf_workflow_dash/lsdb_interrupts/open_prs.py b/src/lf_workflow_dash/lsdb_interrupts/open_prs.py new file mode 100644 index 00000000000..2f69e7f7240 --- /dev/null +++ b/src/lf_workflow_dash/lsdb_interrupts/open_prs.py @@ -0,0 +1,92 @@ +import sys +from datetime import datetime, timezone +from typing import Dict, List + +import requests +from jinja2 import Environment, FileSystemLoader + +from lf_workflow_dash.lsdb_interrupts.github_api import ( + GITHUB_API_BASE, + create_github_session, + get_humanized_updated_at, + get_lsdb_repos, + get_org_repos, + paginate_github_api, +) + + +def get_open_prs(org: str, repos: List[str], token: str) -> List[Dict]: + """Find all OPEN PRs.""" + print("Fetching open PRs for all repositories...") + session = create_github_session(token) + all_prs = [] + for repo in repos: + print(f" {repo}...") + try: + url = f"{GITHUB_API_BASE}/repos/{org}/{repo}/pulls?state=open&per_page=100" + pr_data = paginate_github_api(session, url) + + prs = [ + { + "number": pr["number"], + "title": pr["title"], + "author": pr["user"], + "updatedAt": pr["updated_at"], + "url": pr["html_url"], + "repo": repo, + "is_draft": pr["draft"], + } + for pr in pr_data + ] + print(f" Found {len(prs)} pull requests.") + + all_prs.extend(prs) + except requests.exceptions.RequestException as e: + print(f"Error fetching prs for repo {repo}: {e}") + print(f"Collected {len(all_prs)} open prs.") + return all_prs + + +def write_html_prs(prs: List[Dict], html_file: str): + """Fill in the jinja template, using the prs found""" + print(f"Writing HTML output to {html_file} ...") + + now = datetime.now(timezone.utc) + pr_summaries = [] + for pr in prs: + author = pr["author"]["login"] if pr["author"] else "unknown" + pr_summaries.append( + { + "updated_human": get_humanized_updated_at(pr["updatedAt"], now), + "author": author, + "title": pr["title"].replace("&", "&").replace("<", "<").replace(">", ">"), + "url": pr["url"], + "repo": pr["repo"], + "is_draft": "DRAFT" if pr["is_draft"] else "", + "is_bot": author in ["dependabot[bot]", "Copilot"], + } + ) + + environment = Environment(loader=FileSystemLoader("templates/")) + template = environment.get_template("pr_list.jinja") + with open(html_file, mode="w", encoding="utf-8") as results: + results.write(template.render({"all_prs": pr_summaries, "num_results": len(pr_summaries)})) + print(f"HTML output written to {html_file}") + + +def main(token): + """Convenience method to do the work.""" + repos = get_lsdb_repos(token) + prs = get_open_prs("astronomy-commons", repos, token) + prs.sort(key=lambda x: x["updatedAt"], reverse=True) + write_html_prs(prs, "html/lsdb_prs.html") + + repos = get_org_repos("lincc-frameworks", token) + prs = get_open_prs("lincc-frameworks", repos, token) + prs.sort(key=lambda x: x["updatedAt"], reverse=True) + write_html_prs(prs, "html/lincc_prs.html") + + +if __name__ == "__main__": + TOKEN = sys.argv[1] + main(TOKEN) diff --git a/templates/pr_list.jinja b/templates/pr_list.jinja new file mode 100644 index 00000000000..bd86e4f2ea3 --- /dev/null +++ b/templates/pr_list.jinja @@ -0,0 +1,43 @@ + + + + +Pull Requests + + + +

Pull Requests ({{num_results}}) (most recent activity first)

+ + + + + + + + + + + + +{% for pr in all_prs %} + + + {{pr.author}} + + + + +{% endfor %} + +
Last ActivityAuthorRepoDraft?Title
{{pr.updated_human}}{{pr.repo}}{{pr.is_draft}}{{pr.title}}
+ + \ No newline at end of file From 6776f28b9538dadde6bd8ed0e834e1577e2654cd Mon Sep 17 00:00:00 2001 From: Melissa DeLucchi Date: Mon, 1 Dec 2025 11:15:08 -0500 Subject: [PATCH 2/3] Run the thing. --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f8b49f2638d..5c3ed23caa3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -28,6 +28,7 @@ jobs: python update_dashboard.py ${{ secrets.GITHUB_TOKEN }} config/tracked_incubator.yaml html/incubator.html python update_dashboard.py ${{ secrets.GITHUB_TOKEN }} config/lsdb_workflows.yaml html/lsdb.html python src/lf_workflow_dash/lsdb_interrupts/external_issues.py ${{ secrets.GITHUB_TOKEN }} html/lsdb_issues.html + python src/lf_workflow_dash/lsdb_interrupts/open_prs.py ${{ secrets.GITHUB_TOKEN }} - name: Deploy to Github pages uses: JamesIves/github-pages-deploy-action@v4 with: From 8b79a997ad9d670b8449e4ee8c74a56dda93a053 Mon Sep 17 00:00:00 2001 From: Melissa DeLucchi Date: Mon, 1 Dec 2025 12:38:28 -0500 Subject: [PATCH 3/3] Ooops --- src/lf_workflow_dash/lsdb_interrupts/external_issues.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lf_workflow_dash/lsdb_interrupts/external_issues.py b/src/lf_workflow_dash/lsdb_interrupts/external_issues.py index 683c17fd691..9ff1b7554a3 100644 --- a/src/lf_workflow_dash/lsdb_interrupts/external_issues.py +++ b/src/lf_workflow_dash/lsdb_interrupts/external_issues.py @@ -105,7 +105,7 @@ def write_html_issues(external_issues: List[Dict], html_file: str): def main(token, out_file): """Convenience method to do the work.""" - repos = get_lsdb_repos("astronomy-commons", token) + repos = get_lsdb_repos(token) external_issues = get_open_issues("astronomy-commons", repos, TEAM_MEMBERS, token) # Sort by most recent activity external_issues.sort(key=lambda x: x["updatedAt"], reverse=True)