|
| 1 | +#!/usr/bin/env python3 |
| 2 | +""" |
| 3 | +Validates the .github/zombienet-flaky-tests file to ensure: |
| 4 | +1. Each entry has the correct format: <test-name>:<issue-number> |
| 5 | +2. The referenced GitHub issue exists |
| 6 | +3. The issue is open (optional warning) |
| 7 | +""" |
| 8 | + |
| 9 | +import sys |
| 10 | +import os |
| 11 | +import re |
| 12 | +import json |
| 13 | +import urllib.request |
| 14 | +import urllib.error |
| 15 | +from typing import List, Tuple, Optional |
| 16 | + |
| 17 | +GITHUB_API_BASE = "https://api.github.com" |
| 18 | + |
| 19 | +def parse_flaky_tests_file(file_path: str) -> List[Tuple[int, str, Optional[int]]]: |
| 20 | + """ |
| 21 | + Parse the zombienet-flaky-tests file. |
| 22 | + |
| 23 | + Returns: |
| 24 | + List of tuples: (line_number, test_name, issue_number) |
| 25 | + """ |
| 26 | + entries = [] |
| 27 | + |
| 28 | + with open(file_path, 'r') as f: |
| 29 | + for line_num, line in enumerate(f, start=1): |
| 30 | + line = line.strip() |
| 31 | + if not line: |
| 32 | + continue |
| 33 | + # Parse the format: test-name:issue-number |
| 34 | + match = re.match(r'^([^:]+):(\d+)$', line) |
| 35 | + if match: |
| 36 | + test_name = match.group(1) |
| 37 | + issue_number = int(match.group(2)) |
| 38 | + entries.append((line_num, test_name, issue_number)) |
| 39 | + else: |
| 40 | + # Invalid format |
| 41 | + entries.append((line_num, line, None)) |
| 42 | + |
| 43 | + return entries |
| 44 | + |
| 45 | +def check_issue_exists(issue_number: int, github_token: Optional[str] = None) -> Tuple[bool, Optional[str], Optional[str]]: |
| 46 | + """ |
| 47 | + Check if a GitHub issue exists and get its state. |
| 48 | + |
| 49 | + Returns: |
| 50 | + Tuple of (exists, state, title) |
| 51 | + - exists: True if issue exists |
| 52 | + - state: 'open' or 'closed' if exists, None otherwise |
| 53 | + - title: Issue title if exists, None otherwise |
| 54 | + """ |
| 55 | + repo_owner = os.environ.get('GITHUB_REPOSITORY_OWNER', 'paritytech') |
| 56 | + repo_name = os.environ.get('GITHUB_REPOSITORY', 'paritytech/polkadot-sdk').split('/')[-1] |
| 57 | + |
| 58 | + api_url = f"{GITHUB_API_BASE}/repos/{repo_owner}/{repo_name}/issues/{issue_number}" |
| 59 | + |
| 60 | + # Add GitHub token if available for higher rate limits |
| 61 | + headers = {'User-Agent': 'polkadot-sdk-cmd-bot'} |
| 62 | + if github_token: |
| 63 | + headers['Authorization'] = f'token {github_token}' |
| 64 | + |
| 65 | + req = urllib.request.Request(api_url, headers=headers) |
| 66 | + |
| 67 | + try: |
| 68 | + with urllib.request.urlopen(req) as response: |
| 69 | + if response.getcode() == 200: |
| 70 | + data = json.loads(response.read().decode('utf-8')) |
| 71 | + return True, data.get('state'), data.get('title') |
| 72 | + else: |
| 73 | + print(f"Warning: Unexpected status code {response.status} for issue #{issue_number}", file=sys.stderr) |
| 74 | + return False, None, None |
| 75 | + |
| 76 | + except urllib.error.HTTPError as e: |
| 77 | + if e.code == 404: |
| 78 | + return False, None, None |
| 79 | + else: |
| 80 | + print(f"Warning: HTTP error {e.code} for issue #{issue_number}", file=sys.stderr) |
| 81 | + return False, None, None |
| 82 | + |
| 83 | + except urllib.error.URLError as e: |
| 84 | + print(f"Warning: Failed to check issue #{issue_number}: {e.reason}", file=sys.stderr) |
| 85 | + return False, None, None |
| 86 | + |
| 87 | + except Exception as e: |
| 88 | + print(f"Warning: Unexpected error checking issue #{issue_number}: {e}", file=sys.stderr) |
| 89 | + return False, None, None |
| 90 | + |
| 91 | +def validate_flaky_tests(file_path: str, github_token: Optional[str] = None) -> bool: |
| 92 | + """ |
| 93 | + Validate the zombienet-flaky-tests file. |
| 94 | + |
| 95 | + Returns: |
| 96 | + True if validation passes, False otherwise |
| 97 | + """ |
| 98 | + if not os.path.exists(file_path): |
| 99 | + print(f"Error: File not found: {file_path}", file=sys.stderr) |
| 100 | + return False |
| 101 | + |
| 102 | + entries = parse_flaky_tests_file(file_path) |
| 103 | + |
| 104 | + if not entries: |
| 105 | + print("Warning: No entries found in zombienet-flaky-tests file") |
| 106 | + return True |
| 107 | + |
| 108 | + has_errors = False |
| 109 | + warnings = [] |
| 110 | + |
| 111 | + print(f"Validating {len(entries)} entries in {file_path}...") |
| 112 | + print() |
| 113 | + |
| 114 | + for line_num, test_name, issue_number in entries: |
| 115 | + if issue_number is None: |
| 116 | + print(f"❌ Line {line_num}: Missing required issue number", file=sys.stderr) |
| 117 | + print(f" Entry: '{test_name}'", file=sys.stderr) |
| 118 | + print(f" Expected format: <test-name>:<issue-number>", file=sys.stderr) |
| 119 | + print(f" Example: zombienet-polkadot-test-name:1234", file=sys.stderr) |
| 120 | + has_errors = True |
| 121 | + continue |
| 122 | + |
| 123 | + exists, state, title = check_issue_exists(issue_number, github_token) |
| 124 | + |
| 125 | + if not exists: |
| 126 | + print(f"❌ Line {line_num}: Issue #{issue_number} does not exist", file=sys.stderr) |
| 127 | + print(f" Test: {test_name}", file=sys.stderr) |
| 128 | + has_errors = True |
| 129 | + elif state == 'closed': |
| 130 | + warnings.append((line_num, test_name, issue_number, title)) |
| 131 | + else: |
| 132 | + print(f"✅ Line {line_num}: {test_name} -> Issue #{issue_number} (open)") |
| 133 | + |
| 134 | + if warnings: |
| 135 | + print() |
| 136 | + print("⚠️ Warnings (closed issues):") |
| 137 | + for line_num, test_name, issue_number, title in warnings: |
| 138 | + print(f" Line {line_num}: Issue #{issue_number} is closed: '{title}'") |
| 139 | + print(f" Test: {test_name}") |
| 140 | + print(f" Consider removing this entry if the issue is resolved.") |
| 141 | + |
| 142 | + print() |
| 143 | + if has_errors: |
| 144 | + print("❌ Validation failed with errors", file=sys.stderr) |
| 145 | + return False |
| 146 | + else: |
| 147 | + print("✅ All entries are valid") |
| 148 | + if warnings: |
| 149 | + print(f" ({len(warnings)} warning(s) about closed issues)") |
| 150 | + return True |
| 151 | + |
| 152 | +def main(): |
| 153 | + """Main entry point.""" |
| 154 | + if len(sys.argv) < 2: |
| 155 | + print("Usage: check-zombienet-flaky-tests.py <path-to-zombienet-flaky-tests>", file=sys.stderr) |
| 156 | + sys.exit(1) |
| 157 | + |
| 158 | + file_path = sys.argv[1] |
| 159 | + github_token = os.environ.get('GITHUB_TOKEN') |
| 160 | + |
| 161 | + if not github_token: |
| 162 | + print("Warning: GITHUB_TOKEN not set. API rate limits may apply.", file=sys.stderr) |
| 163 | + print() |
| 164 | + |
| 165 | + success = validate_flaky_tests(file_path, github_token) |
| 166 | + sys.exit(0 if success else 1) |
| 167 | + |
| 168 | +if __name__ == '__main__': |
| 169 | + main() |
0 commit comments