Fetch Trending Repositories #7
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Fetch Trending Repositories | |
| on: | |
| schedule: | |
| # Run once daily at 2 AM UTC (low traffic time) | |
| - cron: '0 2 * * *' | |
| workflow_dispatch: | |
| inputs: | |
| force_refresh: | |
| description: 'Force refresh all platforms (ignore cache)' | |
| required: false | |
| default: 'false' | |
| type: boolean | |
| platforms: | |
| description: 'Specific platforms to update (comma-separated: android,windows,macos,linux)' | |
| required: false | |
| default: 'all' | |
| type: string | |
| jobs: | |
| check-rate-limit: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| can_proceed: ${{ steps.check.outputs.can_proceed }} | |
| remaining: ${{ steps.check.outputs.remaining }} | |
| steps: | |
| - name: Check GitHub API Rate Limit | |
| id: check | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| RATE_LIMIT=$(curl -s -H "Authorization: token $GITHUB_TOKEN" \ | |
| https://api.github.com/rate_limit) | |
| REMAINING=$(echo $RATE_LIMIT | jq -r '.resources.core.remaining') | |
| LIMIT=$(echo $RATE_LIMIT | jq -r '.resources.core.limit') | |
| echo "remaining=$REMAINING" >> $GITHUB_OUTPUT | |
| echo "limit=$LIMIT" >> $GITHUB_OUTPUT | |
| # Need at least 500 requests for a full run (safety margin) | |
| if [ "$REMAINING" -gt 500 ]; then | |
| echo "can_proceed=true" >> $GITHUB_OUTPUT | |
| echo "✓ Rate limit OK: $REMAINING/$LIMIT remaining" | |
| else | |
| echo "can_proceed=false" >> $GITHUB_OUTPUT | |
| echo "✗ Rate limit low: $REMAINING/$LIMIT remaining" | |
| echo "::warning::Skipping run due to low rate limit ($REMAINING remaining)" | |
| fi | |
| fetch-and-update: | |
| needs: check-rate-limit | |
| if: needs.check-rate-limit.outputs.can_proceed == 'true' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 45 | |
| permissions: | |
| contents: write | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: main | |
| fetch-depth: 0 # Full history for better change detection | |
| - name: Initialize cached-data directory | |
| run: | | |
| mkdir -p cached-data/trending | |
| # Add .gitkeep if directory is empty to ensure it's tracked | |
| if [ ! "$(ls -A cached-data/trending)" ]; then | |
| touch cached-data/trending/.gitkeep | |
| fi | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| cache: 'pip' # Cache pip dependencies | |
| - name: Install dependencies | |
| run: | | |
| pip install --upgrade pip | |
| pip install -r scripts/requirements.txt | |
| - name: Cache trending data | |
| uses: actions/cache@v4 | |
| with: | |
| path: cached-data/trending | |
| key: trending-data-${{ hashFiles('cached-data/trending/*.json') }} | |
| restore-keys: | | |
| trending-data- | |
| - name: Fetch trending repositories | |
| id: fetch | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| FORCE_REFRESH: ${{ inputs.force_refresh || 'false' }} | |
| PLATFORMS: ${{ inputs.platforms || 'all' }} | |
| run: | | |
| echo "Starting trending repository fetch..." | |
| echo "Force refresh: $FORCE_REFRESH" | |
| echo "Platforms: $PLATFORMS" | |
| # Ensure directory exists | |
| mkdir -p cached-data/trending | |
| # Run the optimized script | |
| python scripts/fetch_trending.py | |
| # Debug: Show what files exist | |
| echo "Files in cached-data/trending:" | |
| ls -lah cached-data/trending/ || echo "Directory doesn't exist" | |
| # Check if any files were modified or created | |
| if git status --porcelain cached-data/ | grep -q .; then | |
| echo "changed=true" >> $GITHUB_OUTPUT | |
| echo "Changes detected in cached data" | |
| git status --porcelain cached-data/ | |
| else | |
| echo "changed=false" >> $GITHUB_OUTPUT | |
| echo "No changes detected" | |
| fi | |
| - name: Validate JSON outputs | |
| if: steps.fetch.outputs.changed == 'true' | |
| run: | | |
| echo "Validating JSON files..." | |
| for file in cached-data/trending/*.json; do | |
| if ! jq empty "$file" 2>/dev/null; then | |
| echo "::error::Invalid JSON in $file" | |
| exit 1 | |
| fi | |
| echo "✓ Valid: $file" | |
| done | |
| - name: Generate summary | |
| if: steps.fetch.outputs.changed == 'true' | |
| run: | | |
| echo "## Trending Repositories Update Summary" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "**Updated at:** $(date -u +'%Y-%m-%d %H:%M:%S UTC')" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "| Platform | Repositories | Status |" >> $GITHUB_STEP_SUMMARY | |
| echo "|----------|--------------|--------|" >> $GITHUB_STEP_SUMMARY | |
| for platform in android windows macos linux; do | |
| file="cached-data/trending/$platform.json" | |
| if [ -f "$file" ]; then | |
| count=$(jq '.totalCount' "$file") | |
| echo "| $platform | $count | ✓ Updated |" >> $GITHUB_STEP_SUMMARY | |
| else | |
| echo "| $platform | 0 | ✗ Missing |" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| done | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "**API Rate Limit Remaining:** ${{ needs.check-rate-limit.outputs.remaining }}" >> $GITHUB_STEP_SUMMARY | |
| - name: Commit and push changes | |
| if: steps.fetch.outputs.changed == 'true' | |
| run: | | |
| git config user.name "github-actions[bot]" | |
| git config user.email "github-actions[bot]@users.noreply.github.com" | |
| # Stage changes | |
| git add cached-data/trending/*.json | |
| # Create detailed commit message | |
| TIMESTAMP=$(date -u +'%Y-%m-%d %H:%M:%S UTC') | |
| COMMIT_MSG="Update trending repositories - $TIMESTAMP | |
| Updated platforms: | |
| $(cd cached-data/trending && ls -1 *.json | sed 's/.json//' | sed 's/^/- /')" | |
| git commit -m "$COMMIT_MSG" | |
| # Push with retry logic | |
| for i in {1..3}; do | |
| if git push origin main; then | |
| echo "✓ Successfully pushed changes" | |
| break | |
| else | |
| echo "Push attempt $i failed, retrying..." | |
| sleep 5 | |
| git pull --rebase origin main | |
| fi | |
| done | |
| - name: No changes detected | |
| if: steps.fetch.outputs.changed == 'false' | |
| run: | | |
| echo "No changes in trending repositories - cache is still valid" | |
| echo "## No Updates Required" >> $GITHUB_STEP_SUMMARY | |
| echo "Cached data is still fresh (< 23 hours old)" >> $GITHUB_STEP_SUMMARY | |
| notify-on-failure: | |
| needs: [check-rate-limit, fetch-and-update] | |
| if: failure() | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Create failure issue | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const title = `Trending Repositories Fetch Failed - ${new Date().toISOString().split('T')[0]}`; | |
| const body = `The scheduled trending repositories fetch job failed. | |
| **Workflow run:** ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| **Rate limit remaining:** ${{ needs.check-rate-limit.outputs.remaining || 'N/A' }} | |
| Please investigate and re-run manually if needed.`; | |
| // Check if issue already exists | |
| const issues = await github.rest.issues.listForRepo({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| state: 'open', | |
| labels: 'automation,trending-fetch' | |
| }); | |
| const existingIssue = issues.data.find(issue => issue.title === title); | |
| if (!existingIssue) { | |
| await github.rest.issues.create({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| title: title, | |
| body: body, | |
| labels: ['automation', 'trending-fetch', 'bug'] | |
| }); | |
| } |