leaderboard.yml

name: Manage leaderboard

on:
  issue_comment:
    types: [created]
    branches:
      - main

jobs:
  add-entry:
    if: ${{ github.event.issue.pull_request && startsWith(github.event.comment.body, '/add-to-leaderboard') }}
    runs-on: ubuntu-latest
    steps:

      - name: Obtain PR branch
        id: get-branch
        run: echo "branch=$(gh pr view $PR_NO --repo $REPO --json headRefName --jq '.headRefName')" >> $GITHUB_OUTPUT
        env:
          REPO: ${{ github.repository }}
          PR_NO: ${{ github.event.issue.number }}
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Checkout PR branch
        uses: actions/checkout@v2
        with:
          ref: ${{ steps.get-branch.outputs.branch }}

      - name: Extract arguments from comment
        id: extract_args
        run: |
          echo "Extracting arguments..."
          comment="${{ github.event.comment.body }}"
          python_version=$(echo "$comment" | grep -oP '(?<=--python )\S+')
          appworld_version=$(echo "$comment" | grep -oP '(?<=--appworld )\S+')
          experiment_prefix=$(echo "$comment" | grep -oP '(?<=--experiment-prefix )\S+')
          replace_last_flag=$(echo "$comment" | grep -q -- '--replace-last' && echo "--replace-last" || echo "")

          echo "Python version: $python_version"
          echo "Appworld version: $appworld_version"
          echo "Experiment names: ${{ env.experiment_prefix }}_test_normal and ${{ env.experiment_prefix }}_test_challenge"
          echo "Replace last flag: $replace_last_flag"

          echo "python_version=$python_version" >> $GITHUB_ENV
          echo "appworld_version=$appworld_version" >> $GITHUB_ENV
          echo "experiment_prefix=$experiment_prefix" >> $GITHUB_ENV
          echo "replace_last_flag=$replace_last_flag" >> $GITHUB_ENV

      - uses: astral-sh/setup-uv@v3
        with:
          version: "0.4.4"

      - name: Set up Python
        run: uv python install ${{ env.python_version }}

      - name: Install venv
        run: uv venv

      - name: Install dependencies
        run: |
          uv pip install appworld==${{ env.appworld_version }}
          uv run appworld install

      - name: Download appworld data
        run: uv run appworld download data

      - name: Fetch main branch
        run: git fetch origin main

      # - name: Verify PR file changes
      #   run: |
      #       echo "Checking PR for exactly two new files..."
      #       experiment_prefix="${{ env.experiment_prefix }}"
      #       expected_files=("experiments/outputs/${experiment_prefix}_test_challenge/leaderboard.bundle" "experiments/outputs/${experiment_prefix}_test_normal/leaderboard.bundle")
      #       new_files=$(git diff --name-only origin/main..HEAD)

      #       echo "Expected files:"
      #       printf "%s\n" "${expected_files[@]}"
      #       echo "New files in the PR:"
      #       echo "$new_files"

      #       # Sort and compare file lists
      #       expected_sorted=$(printf "%s\n" "${expected_files[@]}")
      #       actual_sorted=$(echo "$new_files" | sort)

      #       if [[ "$expected_sorted" != "$actual_sorted" ]]; then
      #         echo "Error: File list does not match the expected files."
      #         echo "Expected:"
      #         echo "$expected_sorted"
      #         echo "Actual:"
      #         echo "$actual_sorted"
      #         exit 1
      #       fi

      #       echo "PR file check passed. The file list matches exactly."

      - name: Download relevant files
        run: |
          FILE_PATH=experiments/outputs/${experiment_prefix}_test_normal/leaderboard.bundle
          curl -L -o ${FILE_PATH} https://github.com/stonybrooknlp/appworld-leaderboard/raw/${{ steps.get-branch.outputs.branch }}/${FILE_PATH}
          du -sh ${FILE_PATH}
          FILE_PATH=experiments/outputs/${experiment_prefix}_test_challenge/leaderboard.bundle
          curl -L -o ${FILE_PATH} https://github.com/stonybrooknlp/appworld-leaderboard/raw/${{ steps.get-branch.outputs.branch }}/${FILE_PATH}
          du -sh ${FILE_PATH}

      - name: Unpack experiments
        run: |
          uv run appworld unpack ${{ env.experiment_prefix }}_test_normal
          uv run appworld unpack ${{ env.experiment_prefix }}_test_challenge
          ls experiments/outputs/temp_test_test_normal/
          ls experiments/outputs/temp_test_test_challenge/

      - name: Run evaluations
        run: |
          uv run appworld evaluate ${{ env.experiment_prefix }}_test_normal test_normal
          uv run appworld evaluate ${{ env.experiment_prefix }}_test_challenge test_challenge

      - name: Make and add leaderboard entry
        run: uv run appworld make ${{ env.experiment_prefix }}_test_normal ${{ env.experiment_prefix }}_test_challenge ${{env.replace_last_flag}} --save

      - name: Comment with leaderboard entry
        if: ${{ success() }}
        uses: actions/github-script@v6
        with:
          script: |
            const fs = require('fs');
            const entries = JSON.parse(fs.readFileSync('experiments/outputs/_leaderboard.json', 'utf8'));
            const formattedEntry = '```json\n' + JSON.stringify(entries[entries.length - 1], null, 4) + '\n```';
            const commentBody = `### Added leaderboard entry\n${formattedEntry}`;
            const issue_number = context.issue.number;
            await github.rest.issues.createComment({
              ...context.repo,
              issue_number: issue_number,
              body: commentBody,
            });

      - name: Add leaderboard entry
        run: |
            git config user.name "github-actions[bot]"
            git config user.email "github-actions[bot]@users.noreply.github.com"
            git add -f experiments/outputs/_leaderboard.json
            git commit -m "add leaderboard entry"
            git push origin ${{ steps.get-branch.outputs.branch }}
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  remove-entry:
    if: ${{ github.event.issue.pull_request && startsWith(github.event.comment.body, '/remove-from-leaderboard') }}
    runs-on: ubuntu-latest
    steps:

      - name: Obtain PR branch
        id: get-branch
        run: echo "branch=$(gh pr view $PR_NO --repo $REPO --json headRefName --jq '.headRefName')" >> $GITHUB_OUTPUT
        env:
          REPO: ${{ github.repository }}
          PR_NO: ${{ github.event.issue.number }}
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Checkout PR branch
        uses: actions/checkout@v2
        with:
          ref: ${{ steps.get-branch.outputs.branch }}

      - name: Extract arguments from comment
        id: extract_args
        run: |
          echo "Extracting arguments..."
          comment="${{ github.event.comment.body }}"
          entry_id=$(echo "$comment" | grep -oP '(?<=--id )\S+')
          echo "Entry ID: $entry_id"
          echo "entry_id=$entry_id" >> $GITHUB_ENV

      - name: Check if this entry ID was added in this PR
        run: |
          echo "Checking if Entry ID is in the diff..."
          git fetch origin main
          diff_content=$(git diff origin/main..HEAD experiments/outputs/_leaderboard.json)
          echo "$diff_content" > diff_content.txt
          cat diff_content.txt
          if echo "$diff_content" | grep -q "$entry_id"; then
            echo "Entry ID found in diff."
          else
            echo "Error: Entry ID not found in diff."
            exit 1
          fi

      - name: Remove leaderboard entry and comment
        if: env.entry_id
        uses: actions/github-script@v6
        with:
          script: |
            const fs = require('fs');
            const path = require('path');

            const entryId = process.env.entry_id;
            const repoOwner = context.repo.owner;
            const repoName = context.repo.repo;
            const prNumber = context.payload.issue.number;

            const leaderboardPath = path.join(process.cwd(), 'experiments/outputs/_leaderboard.json');

            // Load leaderboard.json
            if (!fs.existsSync(leaderboardPath)) {
              throw new Error('experiments/outputs/_leaderboard.json not found in the repository root.');
            }

            const leaderboard = JSON.parse(fs.readFileSync(leaderboardPath, 'utf8'));
            const updatedLeaderboard = leaderboard.filter(entry => entry.id !== entryId);
            const removedEntries = leaderboard.filter(entry => entry.id === entryId);

            if (removedEntries.length === 0) {
              throw new Error(`No entry found with ID \`${entryId}\`.`);
            }

            // Write the updated leaderboard back to the file
            fs.writeFileSync(leaderboardPath, JSON.stringify(updatedLeaderboard, null, 4));

            // Post a comment showing the removed entry
            const removedEntry = '```json\n' + JSON.stringify(removedEntries[0], null, 4); + '\n```';
            const commentBody = `### Removed leaderboard entry\n${removedEntry}`;
            await github.rest.issues.createComment({
              ...context.repo,
              issue_number: context.issue.number,
              body: commentBody
            });
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Reformat leaderboard in the original format
        run: |
          python - <<EOF
          import json

          with open("experiments/outputs/_leaderboard.json") as file:
            data = json.load(file)

          def normalize_numbers(obj):
            if isinstance(obj, int):
                return float(obj)
            elif isinstance(obj, dict):
                return {k: normalize_numbers(v) for k, v in obj.items()}
            elif isinstance(obj, list):
                return [normalize_numbers(v) for v in obj]
            return obj

          with open("experiments/outputs/_leaderboard.json", "w") as file:
            json.dump(normalize_numbers(data), file, indent=4)
          EOF

      - name: Commit and push
        run: |
            git config user.name "github-actions[bot]"
            git config user.email "github-actions[bot]@users.noreply.github.com"
            git add -f experiments/outputs/_leaderboard.json
            git commit -m "remove leaderboard entry"
            git push origin ${{ steps.get-branch.outputs.branch }}
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}