Test PR 2 #25
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Manage leaderboard | |
| on: | |
| issue_comment: | |
| types: [created] | |
| branches: | |
| - main | |
| jobs: | |
| add-entry: | |
| if: ${{ github.event.issue.pull_request && startsWith(github.event.comment.body, '/add-to-leaderboard') }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Obtain PR branch | |
| id: get-branch | |
| run: echo "branch=$(gh pr view $PR_NO --repo $REPO --json headRefName --jq '.headRefName')" >> $GITHUB_OUTPUT | |
| env: | |
| REPO: ${{ github.repository }} | |
| PR_NO: ${{ github.event.issue.number }} | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Checkout PR branch | |
| uses: actions/checkout@v2 | |
| with: | |
| ref: ${{ steps.get-branch.outputs.branch }} | |
| - name: Extract arguments from comment | |
| id: extract_args | |
| run: | | |
| echo "Extracting arguments..." | |
| comment="${{ github.event.comment.body }}" | |
| python_version=$(echo "$comment" | grep -oP '(?<=--python )\S+') | |
| appworld_version=$(echo "$comment" | grep -oP '(?<=--appworld )\S+') | |
| experiment_prefix=$(echo "$comment" | grep -oP '(?<=--experiment-prefix )\S+') | |
| replace_last_flag=$(echo "$comment" | grep -q -- '--replace-last' && echo "--replace-last" || echo "") | |
| echo "Python version: $python_version" | |
| echo "Appworld version: $appworld_version" | |
| echo "Experiment names: ${{ env.experiment_prefix }}_test_normal and ${{ env.experiment_prefix }}_test_challenge" | |
| echo "Replace last flag: $replace_last_flag" | |
| echo "python_version=$python_version" >> $GITHUB_ENV | |
| echo "appworld_version=$appworld_version" >> $GITHUB_ENV | |
| echo "experiment_prefix=$experiment_prefix" >> $GITHUB_ENV | |
| echo "replace_last_flag=$replace_last_flag" >> $GITHUB_ENV | |
| - uses: astral-sh/setup-uv@v3 | |
| with: | |
| version: "0.4.4" | |
| - name: Set up Python | |
| run: uv python install ${{ env.python_version }} | |
| - name: Install venv | |
| run: uv venv | |
| - name: Install dependencies | |
| run: | | |
| uv pip install appworld==${{ env.appworld_version }} | |
| uv run appworld install | |
| - name: Download appworld data | |
| run: uv run appworld download data | |
| - name: Fetch main branch | |
| run: git fetch origin main | |
| # - name: Verify PR file changes | |
| # run: | | |
| # echo "Checking PR for exactly two new files..." | |
| # experiment_prefix="${{ env.experiment_prefix }}" | |
| # expected_files=("experiments/outputs/${experiment_prefix}_test_challenge/leaderboard.bundle" "experiments/outputs/${experiment_prefix}_test_normal/leaderboard.bundle") | |
| # new_files=$(git diff --name-only origin/main..HEAD) | |
| # echo "Expected files:" | |
| # printf "%s\n" "${expected_files[@]}" | |
| # echo "New files in the PR:" | |
| # echo "$new_files" | |
| # # Sort and compare file lists | |
| # expected_sorted=$(printf "%s\n" "${expected_files[@]}") | |
| # actual_sorted=$(echo "$new_files" | sort) | |
| # if [[ "$expected_sorted" != "$actual_sorted" ]]; then | |
| # echo "Error: File list does not match the expected files." | |
| # echo "Expected:" | |
| # echo "$expected_sorted" | |
| # echo "Actual:" | |
| # echo "$actual_sorted" | |
| # exit 1 | |
| # fi | |
| # echo "PR file check passed. The file list matches exactly." | |
| - name: Download relevant files | |
| run: | | |
| FILE_PATH=experiments/outputs/${experiment_prefix}_test_normal/leaderboard.bundle | |
| curl -L -o ${FILE_PATH} https://github.com/stonybrooknlp/appworld-leaderboard/raw/${{ steps.get-branch.outputs.branch }}/${FILE_PATH} | |
| du -sh ${FILE_PATH} | |
| FILE_PATH=experiments/outputs/${experiment_prefix}_test_challenge/leaderboard.bundle | |
| curl -L -o ${FILE_PATH} https://github.com/stonybrooknlp/appworld-leaderboard/raw/${{ steps.get-branch.outputs.branch }}/${FILE_PATH} | |
| du -sh ${FILE_PATH} | |
| - name: Unpack experiments | |
| run: | | |
| uv run appworld unpack ${{ env.experiment_prefix }}_test_normal | |
| uv run appworld unpack ${{ env.experiment_prefix }}_test_challenge | |
| ls experiments/outputs/temp_test_test_normal/ | |
| ls experiments/outputs/temp_test_test_challenge/ | |
| - name: Run evaluations | |
| run: | | |
| uv run appworld evaluate ${{ env.experiment_prefix }}_test_normal test_normal | |
| uv run appworld evaluate ${{ env.experiment_prefix }}_test_challenge test_challenge | |
| - name: Make and add leaderboard entry | |
| run: uv run appworld make ${{ env.experiment_prefix }}_test_normal ${{ env.experiment_prefix }}_test_challenge ${{env.replace_last_flag}} --save | |
| - name: Comment with leaderboard entry | |
| if: ${{ success() }} | |
| uses: actions/github-script@v6 | |
| with: | |
| script: | | |
| const fs = require('fs'); | |
| const entries = JSON.parse(fs.readFileSync('experiments/outputs/_leaderboard.json', 'utf8')); | |
| const formattedEntry = '```json\n' + JSON.stringify(entries[entries.length - 1], null, 4) + '\n```'; | |
| const commentBody = `### Added leaderboard entry\n${formattedEntry}`; | |
| const issue_number = context.issue.number; | |
| await github.rest.issues.createComment({ | |
| ...context.repo, | |
| issue_number: issue_number, | |
| body: commentBody, | |
| }); | |
| - name: Add leaderboard entry | |
| run: | | |
| git config user.name "github-actions[bot]" | |
| git config user.email "github-actions[bot]@users.noreply.github.com" | |
| git add -f experiments/outputs/_leaderboard.json | |
| git commit -m "add leaderboard entry" | |
| git push origin ${{ steps.get-branch.outputs.branch }} | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| remove-entry: | |
| if: ${{ github.event.issue.pull_request && startsWith(github.event.comment.body, '/remove-from-leaderboard') }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Obtain PR branch | |
| id: get-branch | |
| run: echo "branch=$(gh pr view $PR_NO --repo $REPO --json headRefName --jq '.headRefName')" >> $GITHUB_OUTPUT | |
| env: | |
| REPO: ${{ github.repository }} | |
| PR_NO: ${{ github.event.issue.number }} | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Checkout PR branch | |
| uses: actions/checkout@v2 | |
| with: | |
| ref: ${{ steps.get-branch.outputs.branch }} | |
| - name: Extract arguments from comment | |
| id: extract_args | |
| run: | | |
| echo "Extracting arguments..." | |
| comment="${{ github.event.comment.body }}" | |
| entry_id=$(echo "$comment" | grep -oP '(?<=--id )\S+') | |
| echo "Entry ID: $entry_id" | |
| echo "entry_id=$entry_id" >> $GITHUB_ENV | |
| - name: Check if this entry ID was added in this PR | |
| run: | | |
| echo "Checking if Entry ID is in the diff..." | |
| git fetch origin main | |
| diff_content=$(git diff origin/main..HEAD experiments/outputs/_leaderboard.json) | |
| echo "$diff_content" > diff_content.txt | |
| cat diff_content.txt | |
| if echo "$diff_content" | grep -q "$entry_id"; then | |
| echo "Entry ID found in diff." | |
| else | |
| echo "Error: Entry ID not found in diff." | |
| exit 1 | |
| fi | |
| - name: Remove leaderboard entry and comment | |
| if: env.entry_id | |
| uses: actions/github-script@v6 | |
| with: | |
| script: | | |
| const fs = require('fs'); | |
| const path = require('path'); | |
| const entryId = process.env.entry_id; | |
| const repoOwner = context.repo.owner; | |
| const repoName = context.repo.repo; | |
| const prNumber = context.payload.issue.number; | |
| const leaderboardPath = path.join(process.cwd(), 'experiments/outputs/_leaderboard.json'); | |
| // Load leaderboard.json | |
| if (!fs.existsSync(leaderboardPath)) { | |
| throw new Error('experiments/outputs/_leaderboard.json not found in the repository root.'); | |
| } | |
| const leaderboard = JSON.parse(fs.readFileSync(leaderboardPath, 'utf8')); | |
| const updatedLeaderboard = leaderboard.filter(entry => entry.id !== entryId); | |
| const removedEntries = leaderboard.filter(entry => entry.id === entryId); | |
| if (removedEntries.length === 0) { | |
| throw new Error(`No entry found with ID \`${entryId}\`.`); | |
| } | |
| // Write the updated leaderboard back to the file | |
| fs.writeFileSync(leaderboardPath, JSON.stringify(updatedLeaderboard, null, 4)); | |
| // Post a comment showing the removed entry | |
| const removedEntry = '```json\n' + JSON.stringify(removedEntries[0], null, 4); + '\n```'; | |
| const commentBody = `### Removed leaderboard entry\n${removedEntry}`; | |
| await github.rest.issues.createComment({ | |
| ...context.repo, | |
| issue_number: context.issue.number, | |
| body: commentBody | |
| }); | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Reformat leaderboard in the original format | |
| run: | | |
| python - <<EOF | |
| import json | |
| with open("experiments/outputs/_leaderboard.json") as file: | |
| data = json.load(file) | |
| def normalize_numbers(obj): | |
| if isinstance(obj, int): | |
| return float(obj) | |
| elif isinstance(obj, dict): | |
| return {k: normalize_numbers(v) for k, v in obj.items()} | |
| elif isinstance(obj, list): | |
| return [normalize_numbers(v) for v in obj] | |
| return obj | |
| with open("experiments/outputs/_leaderboard.json", "w") as file: | |
| json.dump(normalize_numbers(data), file, indent=4) | |
| EOF | |
| - name: Commit and push | |
| run: | | |
| git config user.name "github-actions[bot]" | |
| git config user.email "github-actions[bot]@users.noreply.github.com" | |
| git add -f experiments/outputs/_leaderboard.json | |
| git commit -m "remove leaderboard entry" | |
| git push origin ${{ steps.get-branch.outputs.branch }} | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} |