diff --git a/.github/workflows/pr-verify.yml b/.github/workflows/pr-verify.yml new file mode 100644 index 0000000..b20785d --- /dev/null +++ b/.github/workflows/pr-verify.yml @@ -0,0 +1,81 @@ +name: PR-Verify + +on: + push: + branches: + - main + issues: + types: [labeled] + issue_comment: + types: [created] + pull_request: + types: [labeled] + +permissions: + contents: write + issues: write + pull-requests: write + +jobs: + pr-verify: + runs-on: ubuntu-latest + if: github.event.label.name == 'pr-verify' + steps: + - name: Install jq + run: sudo apt-get install -y jq + - name: Check if label was added by a collaborator + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + is_collaborator=$(curl -s -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github+json" \ + "https://api.github.com/repos/${{ github.repository }}/collaborators/${{ github.event.sender.login }}" | jq -r '.message') + + if [ "$is_collaborator" == "Not Found" ]; then + echo "Label not added by a collaborator. Skipping action." + exit 78 + fi + - name: Checkout code + uses: actions/checkout@v2 + with: + ref: ${{ github.event.pull_request.head.sha }} # ${{ github.event.pull_request.head.ref }} + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.11 + + # TODO: use @main instead of recloning + - name: Checkout external repository + uses: actions/checkout@v2 + with: + repository: "ellenjxu/pr-verify" + path: "pr-verify" + - name: Install Python dependencies + run: | + python -m pip install --upgrade pip + pip install -r ${{ github.workspace }}/pr-verify/requirements.txt + - name: Run Python script from public repository + run: python ${{ github.workspace }}/pr-verify/main.py + + # - name: Install dependencies + # run: | + # pwd + # ls -la + # python -m pip install --upgrade pip + # pip install -r requirements.txt + + # - name: PR-Verify + # uses: ellenjxu/pr-verify@main + # with: + # github_token: ${{ secrets.GITHUB_TOKEN }} + # pr_url: ${{ github.event.pull_request.html_url }} + # openai_api_key: ${{ secrets.OPENAI_API_KEY }} + # env: + # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # PR_URL: ${{ github.event.pull_request.html_url }} + # OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + + # - name: Checkout unit test saved directory + # run: | + # cd pr-verify + # pytest unit_tests diff --git a/README.md b/README.md index 9e7f1b2..d4e6401 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,24 @@ -# LLM PR verifier +# PR-Verify PR-Verify is an automated LLM agent for reviewing and verifying Github PRs. We verify that contributor code works as intended, by generating unit tests and executing the code with LLM-generated "oracle verifiers" (https://arxiv.org/html/2305.14591v3). + +## setup + +1. In repository settings + +- Add OPENAI_AI_KEY to repo secrets +- Enable GH actions push access + +2. Copy over GH action: `.github/workflows/pr-verify.yml` + +That's it! Just add the `pr-verify` label and let PR-Verify do the rest. + +--- + +## todo + +- [ ] GH actions workflow, spin up docker container (code execution sandbox) +- [ ] automated repo installation setup (infer instructions from readme) +- [ ] synopsis/threat analysis comment on PR