Skip to content

Commit

Permalink
Merge pull request #146 from inab/143-integrate-pip-audit-in-the-ci-w…
Browse files Browse the repository at this point in the history 
…orkflows

143 integrate pip audit in the ci workflows
  • Loading branch information
@jmfernandez
jmfernandez authored Jan 20, 2025
2 parents ddb670d + c2d153a commit 0df685c
Show file tree
Hide file tree
Showing 2 changed files with 246 additions and 46 deletions.
219 changes: 174 additions & 45 deletions .github/workflows/pip-audit.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,11 @@ jobs:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13" ]
python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12", "3.13" ]
name: pip-audit python ${{ matrix.python-version }}
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v4
- uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
cache: 'pip'
Expand All @@ -23,53 +23,182 @@ jobs:
architecture: x64
- name: 'Install requirements (standard or constraints ${{ matrix.python-version }})'
run: |
python -mvenv /tmp/PIPAUDIT
source /tmp/PIPAUDIT/bin/activate
pip install --upgrade pip wheel
pip install -r requirements.txt -c constraints-${{ matrix.python-version }}.txt
pip install pip-audit
# - name: 'Freeze Python ${{ matrix.python-version }} constraints'
# run: |
# pip freeze > constraints-${{ matrix.python-version }}.txt
- id: gen-cve-output
uses: pypa/gh-action-pip-audit@v1.1.0
- name: show_markdown
run: |
# echo "storing to file: ${{ steps.gen-cve-output.outputs.internal-be-careful-output }}"
# echo ${{ steps.gen-cve-output.outputs.internal-be-careful-output }} > $GITHUB_WORKSPACE/security_scans.md
# echo "saved."
type -a pip-audit
cat <<EOF
${{ steps.gen-cve-output.outputs.internal-be-careful-output }}
EOF
# - uses: actions/upload-artifact@v3
# with:
# retention-days: 2
# path: constraints-${{ matrix.python-version }}.txt
#
# pull_request_changes:
# # Do this only when it is not a pull request validation
# if: github.event_name != 'pull_request'
# runs-on: ubuntu-latest
# name: Pull request with the newly generated contents
# needs:
# - pre-commit
# steps:
# - uses: actions/checkout@v3
# - uses: actions/download-artifact@v3
# with:
# path: changes-dir
# - name: Move artifacts to their right place
# run: |
# cp -dpr changes-dir/artifact/* .
# rm -r changes-dir/artifact
# - name: Create Pull Request
# id: cpr
# uses: peter-evans/create-pull-request@v5
# with:
# title: Updated constraints (triggered by ${{ github.sha }})
# branch: create-pull-request/patch-constraints
# delete-branch: true
# commit-message: "[create-pull-request] Automatically commit updated contents (constraints)"
# - name: Check outputs
# if: ${{ steps.cpr.outputs.pull-request-number }}
source /tmp/PIPAUDIT/bin/activate
set +e
pip-audit --desc=on --progress-spinner=off -r constraints-${{ matrix.python-version }}.txt --no-deps --disable-pip -f markdown -o /tmp/report-before.md
refreeze=$?
set -e
if [ "$refreeze" != 0 ] ; then
deactivate
python -mvenv /tmp/PIPFREEZE
source /tmp/PIPFREEZE/bin/activate
pip install --upgrade pip wheel
pip install -r requirements.txt
pip freeze > constraints-${{ matrix.python-version }}.txt
# Re-audit the populated environment
deactivate
source /tmp/PIPAUDIT/bin/activate
set +e
pip-audit --desc=on --progress-spinner=off -r constraints-${{ matrix.python-version }}.txt --no-deps --disable-pip -f markdown -o /tmp/report-after.md
auditres=$?
set -e
if [ "$auditres" = 0 ] ; then
echo "# Fixed dependency issues for Python ${{ matrix.python-version }}" > audit-report-${{ matrix.python-version }}.md
cat /tmp/report-before.md >> audit-report-${{ matrix.python-version }}.md
else
# Time to emit the report
echo "# Dependency issues not solved for Python ${{ matrix.python-version }}" > audit-report-${{ matrix.python-version }}.md
cat /tmp/report-after.md >> audit-report-${{ matrix.python-version }}.md
fi
cat audit-report-${{ matrix.python-version }}.md >> "$GITHUB_STEP_SUMMARY"
fi
- uses: actions/upload-artifact@v4
with:
name: audit-${{ matrix.python-version }}
retention-days: 2
path: |
constraints-${{ matrix.python-version }}.txt
audit-report-${{ matrix.python-version }}.md
pip-audit-22_04:
runs-on: ubuntu-22.04
strategy:
matrix:
python-version: [ "3.7" ]
name: pip-audit python ${{ matrix.python-version }}
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
cache: 'pip'
cache-dependency-path: |
requirements.txt
architecture: x64
- name: 'Install requirements (standard or constraints ${{ matrix.python-version }})'
run: |
python -mvenv /tmp/PIPAUDIT
source /tmp/PIPAUDIT/bin/activate
pip install --upgrade pip wheel
pip install pip-audit
# - name: 'Freeze Python ${{ matrix.python-version }} constraints'
# run: |
# echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}"
# echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}"
# pip freeze > constraints-${{ matrix.python-version }}.txt
- id: gen-cve-output
run: |
source /tmp/PIPAUDIT/bin/activate
set +e
pip-audit --desc=on --progress-spinner=off -r constraints-${{ matrix.python-version }}.txt --no-deps --disable-pip -f markdown -o /tmp/report-before.md
refreeze=$?
set -e
if [ "$refreeze" != 0 ] ; then
deactivate
python -mvenv /tmp/PIPFREEZE
source /tmp/PIPFREEZE/bin/activate
pip install --upgrade pip wheel
pip install -r requirements.txt
pip freeze > constraints-${{ matrix.python-version }}.txt
# Re-audit the populated environment
deactivate
source /tmp/PIPAUDIT/bin/activate
set +e
pip-audit --desc=on --progress-spinner=off -r constraints-${{ matrix.python-version }}.txt --no-deps --disable-pip -f markdown -o /tmp/report-after.md
auditres=$?
set -e
if [ "$auditres" = 0 ] ; then
echo "# Fixed dependency issues for Python ${{ matrix.python-version }}" > audit-report-${{ matrix.python-version }}.md
cat /tmp/report-before.md >> audit-report-${{ matrix.python-version }}.md
else
# Time to emit the report
echo "# Dependency issues not solved for Python ${{ matrix.python-version }}" > audit-report-${{ matrix.python-version }}.md
cat /tmp/report-after.md >> audit-report-${{ matrix.python-version }}.md
fi
cat audit-report-${{ matrix.python-version }}.md >> "$GITHUB_STEP_SUMMARY"
fi
- uses: actions/upload-artifact@v4
with:
name: audit-${{ matrix.python-version }}
retention-days: 2
path: |
constraints-${{ matrix.python-version }}.txt
audit-report-${{ matrix.python-version }}.md
pull_request_changes:
# Do this only when it is not a pull request validation
if: github.event_name != 'pull_request'
runs-on: ubuntu-latest
name: Pull request with the newly generated contents
needs:
- pip-audit
- pip-audit-22_04
steps:
- name: Get analysis timestamp
id: timestamp
run: echo "timestamp=$(date -Is)" >> "$GITHUB_OUTPUT"
- uses: actions/checkout@v4
- uses: actions/download-artifact@v4
id: download
with:
pattern: audit-*
merge-multiple: true
path: changes-dir
- name: Move artifacts to their right place
id: move
run: |
skip=true
if [ -d "${{steps.download.outputs.download-path}}" ] ; then
for con in "${{steps.download.outputs.download-path}}"/constraints-*.txt ; do
case "$con" in
*/constraints-\*.txt)
break
;;
*)
cp -p "$con" .
skip=false
;;
esac
done
for aud in "${{steps.download.outputs.download-path}}"/audit-report*.md ; do
case "$aud" in
*/audit-report-\*.md)
break
;;
*)
cat "$aud" >> pull-body.md
;;
esac
done
fi
ls -l
echo "skip=$skip" >> "$GITHUB_OUTPUT"
- name: Create Pull Request
id: cpr
uses: peter-evans/create-pull-request@v7
if: steps.move.outputs.skip == 'false'
with:
title: Updated constraints due security reasons (triggered on ${{ steps.timestamp.outputs.timestamp }} by ${{ github.sha }})
branch: create-pull-request/patch-audit-constraints
add-paths: constraints-*.txt
delete-branch: true
commit-message: "[create-pull-request] Automatically updated constraints due security reasons"
body-path: pull-body.md
- name: Check outputs
if: ${{ steps.cpr.outputs.pull-request-number }}
run: |
echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}" >> "$GITHUB_STEP_SUMMARY"
echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}" >> "$GITHUB_STEP_SUMMARY"
73 changes: 72 additions & 1 deletion .github/workflows/pre-commit.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,77 @@ jobs:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13" ]
python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12", "3.13" ]
name: Pre-commit python ${{ matrix.python-version }}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 100
- uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
cache: 'pip'
cache-dependency-path: |
requirements.txt
mypy-requirements.txt
dev-requirements.txt
architecture: x64
- name: Changed requirements.txt
id: changed-requirements-txt
uses: tj-actions/changed-files@v44
with:
files: requirements.txt

- name: 'Install requirements (standard or constraints ${{ matrix.python-version }})'
run: |
pip install --upgrade pip wheel
if [ ${{ steps.changed-requirements-txt.outputs.any_changed }} != 'true' ] && [ -f constraints-${{ matrix.python-version }}.txt ] ; then
pip install -r requirements.txt -c constraints-${{ matrix.python-version }}.txt
else
pip install -r requirements.txt
fi
- name: 'Freeze Python ${{ matrix.python-version }} constraints'
run: |
pip freeze > constraints-${{ matrix.python-version }}.txt
- run: |
pip install -r dev-requirements.txt -r mypy-requirements.txt -c constraints-${{ matrix.python-version }}.txt
- name: MyPy cache
uses: actions/cache@v3
with:
path: .mypy_cache/${{ matrix.python-version }}
key: mypy-${{ matrix.python-version }}
- name: 'pre-commit'
uses: pre-commit/action@v3.0.1
# if: ${{ matrix.python-version != '3.6' }}
with:
extra_args: --all -c .pre-commit-config.yaml
# - name: 'pre-commit (custom Python ${{ matrix.python-version }})'
# uses: pre-commit/action@v3.0.0
# if: ${{ matrix.python-version == '3.6' }}
# with:
# extra_args: --all -c .pre-commit-config-gh-${{ matrix.python-version }}.yaml
- name: Check transitive dependencies licences
id: license_check_report
# uses: pilosus/action-pip-license-checker@v1.0.0
# continue-on-error: true
uses: pilosus/action-pip-license-checker@v2.0.0
with:
requirements: constraints-${{ matrix.python-version }}.txt
fail: 'StrongCopyleft'
exclude: 'pylint.*'
- name: Print licences report
if: ${{ always() }}
run: echo "${{ steps.license_check_report.outputs.report }}"
- uses: actions/upload-artifact@v3
with:
retention-days: 2
path: constraints-${{ matrix.python-version }}.txt

pre-commit-22_04:
runs-on: ubuntu-22.04
strategy:
matrix:
python-version: [ "3.7" ]
name: Pre-commit python ${{ matrix.python-version }}
steps:
- uses: actions/checkout@v4
Expand Down Expand Up @@ -85,6 +155,7 @@ jobs:
name: Pull request with the newly generated contents
needs:
- pre-commit
- pre-commit-22_04
steps:
- uses: actions/checkout@v3
- uses: actions/download-artifact@v3
Expand Down

0 comments on commit 0df685c

Please sign in to comment.