Update workflows

City-Bureau · May 8, 2024 · fd6b2bd · fd6b2bd
1 parent c5ba225
commit fd6b2bd
Show file tree

Hide file tree

Showing 3 changed files with 42 additions and 32 deletions.
diff --git a/.github/workflows/archive.yml b/.github/workflows/archive.yml
@@ -7,6 +7,7 @@ on:
 
 env:
   CI: true
+  PYTHON_VERSION: 3.9
   PIPENV_VENV_IN_PROJECT: true
   SCRAPY_SETTINGS_MODULE: city_scrapers.settings.archive
   AUTOTHROTTLE_MAX_DELAY: 30.0
@@ -17,32 +18,31 @@ jobs:
   crawl:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
-      - name: Set up Python 3.9
-        uses: actions/setup-python@v1
+      - name: Set up Python ${{ env.PYTHON_VERSION }}
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.9
+          python-version: ${{ env.PYTHON_VERSION }}
 
       - name: Install Pipenv
-        uses: dschep/install-pipenv-action@v1
+        run: pip install --user pipenv
 
       - name: Cache Python dependencies
-        uses: actions/cache@v1
+        uses: actions/cache@v4
         with:
           path: .venv
-          key: pip-3.9-${{ hashFiles('**/Pipfile.lock') }}
+          key: ${{ env.PYTHON_VERSION }}-${{ hashFiles('**/Pipfile.lock') }}
           restore-keys: |
-            pip-3.9-
+            ${{ env.PYTHON_VERSION }}-
             pip-
 
       - name: Install dependencies
         run: pipenv sync
         env:
-          PIPENV_DEFAULT_PYTHON_VERSION: 3.9
+          PIPENV_DEFAULT_PYTHON_VERSION: ${{ env.PYTHON_VERSION }}
 
       - name: Run scrapers
         run: |
           export PYTHONPATH=$(pwd):$PYTHONPATH
-          chmod u+r+x ./.deploy.sh
           ./.deploy.sh
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -18,18 +18,20 @@ jobs:
         python-version: [3.9]
 
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
 
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 
       - name: Install Pipenv
-        uses: dschep/install-pipenv-action@v1
+        run: pip install --user pipenv
 
       - name: Cache Python dependencies
-        uses: actions/cache@v1
+        uses: actions/cache@v4
         with:
           path: .venv
           key: pip-${{ matrix.python-version }}-${{ hashFiles('**/Pipfile.lock') }}
@@ -42,10 +44,6 @@ jobs:
         env:
           PIPENV_DEFAULT_PYTHON_VERSION: ${{ matrix.python-version }}
 
-      - name: Set up playwright
-        run: |
-          pipenv run playwright install firefox
-  
       - name: Check imports with isort
         run: |
           pipenv run isort . --check-only
@@ -59,8 +57,9 @@ jobs:
           pipenv run flake8 .
 
       - name: Test with pytest
+        # Ignores exit code 5 (no tests collected)
         run: |
-          pipenv run pytest
+          pipenv run pytest || [ $? -eq 5 ]
 
       - name: Validate output with scrapy
         if: github.event_name == 'pull_request'

diff --git a/.github/workflows/cron.yml b/.github/workflows/cron.yml
@@ -2,13 +2,15 @@ name: Cron
 
 on:
   schedule:
-    - cron: "12 8 * * *"
+    - cron: "1 6 * * *"
   workflow_dispatch:
 
 env:
   CI: true
+  PYTHON_VERSION: 3.9
   PIPENV_VENV_IN_PROJECT: true
   SCRAPY_SETTINGS_MODULE: city_scrapers.settings.prod
+  WAYBACK_ENABLED: true
   AUTOTHROTTLE_MAX_DELAY: 30.0
   AUTOTHROTTLE_START_DELAY: 1.5
   AUTOTHROTTLE_TARGET_CONCURRENCY: 3.0
@@ -18,37 +20,46 @@ env:
   AZURE_STATUS_CONTAINER: ${{ secrets.AZURE_STATUS_CONTAINER }}
   SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
 
-
 jobs:
   crawl:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v4
 
-      - name: Set up Python 3.9
-        uses: actions/setup-python@v1
+      - name: Set up Python ${{ env.PYTHON_VERSION }}
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.9
+          python-version: ${{ env.PYTHON_VERSION }}
 
       - name: Install Pipenv
-        uses: dschep/install-pipenv-action@v1
+        run: pip install --user pipenv
+
+      - name: Cache Python dependencies
+        uses: actions/cache@v4
+        with:
+          path: .venv
+          key: ${{ env.PYTHON_VERSION }}-${{ hashFiles('**/Pipfile.lock') }}
+          restore-keys: |
+            ${{ env.PYTHON_VERSION }}-
+            pip-
 
       - name: Install dependencies
         run: pipenv sync
         env:
-          PIPENV_DEFAULT_PYTHON_VERSION: 3.9
-
-      - name: Set up playwright
-        run: |
-          pipenv run playwright install firefox
+          PIPENV_DEFAULT_PYTHON_VERSION: ${{ env.PYTHON_VERSION }}
 
       - name: Run scrapers
         run: |
           export PYTHONPATH=$(pwd):$PYTHONPATH
-          chmod u+r+x ./.deploy.sh
           ./.deploy.sh
 
       - name: Combine output feeds
         run: |
           export PYTHONPATH=$(pwd):$PYTHONPATH
           pipenv run scrapy combinefeeds -s LOG_ENABLED=False
+
+      - name: Prevent workflow deactivation
+        uses: gautamkrishnar/keepalive-workflow@v1
+        with:
+          committer_username: "citybureau-bot"
+          committer_email: "documenters@citybureau.org"