From e398cac775118ec6a724ea84881a6d98e054edb7 Mon Sep 17 00:00:00 2001
From: sarayourfriend <24264157+sarayourfriend@users.noreply.github.com>
Date: Fri, 3 Nov 2023 13:23:00 +1100
Subject: [PATCH 1/3] Update self-hosted renovate configuration

---
 .github/dependabot.yml         | 142 ---------------------------------
 .github/renovate.json          |  72 ++++++++++++++++-
 .github/workflows/renovate.yml |  85 +++++++++++++++++++-
 .pre-commit-config.yaml        |   7 ++
 4 files changed, 156 insertions(+), 150 deletions(-)
 delete mode 100644 .github/dependabot.yml

diff --git a/.github/dependabot.yml b/.github/dependabot.yml
deleted file mode 100644
index 1346acd8af1..00000000000
--- a/.github/dependabot.yml
+++ /dev/null
@@ -1,142 +0,0 @@
-#################################
-# Dependabot Configuration File #
-#################################
-
-# current Github-native version of Dependabot
-version: 2
-
-updates:
-  # Enable version updates for Python libs in Openverse API
-  - package-ecosystem: pip
-    # Look for a `Pipfile` in the `/api` directory
-    directory: /api
-    # Check for updates once a month
-    schedule:
-      interval: monthly
-    labels:
-      - "💻 aspect: code"
-      - "🧰 goal: internal improvement"
-      - "🟩 priority: low"
-      - "🐍 tech: python"
-      - "🧱 stack: api"
-      - dependencies
-
-  # Enable version updates for Python libs in ingestion server
-  - package-ecosystem: pip
-    # Look for a `Pipfile` in the `/ingestion_server` directory
-    directory: /ingestion_server
-    # Check for updates once a month
-    schedule:
-      interval: monthly
-    labels:
-      - "💻 aspect: code"
-      - "🧰 goal: internal improvement"
-      - "🟩 priority: low"
-      - "🐍 tech: python"
-      - "🧱 stack: ingestion server"
-      - dependencies
-
-  # Enable version updates for Python dev-libs in the catalog
-  #
-  # `requirements-dev.txt` can be updated more frequently than
-  # `requirements_prod.txt` since they are not pinned by the Airflow constraints
-  # file.
-  - package-ecosystem: pip
-    # Look for requirements file in the `/ingestion_server` directory
-    directory: /catalog
-    # Check for updates once a month
-    schedule:
-      interval: monthly
-    labels:
-      - "💻 aspect: code"
-      - "🧰 goal: internal improvement"
-      - "🟩 priority: low"
-      - "🐍 tech: python"
-      - "🧱 stack: catalog"
-      - dependencies
-
-  # Enable version updates for Python libs in documentation
-  - package-ecosystem: pip
-    # Look for a `Pipfile` in the `/documentation` directory
-    directory: /documentation
-    # Check for updates once a month
-    schedule:
-      interval: monthly
-    labels:
-      - "💻 aspect: code"
-      - "🧰 goal: internal improvement"
-      - "🟩 priority: low"
-      - "🐍 tech: python"
-      - "🧱 stack: documentation"
-      - dependencies
-
-  # Enable version updates for Python libs in automations/python
-  - package-ecosystem: pip
-    # Look for a `Pipfile` in the `/automations/python` directory
-    directory: /automations/python
-    # Check for updates once a month
-    schedule:
-      interval: monthly
-    labels:
-      - "💻 aspect: code"
-      - "🧰 goal: internal improvement"
-      - "🟩 priority: low"
-      - "🐍 tech: python"
-      - "🧱 stack: mgmt"
-      - dependencies
-
-  # Enable version updates in github-actions
-  - package-ecosystem: "github-actions"
-    directory: /
-    schedule:
-      interval: "monthly"
-    labels:
-      - "💻 aspect: code"
-      - "🧰 goal: internal improvement"
-      - "🟩 priority: low"
-      - "🧱 stack: mgmt"
-      - dependencies
-
-  - package-ecosystem: "github-actions"
-    directory: /actions/build-docs
-    schedule:
-      interval: "monthly"
-    labels:
-      - "💻 aspect: code"
-      - "🧰 goal: internal improvement"
-      - "🟩 priority: low"
-      - "🧱 stack: mgmt"
-      - dependencies
-
-  - package-ecosystem: "github-actions"
-    directory: /actions/get-changes
-    schedule:
-      interval: "monthly"
-    labels:
-      - "💻 aspect: code"
-      - "🧰 goal: internal improvement"
-      - "🟩 priority: low"
-      - "🧱 stack: mgmt"
-      - dependencies
-
-  - package-ecosystem: "github-actions"
-    directory: /actions/load-img
-    schedule:
-      interval: "monthly"
-    labels:
-      - "💻 aspect: code"
-      - "🧰 goal: internal improvement"
-      - "🟩 priority: low"
-      - "🧱 stack: mgmt"
-      - dependencies
-
-  - package-ecosystem: "github-actions"
-    directory: /actions/setup-env
-    schedule:
-      interval: "monthly"
-    labels:
-      - "💻 aspect: code"
-      - "🧰 goal: internal improvement"
-      - "🟩 priority: low"
-      - "🧱 stack: mgmt"
-      - dependencies
diff --git a/.github/renovate.json b/.github/renovate.json
index c26690d1eb7..08e8dfabc5d 100644
--- a/.github/renovate.json
+++ b/.github/renovate.json
@@ -5,7 +5,14 @@
   "onboarding": false,
   "platform": "github",
   "repositories": ["WordPress/openverse"],
-  "extends": ["config:base", ":preserveSemverRanges", "schedule:monthly"],
+  "extends": [
+    "config:recommended",
+    ":preserveSemverRanges",
+    ":automergeMinor",
+    ":automergePr",
+    "schedule:monthly",
+    "group:monorepos"
+  ],
   "prCommitsPerRunLimit": 3,
   "labels": [
     "dependencies",
@@ -16,9 +23,66 @@
   "packageRules": [
     {
       "matchDatasources": ["docker"],
-      "labels": ["🐳 tech: docker"]
+      "addLabels": ["🐳 tech: docker"]
+    },
+    {
+      "groupName": "workflows",
+      "matchFileNames": [
+        ".github/{actions,workflows}/**",
+        "automations/**",
+        ".pre-commit-config.yaml"
+      ],
+      "matchManagers": ["pre-commit", "github-actions"],
+      "addLabels": ["🧱 stack: mgmt"]
+    },
+    {
+      "matchDatasources": ["pypi"],
+      "addLabels": ["🐍 tech: python"]
+    },
+    {
+      "matchDatasources": ["npm"],
+      "addLabels": ["🟨 tech: javascript"]
+    },
+    {
+      "matchFileNames": ["{frontend,packages}/**"],
+      "addLabels": ["🧱 stack: frontend"]
+    },
+    {
+      "matchFileNames": ["api/**"],
+      "addLabels": ["🧱 stack: api"]
+    },
+    {
+      "matchFileNames": ["ingestion_server/**"],
+      "addLabels": ["🧱 stack: ingestion server"]
+    },
+    {
+      "matchFileNames": ["catalog/**"],
+      "addLabels": ["🧱 stack: catalog"]
+    },
+    {
+      "matchFileNames": ["documentation/**"],
+      "addLabels": ["🧱 stack: documentation"]
+    },
+    {
+      "groupName": "python dev dependencies",
+      "matchDepTypes": ["dev-packages"],
+      "matchManagers": ["pipenv"],
+      "matchFileNames": ["api/**", "ingestion_server/**", "documentation/**"]
+    },
+    {
+      "groupName": "@openverse/eslint-plugin",
+      "matchFileNames": ["packages/eslint-plugin/**"]
+    },
+    {
+      "groupName": "tailwind",
+      "matchPackagePatterns": ["tailwindcss"]
+    },
+    {
+      "groupName": "@testing-library",
+      "matchPackagePrefixes": ["@testing-library"]
     }
   ],
-  "ignorePaths": ["package.json"],
-  "includeForks": false
+  "forkProcessing": "disabled",
+  "allowScripts": true,
+  "pre-commit": { "enabled": true }
 }
diff --git a/.github/workflows/renovate.yml b/.github/workflows/renovate.yml
index 5fa4c82f99a..53141fd7c19 100644
--- a/.github/workflows/renovate.yml
+++ b/.github/workflows/renovate.yml
@@ -1,10 +1,29 @@
 name: Renovate
 on:
+  # This lets you dispatch a renovate job with different cache options if you want to reset or disable the cache manually.
   workflow_dispatch:
+    inputs:
+      repoCache:
+        description: "Reset or disable the cache?"
+        type: choice
+        default: enabled
+        options:
+          - enabled
+          - disabled
+          - reset
   schedule:
-    # The "*" (#42, asterisk) character has special semantics in YAML, so this
-    # string has to be quoted.
-    - cron: "0/15 * * * *"
+    # Run every 30 minutes:
+    - cron: "0,30 * * * *"
+
+# Adding these as env variables makes it easy to re-use them in different steps and in bash.
+env:
+  cache_archive: renovate_cache.tar.gz
+  # This is the dir renovate provides -- if we set our own directory via cacheDir, we can run into permissions issues.
+  # It is also possible to cache a higher level of the directory, but it has minimal benefit. While renovate execution
+  # time gets faster, it also takes longer to upload the cache as it grows bigger.
+  cache_dir: /tmp/renovate/cache/renovate/repository
+  # This can be manually changed to bust the cache if necessary.
+  cache_key: renovate-cache
 
 jobs:
   renovate:
@@ -12,7 +31,65 @@ jobs:
     if: github.repository == 'WordPress/openverse'
     steps:
       - uses: actions/checkout@v4
+
+      # This third party action allows you to download the cache artifact from different workflow runs
+      # Note that actions/cache doesn't work well because the cache key would need to be computed from
+      # a file within the cache, meaning there would never be any data to restore. With other keys, the
+      # cache wouldn't necessarily upload when it changes. actions/download-artifact also doesn't work
+      # because it only handles artifacts uploaded in the same run, and we want to restore from the
+      # previous successful run.
+      - uses: dawidd6/action-download-artifact@v2
+        if: github.event.inputs.repoCache != 'disabled'
+        continue-on-error: true
+        with:
+          name: ${{ env.cache_key }}
+          path: cache-download
+
+      # Using tar to compress and extract the archive isn't strictly necessary, but it can improve
+      # performance significantly when uploading artifacts with lots of files.
+      - name: Extract renovate cache
+        run: |
+          set -x
+          # Skip if no cache is set, such as the first time it runs.
+          if [ ! -d cache-download ] ; then
+            echo "No cache found."
+            exit 0
+          fi
+
+          # Make sure the directory exists, and extract it there. Note that it's nested in the download directory.
+          mkdir -p "$cache_dir"
+          tar -xzf "cache-download/$cache_archive" -C "$cache_dir"
+
+          # Unfortunately, the permissions expected within renovate's docker container
+          # are different than the ones given after the cache is restored. We have to
+          # change ownership to solve this. We also need to have correct permissions in
+          # the entire /tmp/renovate tree, not just the section with the repo cache.
+          sudo chown -R runneradmin:root /tmp/renovate/
+          ls -R "$cache_dir"
+
       - uses: renovatebot/github-action@v39.1.1
         with:
-          configurationFile: .github/renovate.json
+          configurationFile: .github/renovate-config.js
           token: ${{ secrets.ACCESS_TOKEN }}
+        env:
+          # This enables the cache -- if this is set, it's not necessary to add it to renovate.json.
+          RENOVATE_REPOSITORY_CACHE: ${{ github.event.inputs.repoCache || 'enabled' }}
+
+      # Compression helps performance in the upload step!
+      - name: Compress renovate cache
+        run: |
+          ls "$cache_dir"
+          # The -C is important -- otherwise we end up extracting the files with
+          # their full path, ultimately leading to a nested directory situation.
+          # To solve *that*, we'd have to extract to root (/), which isn't safe.
+          tar -czvf "$cache_archive" -C "$cache_dir" .
+
+      - uses: actions/upload-artifact@v3
+        if: github.event.inputs.repoCache != 'disabled'
+        with:
+          name: ${{ env.cache_key }}
+          path: ${{ env.cache_archive }}
+          # Since this is updated and restored on every run, we don't need to keep it
+          # for long. Just make sure this value is large enough that multiple renovate
+          # runs can happen before older cache archives are deleted.
+          retention-days: 1
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index e8f4b444f3c..8b112b612cb 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -171,3 +171,10 @@ repos:
         entry: bash -c 'just automations/js/render-release-drafter'
         language: system
         pass_filenames: false
+
+  - repo: https://github.com/renovatebot/pre-commit-hooks
+    rev: 37.43.1
+    hooks:
+      - id: renovate-config-validator
+        args:
+          - --strict

From e35777d3b2cc4b805708b06a58793d008ebeee30 Mon Sep 17 00:00:00 2001
From: sarayourfriend <24264157+sarayourfriend@users.noreply.github.com>
Date: Sat, 4 Nov 2023 10:58:50 +1100
Subject: [PATCH 2/3] Update .github/workflows/renovate.yml

---
 .github/workflows/renovate.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/renovate.yml b/.github/workflows/renovate.yml
index 53141fd7c19..15be4ac0b08 100644
--- a/.github/workflows/renovate.yml
+++ b/.github/workflows/renovate.yml
@@ -69,7 +69,7 @@ jobs:
 
       - uses: renovatebot/github-action@v39.1.1
         with:
-          configurationFile: .github/renovate-config.js
+          configurationFile: .github/renovate.json
           token: ${{ secrets.ACCESS_TOKEN }}
         env:
           # This enables the cache -- if this is set, it's not necessary to add it to renovate.json.

From 192548538e85ff61f4066d6f8ce9581964738538 Mon Sep 17 00:00:00 2001
From: sarayourfriend <24264157+sarayourfriend@users.noreply.github.com>
Date: Mon, 6 Nov 2023 18:01:52 +1100
Subject: [PATCH 3/3] Use actions/cache with constant key

---
 .github/workflows/renovate.yml | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/renovate.yml b/.github/workflows/renovate.yml
index 15be4ac0b08..25522493e6e 100644
--- a/.github/workflows/renovate.yml
+++ b/.github/workflows/renovate.yml
@@ -32,18 +32,12 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      # This third party action allows you to download the cache artifact from different workflow runs
-      # Note that actions/cache doesn't work well because the cache key would need to be computed from
-      # a file within the cache, meaning there would never be any data to restore. With other keys, the
-      # cache wouldn't necessarily upload when it changes. actions/download-artifact also doesn't work
-      # because it only handles artifacts uploaded in the same run, and we want to restore from the
-      # previous successful run.
-      - uses: dawidd6/action-download-artifact@v2
+      - uses: actions/cache@v3
         if: github.event.inputs.repoCache != 'disabled'
         continue-on-error: true
         with:
-          name: ${{ env.cache_key }}
           path: cache-download
+          key: ${{ env.cache_key }}
 
       # Using tar to compress and extract the archive isn't strictly necessary, but it can improve
       # performance significantly when uploading artifacts with lots of files.