From e4d1ecf7cc87cdd149842c3d59cba2d1514ead17 Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Tue, 10 Dec 2024 10:40:48 +0400 Subject: [PATCH 1/9] Use ov in CI + CD --- .github/actions/setup-env/action.yml | 7 +++++++ .github/workflows/ci_cd.yml | 19 +++++++------------ 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/.github/actions/setup-env/action.yml b/.github/actions/setup-env/action.yml index 63af914878a..1e654b83487 100644 --- a/.github/actions/setup-env/action.yml +++ b/.github/actions/setup-env/action.yml @@ -21,6 +21,13 @@ inputs: runs: using: "composite" steps: + # Setup dev-env + - name: Setup ov + shell: bash + run: | + ./ov build + ./ov setup-env + # Just - name: Setup just uses: taiki-e/install-action@v2 diff --git a/.github/workflows/ci_cd.yml b/.github/workflows/ci_cd.yml index 59212efa2fe..27c57ca3cc2 100644 --- a/.github/workflows/ci_cd.yml +++ b/.github/workflows/ci_cd.yml @@ -109,23 +109,18 @@ jobs: - name: Setup CI env uses: ./.github/actions/setup-env with: - # PDM is needed to run Vale - setup_python: "true" - # Node.js is needed by lint actions. - install_recipe: "node-install" - - - name: Cache pre-commit envs - uses: actions/cache@v4 - with: - path: ~/.cache/pre-commit - key: ${{ runner.os }}-pre-commit-${{ hashFiles('.pre-commit-config.yaml') }} + setup_python: false + # Python is not needed to run lint. + setup_nodejs: false + # Node.js is not needed to run lint. + install_recipe: "" - name: Run pre-commit to lint files env: GITHUB_TOKEN: ${{ github.token }} run: | - just precommit - just lint + ./ov just precommit + ./ov just lint validate-codeowners: name: Validate CODEOWNERS From 4e71aac129c28784ff543de1a03ffff4dc5ca331 Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Tue, 10 Dec 2024 11:23:53 +0400 Subject: [PATCH 2/9] Ensure NPM cache exists and has mode 777 --- docker/dev_env/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/dev_env/Dockerfile b/docker/dev_env/Dockerfile index f661bf4795b..f3fdfe8968b 100644 --- a/docker/dev_env/Dockerfile +++ b/docker/dev_env/Dockerfile @@ -52,6 +52,7 @@ ENV PATH="${PNPM_BIN}:${N_PREFIX}/bin:${PDM_PYTHONS}:${HOME}/.local/bin:${PATH}" # - n: Node.js distribution manager # - corepack: Node.js package-manager-manager RUN mkdir /pipx \ + && mkdir /opt/.npm \ && dnf5 -y install dnf-plugins-core \ && dnf5 -y config-manager addrepo --from-repofile=https://download.docker.com/linux/fedora/docker-ce.repo \ && dnf5 -y install https://dl.k6.io/rpm/repo.rpm \ From ff81b360e11d43c07dffbb765093d0e1b14237c1 Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Tue, 10 Dec 2024 11:40:37 +0400 Subject: [PATCH 3/9] Move npm cache to a temp directory --- docker/dev_env/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/dev_env/Dockerfile b/docker/dev_env/Dockerfile index f3fdfe8968b..d9299f2c32e 100644 --- a/docker/dev_env/Dockerfile +++ b/docker/dev_env/Dockerfile @@ -18,6 +18,8 @@ ENV HOME="/opt" ENV PDM_PYTHONS="${HOME}/pdm/bin" # location where `n` installs Node.js versions ENV N_PREFIX="${HOME}/n" +# location for `npm`'s cache +ENV NPM_CONFIG_CACHE="/tmp/.npm/" ENV PIPX_GLOBAL_HOME=/pipx @@ -52,7 +54,6 @@ ENV PATH="${PNPM_BIN}:${N_PREFIX}/bin:${PDM_PYTHONS}:${HOME}/.local/bin:${PATH}" # - n: Node.js distribution manager # - corepack: Node.js package-manager-manager RUN mkdir /pipx \ - && mkdir /opt/.npm \ && dnf5 -y install dnf-plugins-core \ && dnf5 -y config-manager addrepo --from-repofile=https://download.docker.com/linux/fedora/docker-ce.repo \ && dnf5 -y install https://dl.k6.io/rpm/repo.rpm \ From 65d314ab1a085f226e63cfcb8f7bbea8d3ba2102 Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Tue, 10 Dec 2024 13:35:12 +0400 Subject: [PATCH 4/9] Run ov as root in CI --- docker/dev_env/Dockerfile | 2 -- docker/dev_env/run.sh | 3 +++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docker/dev_env/Dockerfile b/docker/dev_env/Dockerfile index d9299f2c32e..f661bf4795b 100644 --- a/docker/dev_env/Dockerfile +++ b/docker/dev_env/Dockerfile @@ -18,8 +18,6 @@ ENV HOME="/opt" ENV PDM_PYTHONS="${HOME}/pdm/bin" # location where `n` installs Node.js versions ENV N_PREFIX="${HOME}/n" -# location for `npm`'s cache -ENV NPM_CONFIG_CACHE="/tmp/.npm/" ENV PIPX_GLOBAL_HOME=/pipx diff --git a/docker/dev_env/run.sh b/docker/dev_env/run.sh index 48d35642736..a137ad59e7c 100755 --- a/docker/dev_env/run.sh +++ b/docker/dev_env/run.sh @@ -70,6 +70,9 @@ linux*) docker_group=$(getent group docker | cut -d: -f3) if [ "$1" == "sudo" ]; then user_id="0" + # Use root in CI + elif [ -n "$CI" ]; then + user_id="0" else user_id="$UID" fi From 9a560efbb4bdd9d4f233c81fee48e95bc5c75cc4 Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Tue, 10 Dec 2024 13:43:40 +0400 Subject: [PATCH 5/9] Do not set user in CI --- docker/dev_env/run.sh | 41 ++++++++++++++++++++--------------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/docker/dev_env/run.sh b/docker/dev_env/run.sh index a137ad59e7c..a8fdcb351cb 100755 --- a/docker/dev_env/run.sh +++ b/docker/dev_env/run.sh @@ -65,30 +65,29 @@ if [ "$1" == "sudo" ]; then _cmd=("${@:2}") fi -case "$OSTYPE" in -linux*) - docker_group=$(getent group docker | cut -d: -f3) - if [ "$1" == "sudo" ]; then - user_id="0" - # Use root in CI - elif [ -n "$CI" ]; then - user_id="0" - else - user_id="$UID" - fi - shared_args+=(--user "$user_id:$docker_group") - ;; +if [ -z "$CI" ]; then + case "$OSTYPE" in + linux*) + docker_group=$(getent group docker | cut -d: -f3) + if [ "$1" == "sudo" ]; then + user_id="0" + else + user_id="$UID" + fi + shared_args+=(--user "$user_id:$docker_group") + ;; -darwin*) - # noop, just catching them to avoid the fall-through error case - ;; + darwin*) + # noop, just catching them to avoid the fall-through error case + ;; -*) - printf "Openverse development is only supported on Linux and macOS hosts. Please use WSL to run the Openverse development environment under Linux on Windows computers." >/dev/stderr - exit 1 - ;; + *) + printf "Openverse development is only supported on Linux and macOS hosts. Please use WSL to run the Openverse development environment under Linux on Windows computers." >/dev/stderr + exit 1 + ;; -esac + esac +fi # Ensure cache directory uses, `ov clean` destroys it mkdir -p "$DEV_ENV"/.cache From b2df8c70ab1f8d698b48ab13c2c2df109e90aab6 Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Wed, 11 Dec 2024 06:23:37 +0400 Subject: [PATCH 6/9] Revert changes --- docker/dev_env/run.sh | 38 ++++++++++++++++++-------------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/docker/dev_env/run.sh b/docker/dev_env/run.sh index a8fdcb351cb..48d35642736 100755 --- a/docker/dev_env/run.sh +++ b/docker/dev_env/run.sh @@ -65,29 +65,27 @@ if [ "$1" == "sudo" ]; then _cmd=("${@:2}") fi -if [ -z "$CI" ]; then - case "$OSTYPE" in - linux*) - docker_group=$(getent group docker | cut -d: -f3) - if [ "$1" == "sudo" ]; then - user_id="0" - else - user_id="$UID" - fi - shared_args+=(--user "$user_id:$docker_group") - ;; +case "$OSTYPE" in +linux*) + docker_group=$(getent group docker | cut -d: -f3) + if [ "$1" == "sudo" ]; then + user_id="0" + else + user_id="$UID" + fi + shared_args+=(--user "$user_id:$docker_group") + ;; - darwin*) - # noop, just catching them to avoid the fall-through error case - ;; +darwin*) + # noop, just catching them to avoid the fall-through error case + ;; - *) - printf "Openverse development is only supported on Linux and macOS hosts. Please use WSL to run the Openverse development environment under Linux on Windows computers." >/dev/stderr - exit 1 - ;; +*) + printf "Openverse development is only supported on Linux and macOS hosts. Please use WSL to run the Openverse development environment under Linux on Windows computers." >/dev/stderr + exit 1 + ;; - esac -fi +esac # Ensure cache directory uses, `ov clean` destroys it mkdir -p "$DEV_ENV"/.cache From 0319ce14c740d66caf9c3d5cf2a0bf26344d5e9e Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Wed, 11 Dec 2024 06:24:12 +0400 Subject: [PATCH 7/9] Remove redundant 'precommit' step --- .github/workflows/ci_cd.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci_cd.yml b/.github/workflows/ci_cd.yml index 27c57ca3cc2..f70e7d89a97 100644 --- a/.github/workflows/ci_cd.yml +++ b/.github/workflows/ci_cd.yml @@ -119,7 +119,6 @@ jobs: env: GITHUB_TOKEN: ${{ github.token }} run: | - ./ov just precommit ./ov just lint validate-codeowners: From b82b3819fdf86e5ac3ac5526b372c1ec3f824147 Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Wed, 11 Dec 2024 06:34:47 +0400 Subject: [PATCH 8/9] Bust npm cache before attempting to lint --- .github/workflows/ci_cd.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci_cd.yml b/.github/workflows/ci_cd.yml index f70e7d89a97..b7277f6bf6d 100644 --- a/.github/workflows/ci_cd.yml +++ b/.github/workflows/ci_cd.yml @@ -119,6 +119,7 @@ jobs: env: GITHUB_TOKEN: ${{ github.token }} run: | + ./ov rm -rf /opt/.npm ./ov just lint validate-codeowners: From 7e6f5d86023f220f53f25c37ee3a637020634515 Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Wed, 11 Dec 2024 06:37:25 +0400 Subject: [PATCH 9/9] Use `sudo` when removing files owned by root --- .github/workflows/ci_cd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_cd.yml b/.github/workflows/ci_cd.yml index b7277f6bf6d..e81e6f8c0b8 100644 --- a/.github/workflows/ci_cd.yml +++ b/.github/workflows/ci_cd.yml @@ -119,7 +119,7 @@ jobs: env: GITHUB_TOKEN: ${{ github.token }} run: | - ./ov rm -rf /opt/.npm + ./ov sudo rm -rf /opt/.npm ./ov just lint validate-codeowners: