Merge branch 'master' into jv-which

.github/dockerfiles/Dockerfile.32-bit

@@ -1,4 +1,4 @@
-ARG BASE=docker.pkg.github.com/erlang/otp/i386-debian-base
+ARG BASE=ghcr.io/erlang/otp/i386-debian-base
 FROM $BASE
 
 ARG MAKEFLAGS=-j4

.github/dockerfiles/Dockerfile.64-bit

@@ -1,4 +1,4 @@
-ARG BASE=docker.pkg.github.com/erlang/otp/ubuntu-base
+ARG BASE=ghcr.io/erlang/otp/ubuntu-base
 FROM $BASE
 
 ARG MAKEFLAGS=$MAKEFLAGS

.github/dockerfiles/Dockerfile.clang

@@ -1,4 +1,4 @@
-ARG BASE=docker.pkg.github.com/erlang/otp/ubuntu-base
+ARG BASE=ghcr.io/erlang/otp/ubuntu-base
 FROM $BASE
 ## We do a SSA lint check here
 ENV ERL_COMPILER_OPTIONS=ssalint
@@ -9,7 +9,7 @@ ENV MAKEFLAGS=$MAKEFLAGS \
     ERL_TOP=/buildroot/otp \
     PATH=/otp/bin:/buildroot/otp/bin:$PATH
 
-RUN sudo apt-get install -y clang
+RUN sudo apt-get update && sudo apt-get install -y clang
 
 ARG ARCHIVE=./otp.tar.gz
 COPY $ARCHIVE /buildroot/otp.tar.gz

.github/dockerfiles/Dockerfile.cross-compile

@@ -1,7 +1,7 @@
 ##
 ## This docker file will build Erlang on 32-bit to 64-bit x86
 ##
-ARG BASE=docker.pkg.github.com/erlang/otp/i386-debian-base
+ARG BASE=ghcr.io/erlang/otp/i386-debian-base
 FROM $BASE as build
 
 ARG MAKEFLAGS=-j4

.github/dockerfiles/Dockerfile.ubuntu-base

@@ -52,9 +52,11 @@ RUN apt-get install -y git curl && \
     chmod +x /usr/bin/kerl && \
     kerl update releases && \
     LATEST=$(kerl list releases | tail -1 | awk -F '.' '{print $1}') && \
-    echo "/usr/local/lib/erlang-${LATEST}/bin" > /home/${USER}/LATEST && \
     for release in $(seq $(( LATEST - 2 )) $(( LATEST ))); do \
       VSN=$(kerl list releases | grep "^$release" | tail -1); \
+      if [ $release = $LATEST ]; then \
+        echo "/usr/local/lib/erlang-${VSN}/bin" > /home/${USER}/LATEST; \
+      fi && \
       kerl build ${VSN} ${VSN} && \
       kerl install ${VSN} /usr/local/lib/erlang-${VSN}; \
     done && \
@@ -66,11 +68,12 @@ RUN apt-get install -y git curl && \
 ARG EXTRA_LIBS="erlang erlang-doc"
 RUN apt-get install -y \
     unixodbc odbc-postgresql postgresql ssh openssh-server groff-base gdb \
-    tinyproxy bind9 nsd expect vsftpd python emacs nano vim \
-    linux-tools-common linux-tools-generic linux-tools-`uname -r` jq \
+    tinyproxy knot ldnsutils expect vsftpd python emacs nano vim \
+    linux-tools-common linux-tools-generic jq \
     xvfb libgl1-mesa-dri && \
     for lib in ${EXTRA_LIBS}; do apt-get install -y ${lib}; done && \
     if [ ! -f /etc/apache2/apache2.conf ]; then apt-get install -y apache2; fi
+RUN apt-get install -y linux-tools-$(uname -r) || true
 
 ## We use tmux to test terminals
 RUN apt-get install -y libevent-dev libutf8proc-dev && \
@@ -102,7 +105,7 @@ COPY --chown=${USER}:${GROUP} dockerfiles/init.sh /buildroot/
 WORKDIR /buildroot/
 
 ## Install test tools rebar3, proper and jsx
-RUN export PATH="$(cat /home/${USER}/LATEST)/${PATH}" && \
+RUN export PATH="$(cat /home/${USER}/LATEST):${PATH}" && \
     latest () { \
     local VSN=$(curl -sL "https://api.github.com/repos/$1/tags" | jq -r ".[] | .name" | grep -E '^v?[0-9]' | sort -V | tail -1); \
     curl -sL "https://github.com/$1/archive/$VSN.tar.gz" > $(basename $1).tar.gz; \

.github/dockerfiles/init.sh

@@ -10,7 +10,8 @@ sudo /usr/sbin/sshd
 
 sudo service postgresql start
 
-sudo -E bash -c "apt-get update && apt-get install -y linux-tools-common linux-tools-generic linux-tools-`uname -r`"
+sudo -E bash -c "apt-get update && apt-get install -y linux-tools-common linux-tools-generic"
+sudo -E bash -c "apt-get install -y linux-tools-$(uname-r)" || true
 
 sudo bash -c "Xvfb :99 -ac -screen 0 1920x1080x24 -nolisten tcp" &
 export DISPLAY=:99

.github/scripts/build-base-image.sh

@@ -12,6 +12,10 @@ esac
 
 if [ -z "${BASE_TAG}" ]; then
     BASE_TAG=$(grep "ARG BASE=" ".github/dockerfiles/Dockerfile.${2}" | head -1 | tr '=' ' ' | awk '{print $3}')
+    ## If this script is used on pre 25 releases
+    if [ -z "${BASE_TAG}" ]; then
+        BASE_TAG=$(grep "FROM " ".github/dockerfiles/Dockerfile.${2}" | head -1 | awk '{print $2}')
+    fi
 fi
 
 case "${BASE_TAG}" in

.github/scripts/restore-from-prebuilt.sh

@@ -33,7 +33,7 @@ if [ -z "${CONFIGURE}" ] || [ "${CONFIGURE}" = "false" ]; then
 fi
 
 ## If bootstrap has been changed, we do not use the cached .beam files
-EXCLUDE_BOOTSTRAP=""
+EXCLUDE_BOOTSTRAP=()
 if [ "${BOOTSTRAP}" = "true" ]; then
     find "${CACHE_DIR}/otp/lib" -name "*.beam" -exec rm -f {} \;
 else

.github/scripts/sync-github-prs.es

@@ -5,9 +5,14 @@
 %% into the Target folder. It tries its best to not create too large
 %% files so that gh will still be happy with us when this is published to
 %% gh pages
+-module('sync-github-prs').
 -mode(compile).
 
 main([Repo, Target]) ->
+
+    io:format("Updating PRs in ~ts, current PRs are: ~p~n",
+              [Target, filelib:wildcard(filename:join(Target,"*"))]),
+
     AllOpenPrs = ghapi("gh api --paginate -X GET /repos/"++Repo++"/pulls -f state=open"),
     %% Download all updates, there really should not be any to download as they
     %% are updated when a PR is updated, but we do it anyways just to be safe.
@@ -26,7 +31,10 @@ main([Repo, Target]) ->
                   false ->
                       cmd("rm -rf " ++ filename:join(Target,PRNo))
               end
-      end, AllPrs);
+      end, AllPrs),
+
+    purge_prs(Target);
+
 main([Repo, Target, PRNo]) ->
     handle_prs(Repo, Target, [ghapi("gh api /repos/"++Repo++"/pulls/"++PRNo)]).
 
@@ -68,6 +76,9 @@ handle_pr(_Repo, Target,
             io:format("Checking for ~ts~n", [filename:join(PRDir, Ident)]),
             case file:read_file_info(filename:join(PRDir, Ident)) of
                 {error, enoent} ->
+                    io:format("Did not find ~ts. Files in dir are: ~p~n",
+                              [filename:join(PRDir, Ident),
+                               filelib:wildcard(filename:join(PRDir, "*"))]),
                     cmd("rm -rf "++PRDir),
                     ok = file:make_dir(PRDir),
                     ok = file:write_file(filename:join(PRDir,Ident), integer_to_list(Number)),
@@ -154,6 +165,30 @@ purge_suite(SuiteFilePath) ->
               end, filelib:wildcard(filename:join(SuiteDir,"*.html")))
     end.
 
+%% If we have more the 10 GB of PR data we need to remove some otherwise
+%% github actions will not work them. So we purge the largest files until we
+%% reach the 10 GB limit.
+purge_prs(Target) ->
+    %% Start by deleting all data from common_test test runs as they are huge.
+    os:cmd("rm -rf "++Target++"*/ct_logs/ct_run*/*common_test_test*/run*/log_private/ct_run*"),
+    Files = string:split(cmd("find " ++ Target ++ " -type f -a -name \! suite.log.html -exec du -a {} \+"),"\n",all),
+    SortedFiles =
+        lists:sort(fun([A|_]=As,[B|_]=Bs) ->
+                               binary_to_integer(A) >= binary_to_integer(B)
+                   end, [string:split(F,"\t") || F <- Files, F =/= <<>>]),
+    purge_prs(SortedFiles, Target, get_directory_size(Target)).
+purge_prs(Files, Target, Size) when Size > 10_000_000_000 ->
+    {H,T} = lists:split(10, Files),
+    [file:write_file(File, io_lib:format("Large file (~p bytes) truncated", [Sz]))
+     || [Sz, File] <- H],
+    purge_prs(T, Target, get_directory_size(Target));
+purge_prs(_, _, _) ->
+    ok.
+
+get_directory_size(Dir) ->
+    binary_to_integer(hd(string:split(cmd("du -b --max-depth=0 " ++ Dir),"\t"))).
+
+
 ghapi(CMD) ->
     decode(cmd(CMD)).
 

.github/scripts/sync-github-releases.sh

@@ -211,37 +211,40 @@ if [ ${UPLOADED} = true ]; then
 fi
 
 ## If no assets were uploaded, we try to build one instead
-if [ ${UPLOADED} = false ] && [ ${#MISSING_PREBUILD[0]} != 0 ]; then
-    name="${MISSING_PREBUILD[0]}"
-    stripped_name=$(_strip_name "${name}")
-    git clone https://github.com/erlang/otp -b "${name}" otp_src
-    if [ -f otp_src/.github/scripts/init-pre-release.sh ]; then
-        (cd otp_src && ERL_TOP=$(pwd) .github/scripts/init-pre-release.sh)
-    else
-        (cd otp_src && ERL_TOP=$(pwd) ../.github/scripts/init-pre-release.sh)
-    fi
-    case ${stripped_name} in
-        23.**)
-            ## The 32-bit dockerfile build the doc chunks which we want
-            ## to include in VSN >= 23.
-            docker build -t otp --build-arg ARCHIVE=otp_src/otp_src.tar.gz \
-                   -f otp_src/.github/dockerfiles/Dockerfile.32-bit .
-            ;;
-        *)
-            docker build -t otp --build-arg ARCHIVE=otp_src/otp_src.tar.gz \
-                   -f otp_src/.github/dockerfiles/Dockerfile.64-bit .
-            ;;
-    esac
-    docker run -v "$PWD":/github otp \
-           "/github/scripts/build-otp-tar -o /github/otp_clean_src.tar.gz /github/otp_src.tar.gz -b /buildroot/otp/ /buildroot/otp.tar.gz"
-    .github/scripts/release-docs.sh
-    .github/scripts/create-artifacts.sh downloads "${name}"
-
-    ## Delete any artifacts that we should not upload
-    for artifact in dowloads/*; do
-        if ! echo "${RI[@]}" | grep "${artifact}" 2> /dev/null > /dev/null; then
-            rm -f "downloads/${artifact}"
+if [ ${UPLOADED} = false ]; then
+    for name in "${MISSING_PREBUILD[@]}"; do
+        stripped_name=$(_strip_name "${name}")
+        release=$(echo "${stripped_name}" | awk -F. '{print $1}')
+        if [[ $release < 24 ]]; then
+            ## Releases before 24 are no longer supported and are a bit different
+            ## from 24+ so I've removed support for them
+            echo "Skipping old release ${name}"
+            continue;
+        fi
+        echo "Building pre-build and docs for ${name}"
+        git clone https://github.com/erlang/otp -b "${name}" otp_src
+        if [ -f otp_src/.github/scripts/init-pre-release.sh ]; then
+            (cd otp_src && ERL_TOP=$(pwd) .github/scripts/init-pre-release.sh)
+        else
+            (cd otp_src && ERL_TOP=$(pwd) ../.github/scripts/init-pre-release.sh)
         fi
+        (cd otp_src && BASE_USE_CACHE=false GITHUB_OUTPUT=.tmp ../.github/scripts/build-base-image.sh maint-${release} 64-bit)
+        docker build -t otp --build-arg ARCHIVE=otp_src/otp_src.tar.gz \
+               -f otp_src/.github/dockerfiles/Dockerfile.64-bit .
+        docker run -v "$PWD":/github otp \
+               "/github/scripts/build-otp-tar -o /github/otp_clean_src.tar.gz /github/otp_src.tar.gz -b /buildroot/otp/ /buildroot/otp.tar.gz"
+        .github/scripts/release-docs.sh
+        .github/scripts/create-artifacts.sh downloads "${name}"
+
+        ## Delete any artifacts that we should not upload
+        for artifact in dowloads/*; do
+            if ! echo "${RI[@]}" | grep "${artifact}" 2> /dev/null > /dev/null; then
+                rm -f "downloads/${artifact}"
+            fi
+        done
+        _upload_artifacts "${name}"
+
+        ## We only update one release per call to sync-github-releases
+        break
     done
-    _upload_artifacts "${name}"
 fi

.github/workflows/actions-updater.yaml

@@ -0,0 +1,22 @@
+name: GitHub Actions Updater
+
+on:
+  schedule:
+    # Automatically run on the 1st of every month
+    - cron:  '0 0 1 * *'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Run GitHub Actions Version Updater
+        uses: saadmk11/github-actions-version-updater@v0.7.3
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          commit_message: "Updating GitHub actions to their latest versions"
+          pull_request_labels: "team:IS"

.github/workflows/add-to-project.yaml

@@ -16,11 +16,11 @@ jobs:
     steps:
       - name: Generate token
         id: generate_token
-        uses: tibdex/github-app-token@v1.5.2
+        uses: tibdex/github-app-token@v1.8.0
         with:
           app_id: ${{ secrets.APP_ID }}
           private_key: ${{ secrets.APP_PEM }}
-      - uses: actions/add-to-project@v0.0.3
+      - uses: actions/add-to-project@v0.4.0
         with:
           project-url: https://github.com/orgs/erlang/projects/13
           github-token: ${{ steps.generate_token.outputs.token }}