From b551b52d64669f26a88d7031ef9f6b3689ce4da9 Mon Sep 17 00:00:00 2001 From: Robert Timm Date: Fri, 1 Mar 2024 09:54:25 +0000 Subject: [PATCH 01/11] wip: temurin as baseimage --- build/WDQS/Dockerfile | 15 +++++++++------ build/WDQS/entrypoint.sh | 6 +++--- variables.env | 2 +- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/build/WDQS/Dockerfile b/build/WDQS/Dockerfile index 6484fdcc8..997090111 100644 --- a/build/WDQS/Dockerfile +++ b/build/WDQS/Dockerfile @@ -26,12 +26,15 @@ RUN set -x; \ FROM ${JDK_IMAGE_URL} LABEL org.opencontainers.image.source="https://github.com/wmde/wikibase-release-pipeline" -# Blazegraph scripts require bash -# Install gettext for envsubst command, (it needs libintl package) -# Install curl for the loadData.sh wdqs script (if someone needs it) -RUN set -x ; \ - apk --no-cache add bash=\<4.5.0 gettext=\<0.19.8.2 libintl=\<0.19.8.2 curl=\<7.64.999 su-exec=\~0.2 && \ - addgroup -g 66 -S blazegraph && adduser -S -G blazegraph -u 666 -s /bin/bash blazegraph +# # Blazegraph scripts require bash +# # Install curl for the loadData.sh wdqs script (if someone needs it) +RUN apt-get update && \ + apt-get install --yes --no-install-recommends bash gettext curl && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +RUN addgroup --system blazegraph && \ + adduser --system --gid $(getent group blazegraph | cut -d: -f3) --shell /bin/bash blazegraph COPY --from=fetcher --chown=blazegraph:blazegraph /tmp/wdqs-service /wdqs diff --git a/build/WDQS/entrypoint.sh b/build/WDQS/entrypoint.sh index 9961142b8..9e44bdad1 100755 --- a/build/WDQS/entrypoint.sh +++ b/build/WDQS/entrypoint.sh @@ -17,12 +17,12 @@ export BLAZEGRAPH_OPTS="${BLAZEGRAPH_EXTRA_OPTS} -DwikibaseHost=${WIKIBASE_HOST} export UPDATER_OPTS="-DwikibaseHost=${WIKIBASE_HOST} -DwikibaseMaxDaysBack=${WIKIBASE_MAX_DAYS_BACK}" envsubst < /templates/mwservices.json > /wdqs/mwservices.json -chown 666:66 /wdqs/mwservices.json +chown blazegraph:blazegraph /wdqs/mwservices.json # The data directory should always be owned by the blazegraph user # This used to be owned by root (https://phabricator.wikimedia.org/T237248) if [ -d /wdqs/data/ ]; then - chown 666:66 -R /wdqs/data/ + chown blazegraph:blazegraph -R /wdqs/data/ fi -su-exec 666:66 "$@" +su --preserve-environment -g blazegraph - blazegraph "$@" diff --git a/variables.env b/variables.env index 9845ecfef..2d41f4886 100644 --- a/variables.env +++ b/variables.env @@ -89,7 +89,7 @@ PHP_IMAGE_URL=php:8.1.27-apache-bookworm # https://hub.docker.com/_/openjdk/ # TODO: move to debian for consistency # TODO: upgrade, end of life -JDK_IMAGE_URL=openjdk:8-jdk-alpine +JDK_IMAGE_URL=eclipse-temurin:8-jre-jammy # https://hub.docker.com/_/debian DEBIAN_IMAGE_URL=debian:bookworm-slim From ae8d292073e35e3ac488cd8a9d19db0bc89d3d1d Mon Sep 17 00:00:00 2001 From: roti Date: Tue, 5 Mar 2024 07:42:07 +0000 Subject: [PATCH 02/11] chore: combine RUN invokations --- build/WDQS/Dockerfile | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/build/WDQS/Dockerfile b/build/WDQS/Dockerfile index 997090111..66be83c97 100644 --- a/build/WDQS/Dockerfile +++ b/build/WDQS/Dockerfile @@ -26,15 +26,17 @@ RUN set -x; \ FROM ${JDK_IMAGE_URL} LABEL org.opencontainers.image.source="https://github.com/wmde/wikibase-release-pipeline" -# # Blazegraph scripts require bash -# # Install curl for the loadData.sh wdqs script (if someone needs it) +# Blazegraph scripts require bash +# Install curl for the loadData.sh wdqs script (if someone needs it) RUN apt-get update && \ apt-get install --yes --no-install-recommends bash gettext curl && \ apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -RUN addgroup --system blazegraph && \ - adduser --system --gid $(getent group blazegraph | cut -d: -f3) --shell /bin/bash blazegraph + rm -rf /var/lib/apt/lists/* && \ + \ + addgroup --system blazegraph && \ + adduser --system \ + --gid $(getent group blazegraph | cut -d: -f3) \ + --shell /bin/bash blazegraph COPY --from=fetcher --chown=blazegraph:blazegraph /tmp/wdqs-service /wdqs From e0ec582b29891110864c1091fabf0de75b8a5463 Mon Sep 17 00:00:00 2001 From: roti Date: Tue, 5 Mar 2024 07:42:50 +0000 Subject: [PATCH 03/11] chore: use root wrapper script again --- build/WDQS/entrypoint.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build/WDQS/entrypoint.sh b/build/WDQS/entrypoint.sh index 9e44bdad1..cb4bcc132 100755 --- a/build/WDQS/entrypoint.sh +++ b/build/WDQS/entrypoint.sh @@ -25,4 +25,5 @@ if [ -d /wdqs/data/ ]; then chown blazegraph:blazegraph -R /wdqs/data/ fi -su --preserve-environment -g blazegraph - blazegraph "$@" +# Start Blazegraph as the blazegraph user, forwardin the path so java is available +su -g blazegraph blazegraph -l -c "export PATH=$PATH; /runBlazegraph.sh" From 647429e827225f0c94464345115385ab76f44c4c Mon Sep 17 00:00:00 2001 From: roti Date: Tue, 5 Mar 2024 07:49:02 +0000 Subject: [PATCH 04/11] fix: preserve-environment --- build/WDQS/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/WDQS/entrypoint.sh b/build/WDQS/entrypoint.sh index cb4bcc132..ce08d1bdd 100755 --- a/build/WDQS/entrypoint.sh +++ b/build/WDQS/entrypoint.sh @@ -26,4 +26,4 @@ if [ -d /wdqs/data/ ]; then fi # Start Blazegraph as the blazegraph user, forwardin the path so java is available -su -g blazegraph blazegraph -l -c "export PATH=$PATH; /runBlazegraph.sh" +su --preserve-environment --command "export PATH=$PATH; /runBlazegraph.sh" blazegraph From cbfb5bae2d46de8a424f50763af19a4496691a62 Mon Sep 17 00:00:00 2001 From: roti Date: Tue, 5 Mar 2024 08:00:55 +0000 Subject: [PATCH 05/11] chore: JDK->JRE, base on debian --- build.sh | 2 +- build/WDQS/Dockerfile | 15 +++++++++++++-- variables.env | 8 ++++---- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/build.sh b/build.sh index d3540faff..b4bd28f0d 100755 --- a/build.sh +++ b/build.sh @@ -119,7 +119,7 @@ function build_wdqs { docker build \ $DOCKER_BUILD_CACHE_OPT \ --build-arg DEBIAN_IMAGE_URL="$DEBIAN_IMAGE_URL" \ - --build-arg JDK_IMAGE_URL="$JDK_IMAGE_URL" \ + --build-arg JRE_IMAGE_URL="$JRE_IMAGE_URL" \ --build-arg WDQS_VERSION="$WDQS_VERSION" \ \ -t "$WIKIBASE_SUITE_WDQS_IMAGE_URL" \ diff --git a/build/WDQS/Dockerfile b/build/WDQS/Dockerfile index 66be83c97..a3e97d744 100644 --- a/build/WDQS/Dockerfile +++ b/build/WDQS/Dockerfile @@ -1,5 +1,10 @@ ARG DEBIAN_IMAGE_URL -ARG JDK_IMAGE_URL +ARG JRE_IMAGE_URL + +# ########################################################################### +# hadolint ignore=DL3006 +FROM ${JRE_IMAGE_URL} as jdk + # ########################################################################### # hadolint ignore=DL3006 @@ -21,11 +26,17 @@ RUN set -x; \ mkdir /tmp/wdqs-service; \ tar zxvf $TARBALL -C /tmp/wdqs-service --strip-components=1 + # ########################################################################### # hadolint ignore=DL3006 -FROM ${JDK_IMAGE_URL} +FROM ${DEBIAN_IMAGE_URL} LABEL org.opencontainers.image.source="https://github.com/wmde/wikibase-release-pipeline" +# https://github.com/docker-library/docs/tree/master/eclipse-temurin/README.md#supported-tags-and-respective-dockerfile-links +ENV JAVA_HOME=/opt/java/openjdk +COPY --from=jdk $JAVA_HOME $JAVA_HOME +ENV PATH="${JAVA_HOME}/bin:${PATH}" + # Blazegraph scripts require bash # Install curl for the loadData.sh wdqs script (if someone needs it) RUN apt-get update && \ diff --git a/variables.env b/variables.env index 2d41f4886..5f23ca12a 100644 --- a/variables.env +++ b/variables.env @@ -86,10 +86,10 @@ NODE_IMAGE_URL=node:14.21.3-alpine3.17 # https://hub.docker.com/_/php PHP_IMAGE_URL=php:8.1.27-apache-bookworm -# https://hub.docker.com/_/openjdk/ -# TODO: move to debian for consistency -# TODO: upgrade, end of life -JDK_IMAGE_URL=eclipse-temurin:8-jre-jammy +# https://hub.docker.com/_/eclipse-temurin +# https://github.com/docker-library/docs/tree/master/eclipse-temurin/README.md#supported-tags-and-respective-dockerfile-links +# No debian based image released by eclipse-temurin, let take ubuntu here. +JRE_IMAGE_URL=eclipse-temurin:8u402-b06-jre-jammy # https://hub.docker.com/_/debian DEBIAN_IMAGE_URL=debian:bookworm-slim From d3d028f29057960368e7e0ea82a1f72a7dfa4533 Mon Sep 17 00:00:00 2001 From: roti Date: Tue, 5 Mar 2024 08:03:32 +0000 Subject: [PATCH 06/11] docs: comment --- build/WDQS/Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/build/WDQS/Dockerfile b/build/WDQS/Dockerfile index a3e97d744..fdb451324 100644 --- a/build/WDQS/Dockerfile +++ b/build/WDQS/Dockerfile @@ -32,7 +32,10 @@ RUN set -x; \ FROM ${DEBIAN_IMAGE_URL} LABEL org.opencontainers.image.source="https://github.com/wmde/wikibase-release-pipeline" -# https://github.com/docker-library/docs/tree/master/eclipse-temurin/README.md#supported-tags-and-respective-dockerfile-links +# Base as many images as possible on debian. +# eclipse-temurin is not available for debian, so we copy the JRE over from +# an eclipse-temurin ubuntu container. +# https://github.com/docker-library/docs/blob/master/eclipse-temurin/README.md#using-a-different-base-image ENV JAVA_HOME=/opt/java/openjdk COPY --from=jdk $JAVA_HOME $JAVA_HOME ENV PATH="${JAVA_HOME}/bin:${PATH}" From 6929e5105397f8052b0f464642d58439d9ad6546 Mon Sep 17 00:00:00 2001 From: roti Date: Tue, 5 Mar 2024 08:24:09 +0000 Subject: [PATCH 07/11] fix: use docker commnd in entrypoint --- build/WDQS/entrypoint.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/build/WDQS/entrypoint.sh b/build/WDQS/entrypoint.sh index ce08d1bdd..5a174ecb3 100755 --- a/build/WDQS/entrypoint.sh +++ b/build/WDQS/entrypoint.sh @@ -25,5 +25,7 @@ if [ -d /wdqs/data/ ]; then chown blazegraph:blazegraph -R /wdqs/data/ fi -# Start Blazegraph as the blazegraph user, forwardin the path so java is available -su --preserve-environment --command "export PATH=$PATH; /runBlazegraph.sh" blazegraph +# Start as the blazegraph user. +# --preserve-environment does not preserve PATH, so we manually +# set the PATH again. Java cannot be found otherwise. +su --preserve-environment --command "export PATH=$PATH; $*" blazegraph From f09b465cada0af0a702b101ca4fe3ddb09d3ce61 Mon Sep 17 00:00:00 2001 From: roti Date: Tue, 5 Mar 2024 08:29:09 +0000 Subject: [PATCH 08/11] fix: lint --- build/WDQS/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build/WDQS/Dockerfile b/build/WDQS/Dockerfile index fdb451324..a3e0fcea7 100644 --- a/build/WDQS/Dockerfile +++ b/build/WDQS/Dockerfile @@ -42,6 +42,7 @@ ENV PATH="${JAVA_HOME}/bin:${PATH}" # Blazegraph scripts require bash # Install curl for the loadData.sh wdqs script (if someone needs it) +SHELL [ "/bin/bash", "-o", "pipefail", "-c" ] RUN apt-get update && \ apt-get install --yes --no-install-recommends bash gettext curl && \ apt-get clean && \ @@ -49,7 +50,7 @@ RUN apt-get update && \ \ addgroup --system blazegraph && \ adduser --system \ - --gid $(getent group blazegraph | cut -d: -f3) \ + --gid "$(getent group blazegraph | cut -d: -f3)" \ --shell /bin/bash blazegraph COPY --from=fetcher --chown=blazegraph:blazegraph /tmp/wdqs-service /wdqs From d497b284c6c29c4756f9c1b8a5837155f9fdb742 Mon Sep 17 00:00:00 2001 From: roti Date: Tue, 5 Mar 2024 14:51:45 +0000 Subject: [PATCH 09/11] chore: bump jre to 21 --- build/WDQS/Dockerfile | 4 ++-- variables.env | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build/WDQS/Dockerfile b/build/WDQS/Dockerfile index a3e0fcea7..d61ebc708 100644 --- a/build/WDQS/Dockerfile +++ b/build/WDQS/Dockerfile @@ -3,7 +3,7 @@ ARG JRE_IMAGE_URL # ########################################################################### # hadolint ignore=DL3006 -FROM ${JRE_IMAGE_URL} as jdk +FROM ${JRE_IMAGE_URL} as jre # ########################################################################### @@ -37,7 +37,7 @@ LABEL org.opencontainers.image.source="https://github.com/wmde/wikibase-release- # an eclipse-temurin ubuntu container. # https://github.com/docker-library/docs/blob/master/eclipse-temurin/README.md#using-a-different-base-image ENV JAVA_HOME=/opt/java/openjdk -COPY --from=jdk $JAVA_HOME $JAVA_HOME +COPY --from=jre $JAVA_HOME $JAVA_HOME ENV PATH="${JAVA_HOME}/bin:${PATH}" # Blazegraph scripts require bash diff --git a/variables.env b/variables.env index 5f23ca12a..a6291858c 100644 --- a/variables.env +++ b/variables.env @@ -89,7 +89,7 @@ PHP_IMAGE_URL=php:8.1.27-apache-bookworm # https://hub.docker.com/_/eclipse-temurin # https://github.com/docker-library/docs/tree/master/eclipse-temurin/README.md#supported-tags-and-respective-dockerfile-links # No debian based image released by eclipse-temurin, let take ubuntu here. -JRE_IMAGE_URL=eclipse-temurin:8u402-b06-jre-jammy +JRE_IMAGE_URL=eclipse-temurin:21.0.2_13-jre-jammy # https://hub.docker.com/_/debian DEBIAN_IMAGE_URL=debian:bookworm-slim From 014421ea9b7a3e983da5d698b35b6931af1d0cf8 Mon Sep 17 00:00:00 2001 From: roti Date: Tue, 5 Mar 2024 15:55:46 +0000 Subject: [PATCH 10/11] docs: note about sticking to java 8 --- variables.env | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/variables.env b/variables.env index 2c65874b9..c978d0bb0 100644 --- a/variables.env +++ b/variables.env @@ -88,7 +88,8 @@ PHP_IMAGE_URL=php:8.1.27-apache-bookworm # https://hub.docker.com/_/eclipse-temurin # https://github.com/docker-library/docs/tree/master/eclipse-temurin/README.md#supported-tags-and-respective-dockerfile-links # No debian based image released by eclipse-temurin, let take ubuntu here. -JRE_IMAGE_URL=eclipse-temurin:21.0.2_13-jre-jammy +# As of March 2024, WDQS' startup scripts depend on the JRE 8 command line option PrintGCDateStamps. +JRE_IMAGE_URL=eclipse-temurin:8u402-b06-jre-jammy # https://hub.docker.com/_/debian DEBIAN_IMAGE_URL=debian:bookworm-slim From 4c97a18bbfca47c687168baaf825819f277c4b4f Mon Sep 17 00:00:00 2001 From: roti Date: Tue, 5 Mar 2024 16:04:12 +0000 Subject: [PATCH 11/11] docs: ticket about wdqs java update --- variables.env | 1 + 1 file changed, 1 insertion(+) diff --git a/variables.env b/variables.env index c978d0bb0..99e00e01b 100644 --- a/variables.env +++ b/variables.env @@ -89,6 +89,7 @@ PHP_IMAGE_URL=php:8.1.27-apache-bookworm # https://github.com/docker-library/docs/tree/master/eclipse-temurin/README.md#supported-tags-and-respective-dockerfile-links # No debian based image released by eclipse-temurin, let take ubuntu here. # As of March 2024, WDQS' startup scripts depend on the JRE 8 command line option PrintGCDateStamps. +# https://phabricator.wikimedia.org/T316103 JRE_IMAGE_URL=eclipse-temurin:8u402-b06-jre-jammy # https://hub.docker.com/_/debian