From ccd05e283a0418de25f701942a09b604482bf0ca Mon Sep 17 00:00:00 2001 From: Vincent Date: Wed, 22 May 2024 15:47:12 +0200 Subject: [PATCH 01/12] WIP --- infra/docker/database/pg16_debian.Dockerfile | 51 ++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 infra/docker/database/pg16_debian.Dockerfile diff --git a/infra/docker/database/pg16_debian.Dockerfile b/infra/docker/database/pg16_debian.Dockerfile new file mode 100644 index 0000000000..8c0fa954d2 --- /dev/null +++ b/infra/docker/database/pg16_debian.Dockerfile @@ -0,0 +1,51 @@ +ARG TO_PG_MAJOR +ARG TIMESCALEDB_VERSION +ARG POSTGIS_VERSION +ARG DISTRIBUTION + +FROM postgres:$TO_PG_MAJOR-$DISTRIBUTION +ARG TO_PG_MAJOR +ARG TIMESCALEDB_VERSION +ARG POSTGIS_VERSION + +ENV PGBINNEW /usr/lib/postgresql/$TO_PG_MAJOR/bin + +ENV PGDATANEW /var/lib/postgresql/$TO_PG_MAJOR/data + +RUN set -eux; \ + mkdir -p "$PGDATANEW"; \ + chown -R postgres:postgres /var/lib/postgresql + +WORKDIR /var/lib/postgresql + +COPY infra/docker/database/docker-upgrade /usr/local/bin/ + +ENTRYPOINT ["docker-upgrade"] + +# recommended: --link +CMD ["pg_upgrade"] + + +# Install TimescaleDB extension in both versions of Postgres +RUN apt-get update +RUN apt-get install -y wget lsb-release +RUN echo "deb https://packagecloud.io/timescale/timescaledb/debian/ $(lsb_release -c -s) main" +RUN echo "deb https://packagecloud.io/timescale/timescaledb/debian/ $(lsb_release -c -s) main" > /etc/apt/sources.list.d/timescaledb.list +RUN wget --quiet -O - https://packagecloud.io/timescale/timescaledb/gpgkey | apt-key add - +RUN apt-get update + +RUN \ + TIMESCALEDB_MAJOR=$(echo $TIMESCALEDB_VERSION | cut -c1) && \ + apt-get install -y timescaledb-$TIMESCALEDB_MAJOR-postgresql-$FROM_PG_MAJOR=$TIMESCALEDB_VERSION* timescaledb-$TIMESCALEDB_MAJOR-loader-postgresql-$FROM_PG_MAJOR=$TIMESCALEDB_VERSION* && \ + apt-get install -y timescaledb-$TIMESCALEDB_MAJOR-postgresql-$TO_PG_MAJOR=$TIMESCALEDB_VERSION* timescaledb-$TIMESCALEDB_MAJOR-loader-postgresql-$TO_PG_MAJOR=$TIMESCALEDB_VERSION* + +# Install PostGIS extension in both versions of Postgres +RUN apt update +RUN \ + POSTGIS_MAJOR=$(echo $POSTGIS_VERSION | cut -c1) && \ + apt install -y --no-install-recommends \ + postgresql-$FROM_PG_MAJOR-postgis-$POSTGIS_MAJOR=$POSTGIS_VERSION* \ + postgresql-$FROM_PG_MAJOR-postgis-$POSTGIS_MAJOR-scripts \ + postgresql-$TO_PG_MAJOR-postgis-$POSTGIS_MAJOR=$POSTGIS_VERSION* \ + postgresql-$TO_PG_MAJOR-postgis-$POSTGIS_MAJOR-scripts \ + && rm -rf /var/lib/apt/lists/* \ No newline at end of file From 288326caa5779854c41c7b170fa0df51c6404650 Mon Sep 17 00:00:00 2001 From: Vincent Date: Wed, 22 May 2024 18:09:05 +0200 Subject: [PATCH 02/12] Add debian based database image --- .github/workflows/database.yml | 5 +++ infra/docker/database/pg16_debian.Dockerfile | 34 ++++---------------- 2 files changed, 11 insertions(+), 28 deletions(-) diff --git a/.github/workflows/database.yml b/.github/workflows/database.yml index f481cbda7f..d3cea3be54 100644 --- a/.github/workflows/database.yml +++ b/.github/workflows/database.yml @@ -33,6 +33,11 @@ jobs: postgis_version: "3.4.2" dockerfile: "pg13_16.Dockerfile" image_version: "pg16-ts2.14.2-postgis3.4.2" + - pg_major: "16" + timescaledb_version: "2.14.2" + postgis_version: "3.4.2" + dockerfile: "pg16_debian.Dockerfile" + image_version: "bookworm-pg16-ts2.14.2-postgis3.4.2" permissions: contents: read packages: write diff --git a/infra/docker/database/pg16_debian.Dockerfile b/infra/docker/database/pg16_debian.Dockerfile index 8c0fa954d2..9e1d8fbb65 100644 --- a/infra/docker/database/pg16_debian.Dockerfile +++ b/infra/docker/database/pg16_debian.Dockerfile @@ -1,31 +1,12 @@ -ARG TO_PG_MAJOR +ARG PG_MAJOR ARG TIMESCALEDB_VERSION ARG POSTGIS_VERSION -ARG DISTRIBUTION -FROM postgres:$TO_PG_MAJOR-$DISTRIBUTION -ARG TO_PG_MAJOR +FROM postgres:$PG_MAJOR-bookworm +ARG PG_MAJOR ARG TIMESCALEDB_VERSION ARG POSTGIS_VERSION -ENV PGBINNEW /usr/lib/postgresql/$TO_PG_MAJOR/bin - -ENV PGDATANEW /var/lib/postgresql/$TO_PG_MAJOR/data - -RUN set -eux; \ - mkdir -p "$PGDATANEW"; \ - chown -R postgres:postgres /var/lib/postgresql - -WORKDIR /var/lib/postgresql - -COPY infra/docker/database/docker-upgrade /usr/local/bin/ - -ENTRYPOINT ["docker-upgrade"] - -# recommended: --link -CMD ["pg_upgrade"] - - # Install TimescaleDB extension in both versions of Postgres RUN apt-get update RUN apt-get install -y wget lsb-release @@ -36,16 +17,13 @@ RUN apt-get update RUN \ TIMESCALEDB_MAJOR=$(echo $TIMESCALEDB_VERSION | cut -c1) && \ - apt-get install -y timescaledb-$TIMESCALEDB_MAJOR-postgresql-$FROM_PG_MAJOR=$TIMESCALEDB_VERSION* timescaledb-$TIMESCALEDB_MAJOR-loader-postgresql-$FROM_PG_MAJOR=$TIMESCALEDB_VERSION* && \ - apt-get install -y timescaledb-$TIMESCALEDB_MAJOR-postgresql-$TO_PG_MAJOR=$TIMESCALEDB_VERSION* timescaledb-$TIMESCALEDB_MAJOR-loader-postgresql-$TO_PG_MAJOR=$TIMESCALEDB_VERSION* + apt-get install -y timescaledb-$TIMESCALEDB_MAJOR-postgresql-$PG_MAJOR=$TIMESCALEDB_VERSION* timescaledb-$TIMESCALEDB_MAJOR-loader-postgresql-$PG_MAJOR=$TIMESCALEDB_VERSION* # Install PostGIS extension in both versions of Postgres RUN apt update RUN \ POSTGIS_MAJOR=$(echo $POSTGIS_VERSION | cut -c1) && \ apt install -y --no-install-recommends \ - postgresql-$FROM_PG_MAJOR-postgis-$POSTGIS_MAJOR=$POSTGIS_VERSION* \ - postgresql-$FROM_PG_MAJOR-postgis-$POSTGIS_MAJOR-scripts \ - postgresql-$TO_PG_MAJOR-postgis-$POSTGIS_MAJOR=$POSTGIS_VERSION* \ - postgresql-$TO_PG_MAJOR-postgis-$POSTGIS_MAJOR-scripts \ + postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR=$POSTGIS_VERSION* \ + postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR-scripts \ && rm -rf /var/lib/apt/lists/* \ No newline at end of file From 7e3dc112bfed8084b7d1f7bf8f15aa1675becae3 Mon Sep 17 00:00:00 2001 From: Vincent Date: Thu, 23 May 2024 09:26:48 +0200 Subject: [PATCH 03/12] Use debian based image for postgres 13 and 16 --- .github/workflows/database.yml | 5 - infra/docker/database/pg13_16.Dockerfile | 104 ++++++------------- infra/docker/database/pg16_debian.Dockerfile | 29 ------ 3 files changed, 30 insertions(+), 108 deletions(-) delete mode 100644 infra/docker/database/pg16_debian.Dockerfile diff --git a/.github/workflows/database.yml b/.github/workflows/database.yml index d3cea3be54..f481cbda7f 100644 --- a/.github/workflows/database.yml +++ b/.github/workflows/database.yml @@ -33,11 +33,6 @@ jobs: postgis_version: "3.4.2" dockerfile: "pg13_16.Dockerfile" image_version: "pg16-ts2.14.2-postgis3.4.2" - - pg_major: "16" - timescaledb_version: "2.14.2" - postgis_version: "3.4.2" - dockerfile: "pg16_debian.Dockerfile" - image_version: "bookworm-pg16-ts2.14.2-postgis3.4.2" permissions: contents: read packages: write diff --git a/infra/docker/database/pg13_16.Dockerfile b/infra/docker/database/pg13_16.Dockerfile index 8dac82b730..cedebdb38c 100644 --- a/infra/docker/database/pg13_16.Dockerfile +++ b/infra/docker/database/pg13_16.Dockerfile @@ -2,79 +2,35 @@ ARG PG_MAJOR ARG TIMESCALEDB_VERSION ARG POSTGIS_VERSION -FROM timescale/timescaledb:${TIMESCALEDB_VERSION}-pg${PG_MAJOR} +FROM postgres:$PG_MAJOR-bookworm +ARG PG_MAJOR +ARG TIMESCALEDB_VERSION ARG POSTGIS_VERSION -RUN set -eux \ - && apk add --no-cache --virtual .fetch-deps \ - ca-certificates \ - openssl \ - tar \ - \ - && wget -O postgis.tar.gz "https://github.com/postgis/postgis/archive/${POSTGIS_VERSION}.tar.gz" \ - && mkdir -p /usr/src/postgis \ - && tar \ - --extract \ - --file postgis.tar.gz \ - --directory /usr/src/postgis \ - --strip-components 1 \ - && rm postgis.tar.gz \ - \ - && apk add --no-cache --virtual .build-deps \ - \ - gdal-dev \ - geos-dev \ - proj-dev \ - proj-util \ - sfcgal-dev \ - \ - # The upstream variable, '$DOCKER_PG_LLVM_DEPS' contains - # the correct versions of 'llvm-dev' and 'clang' for the current version of PostgreSQL. - # This improvement has been discussed in https://github.com/docker-library/postgres/pull/1077 - $DOCKER_PG_LLVM_DEPS \ - \ - autoconf \ - automake \ - cunit-dev \ - file \ - g++ \ - gcc \ - gettext-dev \ - git \ - json-c-dev \ - libtool \ - libxml2-dev \ - make \ - pcre2-dev \ - perl \ - protobuf-c-dev \ - \ -# build PostGIS - with Link Time Optimization (LTO) enabled - && cd /usr/src/postgis \ - && gettextize \ - && ./autogen.sh \ - && ./configure \ - --enable-lto \ - && make -j$(nproc) \ - && make install \ - \ -# add .postgis-rundeps - && apk add --no-cache --virtual .postgis-rundeps \ - \ - gdal \ - geos \ - proj \ - sfcgal \ - \ - json-c \ - libstdc++ \ - pcre2 \ - protobuf-c \ - \ - # ca-certificates: for accessing remote raster files - # fix https://github.com/postgis/docker-postgis/issues/307 - ca-certificates \ -# clean - && cd / \ - && rm -rf /usr/src/postgis \ - && apk del .fetch-deps .build-deps \ +RUN \ + apt-get update && \ + apt-get install -y \ + wget \ + lsb-release + +# Install TimescaleDB extension +RUN echo "deb https://packagecloud.io/timescale/timescaledb/debian/ $(lsb_release -c -s) main" +RUN echo "deb https://packagecloud.io/timescale/timescaledb/debian/ $(lsb_release -c -s) main" > /etc/apt/sources.list.d/timescaledb.list +RUN wget --quiet -O - https://packagecloud.io/timescale/timescaledb/gpgkey | apt-key add - + +RUN \ + TIMESCALEDB_MAJOR=$(echo $TIMESCALEDB_VERSION | cut -c1) && \ + apt-get update && \ + apt-get install -y \ + timescaledb-$TIMESCALEDB_MAJOR-postgresql-$PG_MAJOR=$TIMESCALEDB_VERSION* \ + timescaledb-$TIMESCALEDB_MAJOR-loader-postgresql-$PG_MAJOR=$TIMESCALEDB_VERSION* && \ + rm -rf /var/lib/apt/lists/* + +# Install PostGIS extension +RUN \ + POSTGIS_MAJOR=$(echo $POSTGIS_VERSION | cut -c1) && \ + apt update && \ + apt install -y --no-install-recommends \ + postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR=$POSTGIS_VERSION* \ + postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR-scripts && \ + rm -rf /var/lib/apt/lists/* \ No newline at end of file diff --git a/infra/docker/database/pg16_debian.Dockerfile b/infra/docker/database/pg16_debian.Dockerfile deleted file mode 100644 index 9e1d8fbb65..0000000000 --- a/infra/docker/database/pg16_debian.Dockerfile +++ /dev/null @@ -1,29 +0,0 @@ -ARG PG_MAJOR -ARG TIMESCALEDB_VERSION -ARG POSTGIS_VERSION - -FROM postgres:$PG_MAJOR-bookworm -ARG PG_MAJOR -ARG TIMESCALEDB_VERSION -ARG POSTGIS_VERSION - -# Install TimescaleDB extension in both versions of Postgres -RUN apt-get update -RUN apt-get install -y wget lsb-release -RUN echo "deb https://packagecloud.io/timescale/timescaledb/debian/ $(lsb_release -c -s) main" -RUN echo "deb https://packagecloud.io/timescale/timescaledb/debian/ $(lsb_release -c -s) main" > /etc/apt/sources.list.d/timescaledb.list -RUN wget --quiet -O - https://packagecloud.io/timescale/timescaledb/gpgkey | apt-key add - -RUN apt-get update - -RUN \ - TIMESCALEDB_MAJOR=$(echo $TIMESCALEDB_VERSION | cut -c1) && \ - apt-get install -y timescaledb-$TIMESCALEDB_MAJOR-postgresql-$PG_MAJOR=$TIMESCALEDB_VERSION* timescaledb-$TIMESCALEDB_MAJOR-loader-postgresql-$PG_MAJOR=$TIMESCALEDB_VERSION* - -# Install PostGIS extension in both versions of Postgres -RUN apt update -RUN \ - POSTGIS_MAJOR=$(echo $POSTGIS_VERSION | cut -c1) && \ - apt install -y --no-install-recommends \ - postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR=$POSTGIS_VERSION* \ - postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR-scripts \ - && rm -rf /var/lib/apt/lists/* \ No newline at end of file From 779779ef569a30caddf6ff7d66caaf0d6902e289 Mon Sep 17 00:00:00 2001 From: Vincent Date: Thu, 23 May 2024 10:53:07 +0200 Subject: [PATCH 04/12] Add timescaledb to shared_preload_libraries in image --- infra/docker/database/pg13_16.Dockerfile | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/infra/docker/database/pg13_16.Dockerfile b/infra/docker/database/pg13_16.Dockerfile index cedebdb38c..57d5b278e4 100644 --- a/infra/docker/database/pg13_16.Dockerfile +++ b/infra/docker/database/pg13_16.Dockerfile @@ -26,6 +26,13 @@ RUN \ timescaledb-$TIMESCALEDB_MAJOR-loader-postgresql-$PG_MAJOR=$TIMESCALEDB_VERSION* && \ rm -rf /var/lib/apt/lists/* +RUN for file in $(find /usr/share/postgresql -name 'postgresql.conf.sample'); do \ + # We want timescaledb to be loaded in this image by every created cluster + sed -r -i "s/[#]*\s*(shared_preload_libraries)\s*=\s*'(.*)'/\1 = 'timescaledb,\2'/;s/,'/'/" $file \ + # We need to listen on all interfaces, otherwise PostgreSQL is not accessible + && echo "listen_addresses = '*'" >> $file; \ + done + # Install PostGIS extension RUN \ POSTGIS_MAJOR=$(echo $POSTGIS_VERSION | cut -c1) && \ From 4a1516cf8b81150d3425095b890d6ac11e3370f1 Mon Sep 17 00:00:00 2001 From: Vincent Date: Thu, 23 May 2024 13:19:19 +0200 Subject: [PATCH 05/12] Lint Dockerfile --- infra/docker/database/pg13_16.Dockerfile | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/infra/docker/database/pg13_16.Dockerfile b/infra/docker/database/pg13_16.Dockerfile index 57d5b278e4..bb531180ec 100644 --- a/infra/docker/database/pg13_16.Dockerfile +++ b/infra/docker/database/pg13_16.Dockerfile @@ -2,7 +2,7 @@ ARG PG_MAJOR ARG TIMESCALEDB_VERSION ARG POSTGIS_VERSION -FROM postgres:$PG_MAJOR-bookworm +FROM postgres:"$PG_MAJOR"-bookworm ARG PG_MAJOR ARG TIMESCALEDB_VERSION ARG POSTGIS_VERSION @@ -11,7 +11,8 @@ RUN \ apt-get update && \ apt-get install -y \ wget \ - lsb-release + lsb-release && \ + rm -rf /var/lib/apt/lists/* # Install TimescaleDB extension RUN echo "deb https://packagecloud.io/timescale/timescaledb/debian/ $(lsb_release -c -s) main" @@ -19,25 +20,25 @@ RUN echo "deb https://packagecloud.io/timescale/timescaledb/debian/ $(lsb_releas RUN wget --quiet -O - https://packagecloud.io/timescale/timescaledb/gpgkey | apt-key add - RUN \ - TIMESCALEDB_MAJOR=$(echo $TIMESCALEDB_VERSION | cut -c1) && \ + TIMESCALEDB_MAJOR=$(echo "$TIMESCALEDB_VERSION" | cut -c1) && \ apt-get update && \ apt-get install -y \ - timescaledb-$TIMESCALEDB_MAJOR-postgresql-$PG_MAJOR=$TIMESCALEDB_VERSION* \ - timescaledb-$TIMESCALEDB_MAJOR-loader-postgresql-$PG_MAJOR=$TIMESCALEDB_VERSION* && \ + timescaledb-"$TIMESCALEDB_MAJOR"-postgresql-"$PG_MAJOR"="$TIMESCALEDB_VERSION"* \ + timescaledb-"$TIMESCALEDB_MAJOR"-loader-postgresql-"$PG_MAJOR"="$TIMESCALEDB_VERSION"* && \ rm -rf /var/lib/apt/lists/* RUN for file in $(find /usr/share/postgresql -name 'postgresql.conf.sample'); do \ # We want timescaledb to be loaded in this image by every created cluster - sed -r -i "s/[#]*\s*(shared_preload_libraries)\s*=\s*'(.*)'/\1 = 'timescaledb,\2'/;s/,'/'/" $file \ + sed -r -i "s/[#]*\s*(shared_preload_libraries)\s*=\s*'(.*)'/\1 = 'timescaledb,\2'/;s/,'/'/" "$file" \ # We need to listen on all interfaces, otherwise PostgreSQL is not accessible - && echo "listen_addresses = '*'" >> $file; \ + && echo "listen_addresses = '*'" >> "$file"; \ done # Install PostGIS extension RUN \ - POSTGIS_MAJOR=$(echo $POSTGIS_VERSION | cut -c1) && \ + POSTGIS_MAJOR=$(echo "$POSTGIS_VERSION" | cut -c1) && \ apt update && \ apt install -y --no-install-recommends \ - postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR=$POSTGIS_VERSION* \ - postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR-scripts && \ + postgresql-"$PG_MAJOR"-postgis-"$POSTGIS_MAJOR"="$POSTGIS_VERSION"* \ + postgresql-"$PG_MAJOR"-postgis-"$POSTGIS_MAJOR"-scripts && \ rm -rf /var/lib/apt/lists/* \ No newline at end of file From 139da99fb52d890600f253e9b4b0cf8e62cbb48c Mon Sep 17 00:00:00 2001 From: Vincent Date: Thu, 23 May 2024 13:38:24 +0200 Subject: [PATCH 06/12] Update cicd push paths --- .github/workflows/cicd.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index e11353fe43..178cfc1fc3 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -14,8 +14,10 @@ on: - "**.tsx" - "**.yml" - ".github/workflows/cicd.yml" + - ".github/workflows/database.yml" - "infra/configurations/**.properties" - "infra/docker/app/**" + - "infra/docker/database/**" - "**.sql" - "backend/pom.xml" - "backend/build.gradle.kts" From 670bfa5aca95ba4c8f5a59176adc5437f1473d07 Mon Sep 17 00:00:00 2001 From: Vincent Date: Thu, 23 May 2024 15:19:37 +0200 Subject: [PATCH 07/12] Add timescaledb init script --- .../000_install_timescaledb.sh | 56 ++++++++++++ .../001_timescaledb_tune.sh | 88 +++++++++++++++++++ infra/docker/database/pg13_16.Dockerfile | 2 + 3 files changed, 146 insertions(+) create mode 100644 infra/docker/database/docker-entrypoint-initdb.d/000_install_timescaledb.sh create mode 100644 infra/docker/database/docker-entrypoint-initdb.d/001_timescaledb_tune.sh diff --git a/infra/docker/database/docker-entrypoint-initdb.d/000_install_timescaledb.sh b/infra/docker/database/docker-entrypoint-initdb.d/000_install_timescaledb.sh new file mode 100644 index 0000000000..e71ffc8e49 --- /dev/null +++ b/infra/docker/database/docker-entrypoint-initdb.d/000_install_timescaledb.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +create_sql=`mktemp` + +# Checks to support bitnami image with same scripts so they stay in sync +if [ ! -z "${BITNAMI_APP_NAME:-}" ]; then + if [ -z "${POSTGRES_USER:-}" ]; then + POSTGRES_USER=postgres + fi + + if [ -z "${POSTGRESQL_PASSWORD:-}" ]; then + POSTGRESQL_PASSWORD=${POSTGRESQL_POSTGRES_PASSWORD} + fi + + if [ -z "${POSTGRES_DB:-}" ]; then + POSTGRES_DB=${POSTGRESQL_DATABASE} + fi + + if [ -z "${PGDATA:-}" ]; then + PGDATA=${POSTGRESQL_DATA_DIR} + fi +fi + +if [ -z "${POSTGRESQL_CONF_DIR:-}" ]; then + POSTGRESQL_CONF_DIR=${PGDATA} +fi + +cat <${create_sql} +CREATE EXTENSION IF NOT EXISTS timescaledb CASCADE; +EOF + +TS_TELEMETRY='basic' +if [ "${TIMESCALEDB_TELEMETRY:-}" == "off" ]; then + TS_TELEMETRY='off' + + # We delete the job as well to ensure that we do not spam the + # log with other messages related to the Telemetry job. + cat <>${create_sql} +SELECT alter_job(1,scheduled:=false); +EOF +fi + +echo "timescaledb.telemetry_level=${TS_TELEMETRY}" >> ${POSTGRESQL_CONF_DIR}/postgresql.conf + +if [ -z "${POSTGRESQL_PASSWORD:-}" ]; then + POSTGRESQL_PASSWORD=${POSTGRES_PASSWORD:-} +fi +export PGPASSWORD="$POSTGRESQL_PASSWORD" + +# create extension timescaledb in initial databases +psql -U "${POSTGRES_USER}" postgres -f ${create_sql} +psql -U "${POSTGRES_USER}" template1 -f ${create_sql} + +if [ "${POSTGRES_DB:-postgres}" != 'postgres' ]; then + psql -U "${POSTGRES_USER}" "${POSTGRES_DB}" -f ${create_sql} +fi diff --git a/infra/docker/database/docker-entrypoint-initdb.d/001_timescaledb_tune.sh b/infra/docker/database/docker-entrypoint-initdb.d/001_timescaledb_tune.sh new file mode 100644 index 0000000000..707439cf2c --- /dev/null +++ b/infra/docker/database/docker-entrypoint-initdb.d/001_timescaledb_tune.sh @@ -0,0 +1,88 @@ +#!/bin/bash + +NO_TS_TUNE=${NO_TS_TUNE:-""} +TS_TUNE_MEMORY=${TS_TUNE_MEMORY:-""} +TS_TUNE_NUM_CPUS=${TS_TUNE_NUM_CPUS:-""} +TS_TUNE_MAX_CONNS=${TS_TUNE_MAX_CONNS:-""} +TS_TUNE_MAX_BG_WORKERS=${TS_TUNE_MAX_BG_WORKERS:-""} + +if [ ! -z "${NO_TS_TUNE:-}" ]; then + # The user has explicitly requested not to run timescaledb-tune; exit this script + exit 0 +fi + + +if [ -z "${POSTGRESQL_CONF_DIR:-}" ]; then + POSTGRESQL_CONF_DIR=${PGDATA} +fi + +if [ -z "${TS_TUNE_MEMORY:-}" ]; then + # See if we can get the container's total allocated memory from the cgroups metadata + if [ -f /sys/fs/cgroup/memory/memory.limit_in_bytes ]; then + TS_TUNE_MEMORY=$(cat /sys/fs/cgroup/memory/memory.limit_in_bytes) + + if [ "${TS_TUNE_MEMORY}" = "18446744073709551615" ]; then + # Bash seems to error out for numbers greater than signed 64-bit, + # so if the value of limit_in_bytes is the 64-bit UNSIGNED max value + # we should just bail out and hope timescaledb-tune can figure this + # out. If we don't, the next comparison is likely going to fail + # or it might store a negative value which will crash later. + TS_TUNE_MEMORY="" + fi + + FREE_KB=$(grep MemTotal: /proc/meminfo | awk '{print $2}') + FREE_BYTES=$(( ${FREE_KB} * 1024 )) + if [ ${TS_TUNE_MEMORY} -gt ${FREE_BYTES} ]; then + # Something weird is going on if the cgroups memory limit exceeds the total available + # amount of system memory reported by "free", which is the total amount of memory available on the host. + # Most likely, it is this issue: https://github.com/moby/moby/issues/18087 (if no limit is + # set, the max limit is set to the max 64 bit integer). In this case, we just leave + # TS_TUNE_MEMORY blank and let timescaledb-tune derive the memory itself using syscalls. + TS_TUNE_MEMORY="" + else + # Convert the bytes to MB so it plays nicely with timescaledb-tune + TS_TUNE_MEMORY="$(echo ${TS_TUNE_MEMORY} | awk '{print int($1 / 1024 / 1024)}')MB" + fi + fi +fi + +if [ -z "${TS_TUNE_NUM_CPUS:-}" ]; then + # See if we can get the container's available CPUs from the cgroups metadata + if [ -f /sys/fs/cgroup/cpuset/cpuset.cpus ]; then + TS_TUNE_NUM_CPUS=$(cat /sys/fs/cgroup/cpuset/cpuset.cpus) + if [[ ${TS_TUNE_NUM_CPUS} == *-* ]]; then + # The CPU limits have been defined as a range (e.g., 0-3 for 4 CPUs). Subtract them and add 1 + # to convert the range to the number of CPUs. + TS_TUNE_NUM_CPUS=$(echo ${TS_TUNE_NUM_CPUS} | tr "-" " " | awk '{print ($2 - $1) + 1}') + elif [[ ${TS_TUNE_NUM_CPUS} == *,* ]]; then + # The CPU limits have been defined as a comma separated list (e.g., 0,1,2,3 for 4 CPUs). Count each CPU + TS_TUNE_NUM_CPUS=$(echo ${TS_TUNE_NUM_CPUS} | tr "," "\n" | wc -l) + elif [ $(echo -n ${TS_TUNE_NUM_CPUS} | wc -c) -eq 1 ]; then + # The CPU limit has been defined as a single numbered CPU. In this case the CPU limit is 1 + # regardless of what that number is + TS_TUNE_NUM_CPUS=1 + fi + fi +fi + +if [ ! -z "${TS_TUNE_MEMORY:-}" ]; then + TS_TUNE_MEMORY_FLAGS=--memory="${TS_TUNE_MEMORY}" +fi + +if [ ! -z "${TS_TUNE_NUM_CPUS:-}" ]; then + TS_TUNE_NUM_CPUS_FLAGS=--cpus=${TS_TUNE_NUM_CPUS} +fi + +if [ ! -z "${TS_TUNE_MAX_CONNS:-}" ]; then + TS_TUNE_MAX_CONNS_FLAGS=--max-conns=${TS_TUNE_MAX_CONNS} +fi + +if [ ! -z "${TS_TUNE_MAX_BG_WORKERS:-}" ]; then + TS_TUNE_MAX_BG_WORKERS_FLAGS=--max-bg-workers=${TS_TUNE_MAX_BG_WORKERS} +fi + +if [ ! -z "${PG_MAJOR}" ]; then + TS_TUNE_PG_VERSION=--pg-version=${PG_MAJOR} +fi + +/usr/local/bin/timescaledb-tune --quiet --yes --conf-path="${POSTGRESQL_CONF_DIR}/postgresql.conf" ${TS_TUNE_MEMORY_FLAGS} ${TS_TUNE_NUM_CPUS_FLAGS} ${TS_TUNE_MAX_CONNS_FLAGS} ${TS_TUNE_MAX_BG_WORKERS_FLAGS} ${TS_TUNE_PG_VERSION} \ No newline at end of file diff --git a/infra/docker/database/pg13_16.Dockerfile b/infra/docker/database/pg13_16.Dockerfile index bb531180ec..ce051982a2 100644 --- a/infra/docker/database/pg13_16.Dockerfile +++ b/infra/docker/database/pg13_16.Dockerfile @@ -7,6 +7,8 @@ ARG PG_MAJOR ARG TIMESCALEDB_VERSION ARG POSTGIS_VERSION +COPY infra/docker/database/docker-entrypoint-initdb.d/* /docker-entrypoint-initdb.d/ + RUN \ apt-get update && \ apt-get install -y \ From 2a39074513f9a38bca7eaac75b3996a2492f3885 Mon Sep 17 00:00:00 2001 From: Vincent Date: Thu, 23 May 2024 15:32:57 +0200 Subject: [PATCH 08/12] Install timescale tools in image --- infra/docker/database/pg13_16.Dockerfile | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/infra/docker/database/pg13_16.Dockerfile b/infra/docker/database/pg13_16.Dockerfile index ce051982a2..77dbd3ca06 100644 --- a/infra/docker/database/pg13_16.Dockerfile +++ b/infra/docker/database/pg13_16.Dockerfile @@ -2,12 +2,28 @@ ARG PG_MAJOR ARG TIMESCALEDB_VERSION ARG POSTGIS_VERSION +############################ +# Build tools binaries in separate image +############################ +ARG GO_VERSION=1.18.7 +FROM golang:${GO_VERSION}-alpine AS tools + +ENV TOOLS_VERSION 0.8.1 + +RUN apk update && apk add --no-cache git gcc musl-dev \ + && go install github.com/timescale/timescaledb-tune/cmd/timescaledb-tune@latest \ + && go install github.com/timescale/timescaledb-parallel-copy/cmd/timescaledb-parallel-copy@latest + +############################ +# Now build image and copy in tools +############################ FROM postgres:"$PG_MAJOR"-bookworm ARG PG_MAJOR ARG TIMESCALEDB_VERSION ARG POSTGIS_VERSION COPY infra/docker/database/docker-entrypoint-initdb.d/* /docker-entrypoint-initdb.d/ +COPY --from=tools /go/bin/* /usr/local/bin/ RUN \ apt-get update && \ From 6319f897965858ffbdbe90bd44a2907a8fc88254 Mon Sep 17 00:00:00 2001 From: Vincent Date: Thu, 23 May 2024 16:09:45 +0200 Subject: [PATCH 09/12] Rename make command --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e7c04517ee..6c07c8d72e 100644 --- a/Makefile +++ b/Makefile @@ -80,7 +80,7 @@ upgrade-postgres-11-to-13: -v $(PG_13_DATA_VOLUME_NAME):/var/lib/postgresql/13/data \ ghcr.io/mtes-mct/monitorfish/monitorfish-database-upgrade:pg11_to_pg13-ts2.3.1-postgis3.3.4 -O "-c timescaledb.restoring='on'" -O "-c shared_preload_libraries=timescaledb"; -upgrade-postgres-13-to-16-dev: +upgrade-postgres-13-to-16: docker run --rm \ -v $(PG_13_DATA_VOLUME_NAME):/var/lib/postgresql/13/data \ -v $(PG_16_DATA_VOLUME_NAME):/var/lib/postgresql/16/data \ From da435122d817a3dfb23585c1f10090cd21c3de78 Mon Sep 17 00:00:00 2001 From: Vincent Date: Thu, 23 May 2024 17:48:16 +0200 Subject: [PATCH 10/12] Remove version from docker compose files --- docker-compose.yml | 1 - infra/remote/docker-compose.yml | 1 - 2 files changed, 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 6715722ba6..ceda7d950f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,3 @@ -version: "3" services: db: image: ghcr.io/mtes-mct/monitorfish/monitorfish-database:pg16-ts2.14.2-postgis3.4.2 diff --git a/infra/remote/docker-compose.yml b/infra/remote/docker-compose.yml index 4fda5f08fe..3775f4db41 100644 --- a/infra/remote/docker-compose.yml +++ b/infra/remote/docker-compose.yml @@ -1,4 +1,3 @@ -version: "3" services: db: image: ghcr.io/mtes-mct/monitorfish/monitorfish-database:$DATABASE_VERSION From f6cf5828b3d6ad5d4e8fefcc8aaf6436deae4a05 Mon Sep 17 00:00:00 2001 From: Vincent Date: Thu, 23 May 2024 18:26:49 +0200 Subject: [PATCH 11/12] Add timescaledb to shared_preload_libraries in upgrade image --- infra/docker/database/upgrade_pg.Dockerfile | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/infra/docker/database/upgrade_pg.Dockerfile b/infra/docker/database/upgrade_pg.Dockerfile index 10e6078cdc..d423cf13ff 100644 --- a/infra/docker/database/upgrade_pg.Dockerfile +++ b/infra/docker/database/upgrade_pg.Dockerfile @@ -52,6 +52,13 @@ RUN \ apt-get install -y timescaledb-$TIMESCALEDB_MAJOR-postgresql-$FROM_PG_MAJOR=$TIMESCALEDB_VERSION* timescaledb-$TIMESCALEDB_MAJOR-loader-postgresql-$FROM_PG_MAJOR=$TIMESCALEDB_VERSION* && \ apt-get install -y timescaledb-$TIMESCALEDB_MAJOR-postgresql-$TO_PG_MAJOR=$TIMESCALEDB_VERSION* timescaledb-$TIMESCALEDB_MAJOR-loader-postgresql-$TO_PG_MAJOR=$TIMESCALEDB_VERSION* +RUN for file in $(find /usr/share/postgresql -name 'postgresql.conf.sample'); do \ + # We want timescaledb to be loaded in this image by every created cluster + sed -r -i "s/[#]*\s*(shared_preload_libraries)\s*=\s*'(.*)'/\1 = 'timescaledb,\2'/;s/,'/'/" "$file" \ + # We need to listen on all interfaces, otherwise PostgreSQL is not accessible + && echo "listen_addresses = '*'" >> "$file"; \ +done + # Install PostGIS extension in both versions of Postgres RUN apt update RUN \ From 49760934f4c5e84c445459be79125c3a0eaf92f9 Mon Sep 17 00:00:00 2001 From: Vincent Date: Thu, 23 May 2024 18:56:31 +0200 Subject: [PATCH 12/12] Remove duplicate config line from database images --- infra/docker/database/pg13_16.Dockerfile | 2 -- infra/docker/database/upgrade_pg.Dockerfile | 2 -- 2 files changed, 4 deletions(-) diff --git a/infra/docker/database/pg13_16.Dockerfile b/infra/docker/database/pg13_16.Dockerfile index 77dbd3ca06..49ffa22230 100644 --- a/infra/docker/database/pg13_16.Dockerfile +++ b/infra/docker/database/pg13_16.Dockerfile @@ -48,8 +48,6 @@ RUN \ RUN for file in $(find /usr/share/postgresql -name 'postgresql.conf.sample'); do \ # We want timescaledb to be loaded in this image by every created cluster sed -r -i "s/[#]*\s*(shared_preload_libraries)\s*=\s*'(.*)'/\1 = 'timescaledb,\2'/;s/,'/'/" "$file" \ - # We need to listen on all interfaces, otherwise PostgreSQL is not accessible - && echo "listen_addresses = '*'" >> "$file"; \ done # Install PostGIS extension diff --git a/infra/docker/database/upgrade_pg.Dockerfile b/infra/docker/database/upgrade_pg.Dockerfile index d423cf13ff..fedfc6b14e 100644 --- a/infra/docker/database/upgrade_pg.Dockerfile +++ b/infra/docker/database/upgrade_pg.Dockerfile @@ -55,8 +55,6 @@ RUN \ RUN for file in $(find /usr/share/postgresql -name 'postgresql.conf.sample'); do \ # We want timescaledb to be loaded in this image by every created cluster sed -r -i "s/[#]*\s*(shared_preload_libraries)\s*=\s*'(.*)'/\1 = 'timescaledb,\2'/;s/,'/'/" "$file" \ - # We need to listen on all interfaces, otherwise PostgreSQL is not accessible - && echo "listen_addresses = '*'" >> "$file"; \ done # Install PostGIS extension in both versions of Postgres