From 1ef83f379544a5416ed777395c3d8a0a919ae8a2 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Thu, 21 Mar 2024 01:41:20 +0000
Subject: [PATCH 01/35] PMM-12899 Use module and build cache

---
 .github/workflows/main.yml                   |   8 +-
 build/docker/client/Dockerfile               |  18 +-
 build/docker/server/Dockerfile.el9           |  14 +-
 build/docker/server/create_users.sh          |  29 ---
 build/local/README.md                        |  34 ++++
 build/local/build.sh                         | 110 ++++++++++++
 build/local/update.sh                        | 101 +++++++++++
 build/packages/rpm/server/SPECS/grafana.spec |   4 +-
 build/scripts/build-client-binary            |  70 +++++---
 build/scripts/build-client-docker            |  14 +-
 build/scripts/build-client-packages          | 104 +++++------
 build/scripts/build-client-rpm               |  16 +-
 build/scripts/build-client-source            |  39 +++--
 build/scripts/build-client-srpm              |  17 +-
 build/scripts/build-server-rpm               | 175 ++++++++++---------
 build/scripts/build-submodules               |  52 +++---
 build/scripts/vars                           |   6 +-
 17 files changed, 519 insertions(+), 292 deletions(-)
 delete mode 100644 build/docker/server/create_users.sh
 create mode 100644 build/local/README.md
 create mode 100755 build/local/build.sh
 create mode 100755 build/local/update.sh

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 9cb7d3927a..019b6bd5d8 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -142,8 +142,8 @@ jobs:
     env:
       SLACK_WEBHOOK: ${{ secrets.SLACK_TOKEN_PMM_CI }}
       SLACK_CHANNEL: "pmm-ci"
-      SLACK_USERNAME: "PR pipelines bot"
-      SLACK_ICON_EMOJI: ":chestnut:"
+      SLACK_USERNAME: "Build bot"
+      SLACK_ICON_EMOJI: ":octocat:"
       SLACK_COLOR: "#00FF00"
       SLACK_TITLE: "Finished ${{ github.event.repository.name }} workflow"
       SLACK_MESSAGE: "${{ github.event.inputs.repo || github.repository }}:${{ github.event.inputs.branch || github.head_ref }}"
@@ -160,8 +160,8 @@ jobs:
     env:
       SLACK_WEBHOOK: ${{ secrets.SLACK_TOKEN_PMM_CI }}
       SLACK_CHANNEL: "pmm-ci"
-      SLACK_USERNAME: "PR pipelines bot"
-      SLACK_ICON_EMOJI: ":chestnut:"
+      SLACK_USERNAME: "Build bot"
+      SLACK_ICON_EMOJI: ":octocat:"
       SLACK_COLOR: "#FF0000"
       SLACK_TITLE: "Finished ${{ github.event.repository.name }} workflow"
       SLACK_MESSAGE: "Workflow failed: ${{ github.event.inputs.repo || github.repository }}:${{ github.event.inputs.branch || github.head_ref }}"
diff --git a/build/docker/client/Dockerfile b/build/docker/client/Dockerfile
index 2c03161268..6154ddfd08 100644
--- a/build/docker/client/Dockerfile
+++ b/build/docker/client/Dockerfile
@@ -5,19 +5,13 @@ RUN microdnf install shadow-utils jq -y
 RUN groupadd -g 1002 pmm-agent && \
     useradd -u 1002 -r -g pmm-agent -s /sbin/nologin \
             -d /usr/local/percona/pmm \
-            -c "PMM 2.X Client User" pmm-agent
+            -c "PMM 3.x Client User" pmm-agent
 
 FROM redhat/ubi9-micro
 
 ARG VERSION
 ARG BUILD_DATE
 
-LABEL org.opencontainers.image.created ${BUILD_DATE}
-LABEL org.opencontainers.image.licenses Apache-2.0
-LABEL org.opencontainers.image.title Percona Monitoring and Management Client
-LABEL org.opencontainers.image.vendor Percona
-LABEL org.opencontainers.image.version ${VERSION}
-
 COPY LICENSE /licenses/
 COPY --from=users --chown=0:0 /etc/passwd /etc/passwd
 COPY --from=users --chown=0:0 /etc/group /etc/group
@@ -32,10 +26,18 @@ RUN install -d -o pmm-agent -g pmm-agent -m 0775 /usr/local/percona/pmm
 ADD pmm-client.tar.gz /tmp/
 
 RUN cd /tmp/pmm-client-* \
-    && env PMM_USER=pmm-agent PMM_GROUP=root ./install_tarball \
+    && PMM_USER=pmm-agent PMM_GROUP=root ./install_tarball \
     && cd /tmp \
     && rm -rf /tmp/pmm-client*
 
+LABEL org.opencontainers.image.created ${BUILD_DATE}
+LABEL org.opencontainers.image.licenses Apache-2.0
+LABEL org.opencontainers.image.title Percona Monitoring and Management Client
+LABEL org.opencontainers.image.vendor Percona
+LABEL org.opencontainers.image.version ${VERSION}
+LABEL org.opencontainers.image.source https://github.com/percona/pmm/blob/main/build/docker/client/Dockerfile
+LABEL org.opencontainers.image.docs https://docs.percona.com/percona-monitoring-and-management/setting-up/client/index.html#docker
+
 USER pmm-agent
 WORKDIR /usr/local/percona/pmm/
 ENV PATH /usr/local/percona/pmm/bin/:$PATH
diff --git a/build/docker/server/Dockerfile.el9 b/build/docker/server/Dockerfile.el9
index 35dc8ac340..28dae9192e 100644
--- a/build/docker/server/Dockerfile.el9
+++ b/build/docker/server/Dockerfile.el9
@@ -8,12 +8,6 @@ ENV LC_ALL=en_US.utf8
 ENV GF_PLUGIN_DIR=/srv/grafana/plugins
 ENV PS1="[\u@\h \W] # "
 
-LABEL org.opencontainers.image.created ${BUILD_DATE}
-LABEL org.opencontainers.image.licenses AGPL-3.0
-LABEL org.opencontainers.image.title Percona Monitoring and Management
-LABEL org.opencontainers.image.vendor Percona LLC
-LABEL org.opencontainers.image.version ${VERSION}
-
 EXPOSE 8080 8443
 
 WORKDIR /opt
@@ -40,6 +34,14 @@ RUN install -T -p -m 644 /opt/ansible/ansible.cfg /etc/ansible/ansible.cfg && \
     ansible-playbook -vvv -i 'localhost,' -c local /opt/ansible/pmm-docker/update.yml && \
     ansible-playbook -vvv -i 'localhost,' -c local /opt/ansible/pmm/post-build-actions.yml
 
+LABEL org.opencontainers.image.created ${BUILD_DATE}
+LABEL org.opencontainers.image.licenses AGPL-3.0
+LABEL org.opencontainers.image.title Percona Monitoring and Management
+LABEL org.opencontainers.image.vendor Percona LLC
+LABEL org.opencontainers.image.version ${VERSION}
+LABEL org.opencontainers.image.source https://github.com/percona/pmm/blob/main/build/docker/server/Dockerfile.el9
+LABEL org.opencontainers.image.docs https://docs.percona.com/percona-monitoring-and-management/setting-up/server/docker.html
+
 USER pmm
 
 HEALTHCHECK --interval=3s --timeout=2s --start-period=10s --retries=3 CMD curl -sf http://127.0.0.1:8080/v1/readyz
diff --git a/build/docker/server/create_users.sh b/build/docker/server/create_users.sh
deleted file mode 100644
index e37eae3bf8..0000000000
--- a/build/docker/server/create_users.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/bash
-
-users=(
-  "pmm:1000:/bin/false:/home/pmm:pmm"
-  "nginx:1001:/sbin/nologin:/var/cache/nginx:nginx"
-)
-
-for user in "${users[@]}"; do
-  IFS=: read -r name uid shell home_dir group <<< "$user"
-  group_id="$uid"
-
-  # Check if user already exists
-  if id "$name" >/dev/null 2>&1; then
-    echo "User $name already exists"
-    continue
-  fi
-
-  # Create user with home directory if it doesn't exist
-  if [ ! -d "$home_dir" ]; then
-    mkdir -p "$home_dir"
-  fi
-
-  # Create user with specified UID, GID, and shell
-  groupadd -o -g "$group_id" "$group"
-  useradd -o -u "$uid" -g "$group" -G "$group" -s "$shell" -d "$home_dir" -c "$name" -m "$name"
-  chown "$uid:$group_id" "$home_dir"
-
-done
-
diff --git a/build/local/README.md b/build/local/README.md
new file mode 100644
index 0000000000..1517e9da3f
--- /dev/null
+++ b/build/local/README.md
@@ -0,0 +1,34 @@
+# Local builds
+
+This directory contains a set of scripts aimed at providing a simple way to build PMM locally.
+
+## Background
+
+Historically, PMM used to be built using Jenkins. This worked well for the team, but not for the community. The learning curve was, and still is, rather steep, and it is hard for folks, even internally, to contribute to.
+
+Therefore, we decided to make it possible to build PMM locally. This is a work in progress, but we are definitely committed to making it easier to build PMM locally.
+
+The build process is mostly based on bash scripts, which control the build flow. This was an intentional decision early on to make the build process easy to understand and contribute to. Apart from bash and a few other well-known utilitites like `curl` or `make`, it also uses Docker for environment isolation and caching.
+
+The build process is designed to be run on a Linux host. We believe it can be run on other flavors of Linux, including MacOS, with little to no modification (TBC).
+
+## Prerequisites
+
+Below is a list of prerequisites that are required to build PMM locally.
+
+- OS: Linux (tested on Oracle Linux 9.3, CentOS 7, Ubuntu 22.04.3 LTS)
+- Docker: 25.0.2+ (tested on 25.0.2)
+- Docker Compose Plugin: 2.24.7+ (tested on 2.24.7)
+- make: 4.3+ (tested on 4.3)
+- bash (GNU): 5.1+ (tested on 5.1)
+- git: 2.34.1+ (tested on 2.34.1)
+- curl: 7.81.0+ (tested on 7.81.0)
+- yq: 4.42.0+ (tested on 4.42.1)
+- jq: 1.6+ (tested on 1.6)
+
+## Build Steps
+
+1. Install the prerequisites
+2. Clone the PMM repository
+3. Change to the `build/local` directory
+4. Run the `build.sh` script
diff --git a/build/local/build.sh b/build/local/build.sh
new file mode 100755
index 0000000000..f27ec569a8
--- /dev/null
+++ b/build/local/build.sh
@@ -0,0 +1,110 @@
+#!/bin/bash -e
+
+# Update submodules and PR branches
+/bin/bash $(dirname $0)/update.sh
+
+get_branch_name() {
+  local module="${1:-}"
+  local branch_name
+  local path
+
+  path=$(git config -f .gitmodules submodule.${module}.path)
+  cd "$path" || exit 1
+  branch_name=$(git branch --show-current)
+  cd - > /dev/null
+  echo $branch_name
+}
+
+# Define global variables
+pmm_commit=$(git submodule status | grep 'sources/pmm/src' | awk -F ' ' '{print $1}')
+echo $pmm_commit > apiCommitSha
+pmm_branch=$(get_branch_name pmm)
+echo $pmm_branch > apiBranch
+pmm_url=$(git config -f .gitmodules submodule.pmm.url)
+echo $pmm_url > apiURL
+pmm_qa_branch=$(get_branch_name pmm-qa)
+echo $pmm_qa_branch > pmmQABranch
+pmm_qa_commit=$(git submodule status | grep 'pmm-qa' | awk -F ' ' '{print $1}')
+echo $pmm_qa_commit > pmmQACommitSha
+pmm_ui_tests_branch=$(get_branch_name pmm-ui-tests)
+echo $pmm_ui_tests_branch > pmmUITestBranch
+pmm_ui_tests_commit=$(git submodule status | grep 'pmm-ui-tests' | awk -F ' ' '{print $1}')
+echo $pmm_ui_tests_commit > pmmUITestsCommitSha
+fb_commit_sha=$(git rev-parse HEAD)
+echo $fb_commit_sha > fbCommitSha
+
+# We use a special docker image to build various PMM artifacts - `perconalab/rpmbuild:ol9`.
+# Important: the docker container's user need to be able to write to these directories.
+# The docker container's user is `builder` with uid 1000 and gid 1000. You need to make sure
+# that the directories we create on the host are owned by a user with same uid and gid.
+
+# Create cache directories. 
+test -d "${root_dir}/go-path" || mkdir -p "go-path"
+test -d "${root_dir}/go-build" || mkdir -p "go-build"
+
+PATH_TO_SCRIPTS="sources/pmm/src/github.com/percona/pmm/build/scripts"
+export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:ol9 
+
+# Local reference test environment
+# CPU: 4 cores
+# RAM: 16GB
+# OS: Ubuntu 22.04.1 LTS
+
+# Build client source: 4m39s from scratch, 0m27s using cache
+"$PATH_TO_SCRIPTS/build-client-source"
+
+# Build client binary: ??? from scratch, 0m20s using cache
+"$PATH_TO_SCRIPTS/build-client-binary"
+
+# Building client source rpm takes 13s (caching does not apply)
+"$PATH_TO_SCRIPTS/build-client-srpm"
+
+# Building client rpm takes 1m40s
+"$PATH_TO_SCRIPTS/build-client-rpm"
+
+# Building client docker image takes 17s
+GIT_COMMIT=$(git rev-parse HEAD | head -c 8)
+export DOCKER_CLIENT_TAG=local/pmm-client:${GIT_COMMIT}
+"$PATH_TO_SCRIPTS/build-client-docker"
+
+# Building PMM CLient locally (non-CI, i.e. non-Jenkins)
+# total time: 6m26s - build from scratch, no initial cache
+# total time: 2m49s - subsequent build, using cache from prior builds
+
+
+# Building PMM CLient in a CI environment, i.e. Jenkins running on AWS
+# total time: 8m45s - build from scratch, no initial cache
+# total time: ??? - subsequent build, using cache from prior builds
+
+export RPM_EPOCH=1
+export RPMBUILD_DIST="el9"
+"$PATH_TO_SCRIPTS/build-server-rpm" percona-dashboards grafana-dashboards
+"$PATH_TO_SCRIPTS/build-server-rpm" pmm-managed pmm
+"$PATH_TO_SCRIPTS/build-server-rpm" percona-qan-api2 pmm
+"$PATH_TO_SCRIPTS/build-server-rpm" pmm-update pmm
+"$PATH_TO_SCRIPTS/build-server-rpm" dbaas-controller
+"$PATH_TO_SCRIPTS/build-server-rpm" dbaas-tools
+"$PATH_TO_SCRIPTS/build-server-rpm" pmm-dump
+"$PATH_TO_SCRIPTS/build-server-rpm" grafana-db-migrator
+"$PATH_TO_SCRIPTS/build-server-rpm" vmproxy pmm
+
+# 3rd-party
+"$PATH_TO_SCRIPTS/build-server-rpm" victoriametrics
+"$PATH_TO_SCRIPTS/build-server-rpm" alertmanager
+"$PATH_TO_SCRIPTS/build-server-rpm" grafana
+
+export DOCKER_TAG=local/pmm-server:${GIT_COMMIT}
+export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:ol9
+export RPMBUILD_DIST=el9
+export DOCKERFILE=Dockerfile.el9.local
+${PATH_TO_SCRIPTS}/build-server-docker
+
+# Clean up temporary files
+rm -f apiBranch \
+  apiCommitSha \
+	apiURL \
+	fbCommitSha \
+	pmmQABranch \
+	pmmQACommitSha \
+	pmmUITestBranch \
+	pmmUITestsCommitSha
diff --git a/build/local/update.sh b/build/local/update.sh
new file mode 100755
index 0000000000..4922dd9bda
--- /dev/null
+++ b/build/local/update.sh
@@ -0,0 +1,101 @@
+#!/bin/bash -e
+
+needs-to-pull() {
+  local UPSTREAM=${1:-'@{u}'}
+  local LOCAL=$(git rev-parse @)
+  local BASE=$(git merge-base @ "$UPSTREAM")
+  local REMOTE=$(git rev-parse "$UPSTREAM")
+
+  if [ "$LOCAL" = "$REMOTE" ]; then
+    return 1 # false, we are up-to-date
+  fi
+
+  if [ "$LOCAL" = "$BASE" ]; then
+    return 0 # true, we are behind upstream
+  fi
+}
+
+rewind() {
+  local DIR="$1"
+  local BRANCH="$2"
+
+  cd "$DIR"
+  CURRENT=$(git branch --show-current)
+  git fetch
+
+  if [ "$CURRENT" != "$BRANCH" ]; then
+    echo "Currently on $CURRENT, checking out $BRANCH"
+    git checkout "$BRANCH"
+  fi
+
+  if needs-to-pull; then
+    git pull origin
+    echo "Submodule has pulled from upstream"
+    git logs -n 2
+    cd - >/dev/null
+    git add "$DIR"
+  else
+    cd - >/dev/null
+    echo "Submodule is up-to-date with upstream"
+  fi
+}
+
+check-files() {
+  local DIR="$1"
+
+  test -z "DIR" && exit 1
+
+  if [ -d "$DIR/sources" ] && [ -f "$DIR/ci-default.yml" ] && [ -f "$DIR/ci.yml" ]; then
+    return 0
+  fi
+
+  return 1
+}
+
+main() {
+  local DEPS=
+  local CURDIR="$PWD"
+  local DIR=pmm-submodules
+
+  # Thouroughly verify the presence of known files, otherwise bail out
+  if check-files "."; then # pwd is pmm-submodules
+    DIR="."
+  elif [ -d "$DIR" ]; then # pwd is outside pmm-submodules
+    if ! check-files "$DIR"; then
+      echo "FATAL: could not locate known files in ${PWD}/${DIR}"
+      exit 1
+    fi
+  else
+    echo "FATAL: could not locate known files in $PWD"
+    exit 1
+  fi
+
+  cd "$DIR"
+
+  # Join the dependencies from ci-default.yml and ci.yml
+  DEPS=$(yq -o=json eval-all '. as $item ireduce ({}; . *d $item )' ci-default.yml ci.yml | jq '.deps')
+
+  echo "This script rewinds submodule branches as per the joint config of 'ci-default.yml' and 'ci.yml'"
+
+  echo "$DEPS" | jq -c '.[]' | while read -r item; do
+    branch=$(echo "$item" | jq -r '.branch')
+    path=$(echo "$item" | jq -r '.path')
+    name=$(echo "$item" | jq -r '.name')
+    echo
+    echo "Rewinding submodule '$name' ..."
+    echo "path: ${path}, branch: ${branch}"
+
+    rewind "$path" "$branch"
+  done
+
+  echo
+  echo "Printing git status..."
+  git status --short
+  echo
+  echo "Printing git submodule status..."
+  git submodule status
+
+  cd "$CURDIR" > /dev/null
+}
+
+main
diff --git a/build/packages/rpm/server/SPECS/grafana.spec b/build/packages/rpm/server/SPECS/grafana.spec
index bd16ba3121..85976d18a1 100644
--- a/build/packages/rpm/server/SPECS/grafana.spec
+++ b/build/packages/rpm/server/SPECS/grafana.spec
@@ -17,8 +17,8 @@ Version:        %{grafana_version}
 Release:        %{rpm_release}
 Summary:        Grafana is an open source, feature rich metrics dashboard and graph editor
 License:        AGPLv3
-URL:            https://github.com/percona-platform/grafana
-Source0:        https://github.com/percona-platform/grafana/archive/%{commit}.tar.gz
+URL:            https://github.com/percona/grafana
+Source0:        https://github.com/percona/grafana/archive/%{commit}.tar.gz
 ExclusiveArch:  %{ix86} x86_64 %{arm}
 
 BuildRequires: fontconfig
diff --git a/build/scripts/build-client-binary b/build/scripts/build-client-binary
index 82d88c645d..295a5a5b02 100755
--- a/build/scripts/build-client-binary
+++ b/build/scripts/build-client-binary
@@ -5,6 +5,9 @@ set -o xtrace
 
 . $(dirname $0)/vars
 
+# Important: `bin_dir` and `binary_dir` are different directories. `bin_dir` defines the directory for binary files 
+# inside the container, while `binary_dir` defines the directory for binary files outside the container.
+
 extract_source_tarball() {
     if [ ! -d "${source_dir}" ]; then
         mkdir -p $(dirname ${source_dir})
@@ -18,58 +21,71 @@ gobuild_component() {
     local extract_path=${3:-"github.com/percona/$parent_component"}
     local component_path=$4
     local result_file=${5:-bin/$(basename ${component_path:-$extract_path})}
+    local volume_mounts="-v ${tmp_dir}:/home/builder/tmp"
+    local src_dir=/home/builder/tmp/source/pmm-client-${pmm_version}
+    local bin_dir=/home/builder/tmp/binary/pmm-client-${pmm_version}
+    local archive=$(ls ${source_dir} | grep -e "${parent_component}-\w*\.tar\.gz" | head -n1)
+    local build_command
+
+    if [ -d "${root_dir}/go-build" ]; then
+        volume_mounts+=" -v ${root_dir}/go-build:/home/builder/.cache/go-build"
+    fi
+    if [ -d "${root_dir}/go-path" ]; then
+        volume_mounts+=" -v ${root_dir}/go-path:/home/builder/go/pkg/mod"
+    fi
 
     if [ ${component} = 'azure_metrics_exporter' ]; then
         result_file=${result_file/_metrics/}
     fi
 
     if [ -x "${binary_dir}/${result_file}" ]; then
-        echo skip build
+        echo Skipping the build...
         return
     fi
-    if [ ${component} = 'pmm-admin' -o ${component} = 'pmm-agent' ]; then
+    if [[ ${component} =~ pmm-admin|pmm-agent ]]; then
         build_command="cd /tmp/go/src/${extract_path}/${component_path} && make release"
     elif [ ${component} = 'mongodb_exporter' ]; then
         build_command="cd /tmp/go/src/${extract_path} && make build"
-    elif [ ${component} = 'node_exporter' ] || [ ${component} = 'mysqld_exporter' ] || \
-         [ ${component} = 'proxysql_exporter' ] || [ ${component} = 'rds_exporter' ] || \
-         [ ${component} = 'postgres_exporter' ] || [ ${component} = 'azure_metrics_exporter' ]; then
+    elif [[ ${component} =~ node_exporter|mysqld_exporter|proxysql_exporter|rds_exporter|postgres_exporter|azure_metrics_exporter ]]; then
         build_command="cd /tmp/go/src/${extract_path} && make release"
     elif [ ${component} = 'percona-toolkit' ]; then
-        build_command="cd /tmp/go/src/${extract_path}/${component_path} && go build -o \${binary_dir}/${result_file} ."
+        build_command="cd /tmp/go/src/${extract_path}/${component_path} && go build -o ${bin_dir}/${result_file} ."
     else
-        build_command="go build -o \${binary_dir}/${result_file} ./src/${extract_path}/${component_path}"
+        build_command="cd /tmp/go/src/${extract_path}/${component_path} go build -o ${bin_dir}/${result_file} ."
     fi
-    docker run --rm -v ${tmp_dir}:/home/builder/tmp ${rpmbuild_docker_image} sh -c "
+
+    docker run --rm \
+      ${volume_mounts} \
+      ${rpmbuild_docker_image} sh -c "
         set -o errexit
         set -o xtrace
         export GOOS=${GOOS:-linux}
         export GOARCH=${GOARCH:-amd64}
         export CGO_ENABLED=0
         export GO111MODULE=auto
+        export GOPATH=/tmp/go
+        export GOMODCACHE=/home/builder/go/pkg/mod
 
-        # for 'make release'
+        # Setting variable for 'make release'
         export PMM_RELEASE_VERSION=${full_pmm_version}
         export PMM_RELEASE_TIMESTAMP=$(date '+%s')
-        export PMM_RELEASE_FULLCOMMIT=$(cd sources/${parent_component}/src/${extract_path} && git rev-parse HEAD)
-        export COMPONENT_VERSION=$(cd sources/${parent_component}/src/${extract_path} && git describe --abbrev=0 --always)
-        export COMPONENT_BRANCH=$(cd sources/${parent_component}/src/${extract_path} && git describe --always --contains --all)
         export PMM_RELEASE_BRANCH=""
         sudo chown -R builder:builder /home/builder/tmp
 
-        source_dir=/home/builder/tmp/source/pmm-client-${pmm_version}
-        binary_dir=/home/builder/tmp/binary/pmm-client-${pmm_version}
-        export PMM_RELEASE_PATH=\${binary_dir}/bin
-
-        mkdir -p \$binary_dir/bin
+        export PMM_RELEASE_PATH=${bin_dir}/bin
+        if [ "${component}" = vmagent ]; then
+            export PMM_RELEASE_FULLCOMMIT=${vmagent_commit_hash}
+        else
+            export PMM_RELEASE_FULLCOMMIT=$(git -C sources/${parent_component}/src/${extract_path} rev-parse HEAD)
+        fi
 
+        mkdir -p ${bin_dir}/bin
         mkdir -p /tmp/go/src/${extract_path}
-        archive=$(ls ${source_dir} | grep -e "${parent_component}-\w*\.tar\.gz")
-        tar -C /tmp/go/src/${extract_path} --strip-components=1 -zxpf \${source_dir}/\$archive
-        pushd /tmp/go
-            export GOPATH=\$(pwd -P)
-            ${build_command}
-        popd
+
+        tar -C /tmp/go/src/${extract_path} --strip-components=1 -zxpf ${src_dir}/${archive}
+
+        ${build_command}
+
         sudo chown -R $(id -u):$(id -g) /home/builder/tmp
     "
 }
@@ -83,8 +99,7 @@ copy_component() {
 
     if [ ! -d "${component_dir}" ]; then
         mkdir -p ${component_dir}
-        tar \
-            -C ${component_dir} \
+        tar -C ${component_dir} \
             --strip-components=1 \
             -zxpf "$archive"
     fi
@@ -104,7 +119,6 @@ main() {
     gobuild_component "rds_exporter"
     gobuild_component "azure_metrics_exporter"
     gobuild_component "clickhouse_exporter" "" "github.com/Percona-Lab/clickhouse_exporter"
-
     gobuild_component "pmm-admin" "pmm" "" "admin"
     gobuild_component "pmm-agent" "pmm" "" "agent"
 
@@ -121,16 +135,16 @@ main() {
     copy_component "postgres_exporter" "queries-postgres-uptime.yml"
     copy_component "postgres_exporter" "queries-mr.yaml"
     copy_component "postgres_exporter" "queries-lr.yaml"
-
     copy_component "percona-toolkit" "bin/pt-summary"
     copy_component "percona-toolkit" "bin/pt-mysql-summary"
+
     gobuild_component "percona-toolkit" "" "github.com/percona/percona-toolkit" "src/go/pt-mongodb-summary"
     gobuild_component "percona-toolkit" "" "github.com/percona/percona-toolkit" "src/go/pt-pg-summary"
 
     rm -rf ${binary_tarball}
     mkdir -p $(dirname ${binary_tarball}) || :
     
-    # Removes pmm binary in case this is not a PR build - such as RC/GA build
+    # Removes pmm binary (aka CLI) in case this is not a PR build - such as RC/GA build
     if [[ "$(basename ${root_dir})" != pmm-submodules_PR-* ]]; then
         rm -f "${binary_dir}/bin/pmm"
     fi
diff --git a/build/scripts/build-client-docker b/build/scripts/build-client-docker
index ea6287703f..119a47f7a3 100755
--- a/build/scripts/build-client-docker
+++ b/build/scripts/build-client-docker
@@ -20,13 +20,13 @@ fi
 
 CLIENT_IMAGE_VERSION=`echo $DOCKER_CLIENT_TAG | cut -d ':' -f2`
 
-docker build --build-arg BUILD_DATE="`date --rfc-3339=seconds`" \
-             --build-arg VERSION="$CLIENT_IMAGE_VERSION" \
-             --squash \
-             --no-cache \
-             -f ${DOCKER_FILE_LOCATION}/${docker_file} \
-             -t ${DOCKER_CLIENT_TAG} \
-             ${DOCKER_FILE_LOCATION}
+docker buildx build \
+    --build-arg BUILD_DATE="`date --rfc-3339=seconds`" \
+    --build-arg VERSION="$CLIENT_IMAGE_VERSION" \
+    --progress plain \
+    -f ${DOCKER_FILE_LOCATION}/${docker_file} \
+    -t ${DOCKER_CLIENT_TAG} \
+    ${DOCKER_FILE_LOCATION}
 
 if [ -n "${PUSH_DOCKER}" ]; then
     mkdir -p $(dirname ${docker_client_tag_file})
diff --git a/build/scripts/build-client-packages b/build/scripts/build-client-packages
index ae155999aa..97865555b3 100755
--- a/build/scripts/build-client-packages
+++ b/build/scripts/build-client-packages
@@ -17,7 +17,7 @@ Usage: $0 [OPTIONS]
         --build_source_deb  If it is 1 source deb package will be built
         --build_rpm         If it is 1 rpm will be built
         --build_deb         If it is 1 deb will be built
-        --install_deps      Install build dependencies(root previlages are required)
+        --install_deps      Install build dependencies(root privileges are required)
         --branch            Branch from which submodules should be taken(default v3)
         --help) usage ;;
 Example $0 --builddir=/tmp/PMM_CLIENT --get_sources=1 --build_src_rpm=1 --build_rpm=1
@@ -90,13 +90,12 @@ check_workdir(){
     if [ "x$WORKDIR" = "x$CURDIR" ]; then
         echo >&2 "Current directory cannot be used for building!"
         exit 1
-    else
-        if ! test -d "$WORKDIR"; then
-            echo >&2 "$WORKDIR is not a directory."
-            exit 1
-        fi
     fi
-    return
+    
+    if ! test -d "$WORKDIR"; then
+        echo >&2 "$WORKDIR is not a directory."
+        exit 1
+    fi
 }
 
 get_sources(){
@@ -108,11 +107,11 @@ get_sources(){
     git clone $REPO
     retval=$?
     if [ $retval != 0 ]; then
-        echo "There were some issues during repo cloning from github. Please retry one more time"
+        echo "There were some issues during repo cloning from github. Please retry."
         exit 1
     fi
     cd pmm-client
-    if [ ! -z $BRANCH ]; then
+    if [ -n $BRANCH ]; then
         git reset --hard
         git clean -xdf
         git checkout $BRANCH
@@ -122,7 +121,6 @@ get_sources(){
     #
     VERSION=$(pmm_version)
     mv Makefile build/
-    #cat VERSION > $VERSION_FILE
     echo "VERSION=${VERSION}" > $VERSION_FILE
     echo "REVISION=${REVISION}" >> $VERSION_FILE
     echo "RPM_RELEASE=${RPM_RELEASE}" >> $VERSION_FILE
@@ -133,7 +131,6 @@ get_sources(){
     echo "MongoExp_BRANCH_NAME=${MongoExp_BRANCH_NAME}" >> $VERSION_FILE
     echo "MysqlExp_BRANCH_NAME=${MysqlExp_BRANCH_NAME}" >> $VERSION_FILE
     echo "ProxysqlExp_BRANCH_NAME=${ProxysqlExp_BRANCH_NAME}" >> $VERSION_FILE
-    echo "QAN_BRANCH_NAME=${QAN_BRANCH_NAME}" >> $VERSION_FILE
     echo "TOOLKIT_REPO=${TOOLKIT_REPO}" >> $VERSION_FILE
     echo "TOOLKIT_BRANCH_NAME=${TOOLKIT_BRANCH_NAME}" >> $VERSION_FILE
     PRODUCT=pmm-client
@@ -146,10 +143,8 @@ get_sources(){
     echo "BUILD_ID=${BUILD_ID}" >> $VERSION_FILE
     echo "UPLOAD=UPLOAD/experimental/BUILDS/${PRODUCT_NAME}/${VERSION}/${BRANCH_NAME}/${REVISION}/${BUILD_ID}" >> $VERSION_FILE
     echo "MongoExp_REPO=${MongoExp_REPO}" >> $VERSION_FILE
-    echo "TOOLKIT_REPO=${TOOLKIT_REPO}" >> $VERSION_FILE
     echo "MysqlExp_REPO=${MysqlExp_REPO}" >> $VERSION_FILE
     echo "ProxysqlExp_REPO=${ProxysqlExp_REPO}" >> $VERSION_FILE
-    echo "QAN_REPO=${QAN_REPO}" >> $VERSION_FILE
     echo "NodeExp_REPO=${NodeExp_REPO}" >> $VERSION_FILE
     cd ../
     mv ${PRODUCT} ${PRODUCT}-${VERSION}
@@ -179,6 +174,7 @@ get_system(){
 }
 
 install_deps() {
+    local RUN_CMD
     if [ $INSTALL = 0 ]; then
         echo "Dependencies will not be installed"
         return;
@@ -188,7 +184,17 @@ install_deps() {
         exit 1
     fi
     if [ "$OS" = "rpm" ]; then
-        yum -y install git wget rpmdevtools bison yum-utils rpm-build
+        # yum -y install git curl wget rpmdevtools bison yum-utils rpm-build
+        command -v git || RUN_CMD+=" git"
+        command -v curl || RUN_CMD+=" curl"
+        command -v wget || RUN_CMD+=" wget"
+        command -v spectool || RUN_CMD+=" rpmdevtools"
+        command -v bison || RUN_CMD+=" bison"
+        command -v repoquery || RUN_CMD+=" yum-utils"
+        command -v rpmbuild || RUN_CMD+=" rpm-build"
+        if [ -n "$RUN_CMD" ]; then
+            yum -y install "$RUN_CMD"
+        fi
     else
         apt-get update
         apt-get -y install devscripts equivs debhelper
@@ -198,9 +204,9 @@ install_deps() {
 
 get_tar(){
     TARBALL=$1
-    TARFILE=$(basename $(find $WORKDIR/$TARBALL -name 'pmm-client*.tar.gz' | sort | tail -n1) || :)
+    TARFILE=$(basename $(find $WORKDIR/$TARBALL -name "${PACKAGE_NAME}*.tar.gz" | sort | tail -n1) || :)
     if [ -z $TARFILE ]; then
-        TARFILE=$(basename $(find $CURDIR/$TARBALL -name 'pmm-client*.tar.gz' | sort | tail -n1) || :)
+        TARFILE=$(basename $(find $CURDIR/$TARBALL -name "${PACKAGE_NAME}*.tar.gz" | sort | tail -n1) || :)
         if [ -z $TARFILE ]; then
             echo "There is no $TARBALL for build"
             exit 1
@@ -216,9 +222,9 @@ get_tar(){
 get_deb_sources(){
     param=$1
     echo $param
-    FILE=$(basename $(find $WORKDIR/source_deb -name "pmm-client*.$param" | sort | tail -n1) || :)
+    FILE=$(basename $(find $WORKDIR/source_deb -name "${PACKAGE_NAME}*.$param" | sort | tail -n1) || :)
     if [ -z $FILE ]; then
-        FILE=$(basename $(find $CURDIR/source_deb -name "pmm-client*.$param" | sort | tail -n1) || :)
+        FILE=$(basename $(find $CURDIR/source_deb -name "${PACKAGE_NAME}*.$param" | sort | tail -n1) || :)
         if [ -z $FILE ]; then
             echo "There is no sources for build"
             exit 1
@@ -232,7 +238,7 @@ get_deb_sources(){
 }
 
 build_srpm(){
-    if [ $SRPM = 0 ]; then
+    if [ "$SRPM" = 0 ]; then
         echo "SRC RPM will not be created"
         return;
     fi
@@ -242,28 +248,24 @@ build_srpm(){
     fi
     cd $WORKDIR
     get_tar "tarball"
-    
-    #
+
     rm -fr rpmbuild
     ls | grep -v tar.gz | xargs rm -rf
-    
-    #
-    TARFILE=$(basename $(find . -name 'pmm-client-*.tar.gz' | sort | tail -n1))
+
+    TARFILE=$(basename $(find . -name "${PACKAGE_NAME}-*.tar.gz" | sort | tail -n1))
     NAME=$(echo ${TARFILE}| awk -F '-' '{print $1"-"$2}')
     VERSION_TMP=$(echo ${TARFILE}| awk -F '-' '{print $3}')
     VERSION=${VERSION_TMP%.tar.gz}
-    
-    #
+
     mkdir -vp rpmbuild/{SOURCES,SPECS,BUILD,SRPMS,RPMS}
-    
-    #
+
     tar -C ${WORKDIR} -zxpf ${TARFILE} ${NAME}-${VERSION}/rpm
-    #
+
     cd ${WORKDIR}/rpmbuild/SPECS
     cp ${WORKDIR}/${NAME}-${VERSION}/rpm/*.spec .
     cp ${WORKDIR}/${TARFILE} ../SOURCES/
     cd ${WORKDIR}
-    rpmbuild -bs --define "_topdir ${WORKDIR}/rpmbuild" --define "version $VERSION" --define "release $RPM_RELEASE" --define "dist %{nil}" rpmbuild/SPECS/pmm-client.spec
+    rpmbuild -bs --define "_topdir ${WORKDIR}/rpmbuild" --define "version $VERSION" --define "release $RPM_RELEASE" --define "dist %{nil}" "rpmbuild/SPECS/${PACKAGE_NAME}.spec"
     mkdir -p ${WORKDIR}/srpm
     mkdir -p ${CURDIR}/srpm
     cp rpmbuild/SRPMS/*.src.rpm ${CURDIR}/srpm
@@ -271,7 +273,7 @@ build_srpm(){
 }
 
 build_rpm(){
-    if [ $RPM = 0 ]; then
+    if [ "$RPM" = 0 ]; then
         echo "RPM will not be created"
         return;
     fi
@@ -280,9 +282,9 @@ build_rpm(){
         exit 1
     fi
 
-    SRC_RPM=$(basename $(find $WORKDIR/srpm -name 'pmm-client*.src.rpm' | sort | tail -n1) || :)
+    SRC_RPM=$(basename $(find $WORKDIR/srpm -name "${PACKAGE_NAME}*.src.rpm" | sort | tail -n1) || :)
     if [ -z $SRC_RPM ]; then
-        SRC_RPM=$(basename $(find $CURDIR/srpm -name 'pmm-client*.src.rpm' | sort | tail -n1) || :)
+        SRC_RPM=$(basename $(find $CURDIR/srpm -name "${PACKAGE_NAME}*.src.rpm" | sort | tail -n1) || :)
         if [ -z $SRC_RPM ]; then
             echo "There is no src rpm for build"
             echo "You can create it using key --build_src_rpm=1"
@@ -298,6 +300,10 @@ build_rpm(){
     rm -fr rpmbuild
     mkdir -vp rpmbuild/{SOURCES,SPECS,BUILD,SRPMS,RPMS}
     cp $SRC_RPM rpmbuild/SRPMS/
+    if find "${CURDIR}/rpm" -name "${PACKAGE_NAME}-${pmm_version}-*.rpm"; then
+        # Remove older client rpm packages before building a new one
+        rm -f "${CURDIR}/rpm/${PACKAGE_NAME}-${pmm_version}-*.rpm"
+    fi
     rpmbuild --define "_topdir ${WORKDIR}/rpmbuild" \
              --define "version $pmm_version" \
              --define "release $RPM_RELEASE" \
@@ -305,17 +311,17 @@ build_rpm(){
              --rebuild rpmbuild/SRPMS/$SRC_RPM
 
     return_code=$?
-    if [ $return_code != 0 ]; then
+    if [ "$return_code" -ne 0 ]; then
+        echo "Error: rpmbuild exited with a non-zero return code '$return_code'\n"
         exit $return_code
     fi
-    mkdir -p ${WORKDIR}/rpm
+
     mkdir -p ${CURDIR}/rpm
-    cp rpmbuild/RPMS/*/*.rpm ${WORKDIR}/rpm
     cp rpmbuild/RPMS/*/*.rpm ${CURDIR}/rpm
 }
 
 build_source_deb(){
-    if [ $SDEB = 0 ]; then
+    if [ "$SDEB" = 0 ]; then
         echo "source deb package will not be created"
         return;
     fi
@@ -326,17 +332,16 @@ build_source_deb(){
     rm -rf pmm-client*
     get_tar "tarball"
     rm -f *.dsc *.orig.tar.gz *.debian.tar.gz *.changes
-    #
-    TARFILE=$(basename $(find . -name 'pmm-client-*.tar.gz' | sort | tail -n1))
+
+    TARFILE=$(basename $(find . -name "${PACKAGE_NAME}-*.tar.gz" | sort | tail -n1))
     NAME=$(echo ${TARFILE}| awk -F '-' '{print $1"-"$2}')
     VERSION_TMP=$(echo ${TARFILE}| awk -F '-' '{print $3}')
     VERSION=${VERSION_TMP%.tar.gz}
 
     rm -fr ${NAME}-${VERSION}
-    #
+
     NEWTAR=${NAME}_${VERSION}.orig.tar.gz
     mv ${TARFILE} ${NEWTAR}
-    #
 
     git clone $REPO ${NAME}-${VERSION}_all
     pushd ${NAME}-${VERSION}_all
@@ -363,7 +368,7 @@ build_source_deb(){
     mv queries-mr.yaml ../${NAME}-${VERSION}_all/distro/
     mv queries-lr.yaml ../${NAME}-${VERSION}_all/distro/
     mv debian ../${NAME}-${VERSION}_all/
-    mv config/pmm-agent.service ../${NAME}-${VERSION}_all/debian/pmm-client.pmm-agent.service
+    mv config/pmm-agent.service ../${NAME}-${VERSION}_all/debian/${PACKAGE_NAME}.pmm-agent.service
     cd ../
 
     rm -rf ${NAME}-${VERSION}
@@ -396,12 +401,12 @@ build_source_deb(){
 }
 
 build_deb(){
-    if [ $DEB = 0 ]; then
-        echo "source deb package will not be created"
+    if [ "$DEB" = 0 ]; then
+        echo "deb package will not be created"
         return;
     fi
     if [ "$OS" = "rpm" ]; then
-        echo "It is not possible to build source deb here"
+        echo "It is not possible to build deb here"
         exit 1
     fi
     for file in 'dsc' 'orig.tar.gz' 'changes' 'diff.gz'; do
@@ -432,6 +437,7 @@ build_deb(){
 #main
 
 CURDIR=$(pwd)
+PACKAGE_NAME=pmm3-client
 VERSION_FILE=$CURDIR/pmm-client.properties
 args=
 WORKDIR=
@@ -446,15 +452,15 @@ ARCH=
 OS=
 SUBMODULE_BRANCH="v3"
 INSTALL=0
-RPM_RELEASE=6
-DEB_RELEASE=6
+RPM_RELEASE=1
+DEB_RELEASE=1
+REVISION=0
 
 if [ -n "$pmm_release" ]; then
     RPM_RELEASE="$RPM_RELEASE.$pmm_release"
     DEB_RELEASE="$DEB_RELEASE.$pmm_release"
 fi
 
-REVISION=0
 parse_arguments PICK-ARGS-FROM-ARGV "$@"
 get_system
 install_deps
@@ -482,8 +488,6 @@ if [ "$TARBALL" = '1' -o "$SOURCE" = '1' ]; then
 fi
 
 check_workdir
-get_system
-install_deps
 get_sources
 build_srpm
 build_source_deb
diff --git a/build/scripts/build-client-rpm b/build/scripts/build-client-rpm
index 7c178670d0..ae5780ae24 100755
--- a/build/scripts/build-client-rpm
+++ b/build/scripts/build-client-rpm
@@ -5,9 +5,6 @@
 set -o errexit
 set -o xtrace
 
-bin_dir=$(cd $(dirname $0); pwd -P)
-tmp_dir=${root_dir}/tmp
-
 main() {
     local IMAGE=${1:-${rpmbuild_docker_image}}
     docker run --rm -v ${bin_dir}:/home/builder/bin -v ${root_dir}/results:/home/builder/results ${IMAGE} sh -c "
@@ -18,14 +15,11 @@ main() {
         export pmm_release=$pmm_release
         export sudo_path=\$(ls /usr/bin/sudo)
         mkdir -p /tmp/pmm
-        pushd /home/builder/results
-            \$sudo_path /home/builder/bin/build-client-packages \
-                --builddir=/tmp/pmm \
-                --install_deps=1
-            /home/builder/bin/build-client-packages \
-                --builddir=/tmp/pmm \
-                --build_rpm=1
-        popd
+        cd /home/builder/results
+        /home/builder/bin/build-client-packages \
+            --builddir=/tmp/pmm \
+            --build_rpm=1
+
         \$sudo_path chown -R $(id -u):$(id -g) /home/builder/results/rpm
     "
 }
diff --git a/build/scripts/build-client-source b/build/scripts/build-client-source
index b9867f764b..978f09a1b0 100755
--- a/build/scripts/build-client-source
+++ b/build/scripts/build-client-source
@@ -7,11 +7,12 @@ set -o xtrace
 
 prepare_tarball() {
     local repo_name=$1
+    local git_dir
     if [ $repo_name = 'pmm-submodules' ]; then
-        mkdir -p ${root_dir}/sources/${repo_name}
         git_dir="${root_dir}/sources/${repo_name}"
+        mkdir -p "${git_dir}"
     else
-        local git_dir=$(dirname $(find "${root_dir}/sources/${repo_name}" -name .git | head -1))
+        git_dir=$(dirname $(find "${root_dir}/sources/${repo_name}" -name .git | head -1))
     fi
     local commit=$(git -C "${git_dir}" rev-parse --short HEAD)
     local tarball=${source_dir}/${repo_name}-${commit}.tar.gz
@@ -24,43 +25,43 @@ prepare_tarball() {
 
     if [ -f "${tarball}" ]; then
         echo ${tarball} already exists, skipping build
-    else
-        if [ ${repo_name} = 'pmm-submodules' ]; then
-            tar -C sources -zcf ${repo_name}-${commit}.tar.gz  ${repo_name}
-            mv ${repo_name}-${commit}.tar.gz  ${source_dir}/
-        else
-            git -C "${git_dir}" archive \
-                --format=tar.gz \
-                --prefix=${repo_name}-${commit}/ \
-                -o ${tarball} \
-                "${commit}"
-        fi
+        return
+    fi
+
+    if [ ${repo_name} = 'pmm-submodules' ]; then
+        tar -C sources -zcf ${repo_name}-${commit}.tar.gz  ${repo_name}
+        mv ${repo_name}-${commit}.tar.gz  ${source_dir}/
+        return
     fi
+
+    git -C "${git_dir}" archive \
+        --format=tar.gz \
+        --prefix=${repo_name}-${commit}/ \
+        -o ${tarball} \
+        "${commit}"
 }
 
 prepare_vmagent_tarball() {
     local repo_name=vmagent
     local commit_hash=${vmagent_commit_hash}
-    local commit_short=${commit_hash::6}
+    local commit_short=${commit_hash::7}
     local tarball=${source_dir}/${repo_name}-${commit_short}.tar.gz
     local repo_var=$(
-        echo "${repo_name}_commit" \
-            | sed -e 's:-:_:g' \
-            | tr '[[:lower:]]' '[[:upper:]]'
+        echo "${repo_name}_commit" | sed -e 's:-:_:g' | tr '[[:lower:]]' '[[:upper:]]'
     )
     echo "${repo_var}=$commit_short" >> ${client_properties}
 
     if [ -f "${tarball}" ]; then
         echo ${tarball} already exists, skipping download
     else
-        wget https://github.com/VictoriaMetrics/VictoriaMetrics/archive/${commit_hash}.tar.gz -O ${tarball}
+        curl -o ${tarball} -fSL https://github.com/VictoriaMetrics/VictoriaMetrics/archive/${commit_hash}.tar.gz
     fi
 }
 
 main() {
     rm -rf ${source_tarball}
     mkdir -p ${source_dir}/ $(dirname ${source_tarball}) || :
-    echo "PMM_SUBMODULES_COMMIT=$(git rev-parse --short HEAD)" > ${client_properties}
+
     prepare_vmagent_tarball
     prepare_tarball pmm-submodules
     prepare_tarball pmm
diff --git a/build/scripts/build-client-srpm b/build/scripts/build-client-srpm
index aff1082492..20747d0779 100755
--- a/build/scripts/build-client-srpm
+++ b/build/scripts/build-client-srpm
@@ -5,9 +5,6 @@
 set -o errexit
 set -o xtrace
 
-bin_dir=$(cd $(dirname $0); pwd -P)
-tmp_dir=${root_dir}/tmp
-
 main() {
     local IMAGE=${1:-${rpmbuild_docker_image}}
     docker run --rm -v ${bin_dir}:/home/builder/bin -v ${root_dir}/results:/home/builder/results ${IMAGE} sh -c "
@@ -15,17 +12,13 @@ main() {
         set -o xtrace
 
         export pmm_release=$pmm_release
-        export sudo_path=
-        if [ -e /usr/bin/sudo ]; then
-            export sudo_path=\$(ls /usr/bin/sudo)
-        fi
+        export sudo_path=\$(command -v sudo)
         [[ ${IMAGE} = ${rpmbuild_docker_image} ]] || \$sudo_path yum -y install git rpm-build
         mkdir -p /tmp/pmm
-        pushd /home/builder/results
-            /home/builder/bin/build-client-packages \
-                --builddir=/tmp/pmm \
-                --build_src_rpm=1
-        popd
+        cd /home/builder/results
+        /home/builder/bin/build-client-packages \
+            --builddir=/tmp/pmm \
+            --build_src_rpm=1
         \$sudo_path chown -R $(id -u):$(id -g) /home/builder/results/srpm
     "
 }
diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index 4c2cb97df6..9be0b05f3c 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -24,6 +24,7 @@ is_build_needed() {
     local spec_name=$1
     local rpm_version=$2
     local packages=
+    local cache_dir=RELEASE
 
     # Structure of S3 build cache
     # s3://pmm-build-cache/PR-BUILDS/7 - el7
@@ -31,18 +32,17 @@ is_build_needed() {
     # s3://pmm-build-cache/RELEASE/7 - el7
     # s3://pmm-build-cache/RELEASE/9 - el9
 
-    # RPM_EPOCH is set for Feature Builds
-    if [ -n "$RPM_EPOCH" ]; then
-        aws s3 sync \
-            --region us-east-2 \
-            --no-sign-request \
-            s3://pmm-build-cache/PR-BUILDS/${rpmbuild_dist}/${spec_name}-${rpm_version} \
-            ${rpms_dir}/${spec_name}-${rpm_version}
-    else
+    [ "${FORCE_REBUILD}" = 1 ] && return 0
+
+    # RPM_EPOCH is set to 1 for Feature Builds
+    if [ "$RPM_EPOCH" = 1 ]; then
+        cache_dir=PR-BUILDS
+    fi
+    if command -v aws &> /dev/null; then
         aws s3 sync \
             --region us-east-2 \
             --no-sign-request \
-            s3://pmm-build-cache/RELEASE/${rpmbuild_dist}/${spec_name}-${rpm_version} \
+            s3://pmm-build-cache/${cache_dir}/${rpmbuild_dist}/${spec_name}-${rpm_version} \
             ${rpms_dir}/${spec_name}-${rpm_version}
     fi
 
@@ -58,103 +58,120 @@ prepare_specs() {
 
     sudo chown -R $(id -u):$(id -g) ${rpmbuild_dir}/SPECS ${rpmbuild_dir}/SOURCES
     cp ${rpmbuild_dir}/SPECS/${spec_name}.spec ${rpmbuild_dir}/SOURCES/${spec_name}.spec
-    if [ -d "${root_dir}/sources/${repo_name}" ]; then
-        local git_dir=$(dirname $(find "${root_dir}/sources/${repo_name}" -name .git | head -1))
-        local full_commit=$(git -C "${git_dir}" rev-parse HEAD)
-        local short_commit=${full_commit:0:7}
-
-        sed -i -e "s/global commit.*/global commit ${full_commit}/" ${rpmbuild_dir}/SOURCES/${spec_name}.spec
-        if [[ ${spec_name} != "grafana" ]]; then
-            sed -i -e "s/Version:.*/Version: ${pmm_version}/"       ${rpmbuild_dir}/SOURCES/${spec_name}.spec
-        fi
+    if [ ! -d "${root_dir}/sources/${repo_name}" ]; then
+        return
+    fi
+    
+    local git_dir=$(dirname $(find "${root_dir}/sources/${repo_name}" -name .git | head -1))
+    local full_commit=$(git -C "${git_dir}" rev-parse HEAD)
+    local short_commit=${full_commit:0:7}
+
+    sed -i -e "s/global commit.*/global commit ${full_commit}/" ${rpmbuild_dir}/SOURCES/${spec_name}.spec
+    if [ "${spec_name}" != "grafana" ]; then
+        sed -i -e "s/Version:.*/Version: ${pmm_version}/"       ${rpmbuild_dir}/SOURCES/${spec_name}.spec
+    fi
 
-        if [ -z "${full_pmm_version}" ]; then
-            echo 'A full_pmm_version is not specified.'
-            exit 1
-        fi
-        sed -i -e "s/%define full_pmm_version.*/%define full_pmm_version ${full_pmm_version}/" ${rpmbuild_dir}/SOURCES/${spec_name}.spec
+    if [ -z "${full_pmm_version}" ]; then
+        echo 'The full_pmm_version is not specified.'
+        exit 1
+    fi
+    sed -i -e "s/%define full_pmm_version.*/%define full_pmm_version ${full_pmm_version}/" ${rpmbuild_dir}/SOURCES/${spec_name}.spec
 
-        if [ -n "$pmm_release" ]; then
-            sed -i -e "s/\(%define release.*\)/\1.$pmm_release/" ${rpmbuild_dir}/SOURCES/${spec_name}.spec
-            grep -r 'define release' ${rpmbuild_dir}/SOURCES/${spec_name}.spec
-        fi
+    if [ -n "$pmm_release" ]; then
+        sed -i -e "s/\(%define release.*\)/\1.$pmm_release/" ${rpmbuild_dir}/SOURCES/${spec_name}.spec
+        grep -r 'define release' ${rpmbuild_dir}/SOURCES/${spec_name}.spec
+    fi
 
-        if [ -f "${rpmbuild_dir}/SOURCES/${repo_name}-${short_commit}.tar.gz" ]; then
-            echo SOURCES/${repo_name}-${short_commit}.tar.gz already exists, skip archiving...
-        else
-            git -C "${git_dir}" archive \
-                --format=tar.gz \
-                --prefix=${repo_name}-${full_commit}/ \
-                -o ${rpmbuild_dir}/SOURCES/${repo_name}-${short_commit}.tar.gz \
-                "${full_commit}"
-        fi
+    if [ -f "${rpmbuild_dir}/SOURCES/${repo_name}-${short_commit}.tar.gz" ]; then
+        echo SOURCES/${repo_name}-${short_commit}.tar.gz already exists, skip archiving...
+        return
     fi
+
+    git -C "${git_dir}" archive \
+        --format=tar.gz \
+        --prefix=${repo_name}-${full_commit}/ \
+        -o ${rpmbuild_dir}/SOURCES/${repo_name}-${short_commit}.tar.gz \
+        "${full_commit}"
 }
 
 build() {
     local spec_name=$1
     local repo_name=${2:-$1}
     prepare_specs "${spec_name}" "${repo_name}"
+
     local rpm_version=$(get_rpm_version "${spec_name}")
     local CH_VERSION=${rpm_version%-*}
     local CH_TAG=${rpm_version#*-}
+    local cache_dir=RELEASE
+    local volume_mounts="-v ${rpmbuild_dir}/SOURCES:/home/builder/rpm/SOURCES -v ${rpms_dir}:/home/builder/rpm/RPMS"
+    local build_deps=("dbaas-tools.spec" "grafana.spec" "percona-dashboards.spec")
 
-    if  [[ -n "${FORCE_REBUILD}" ]] || is_build_needed "${spec_name}" "${rpm_version}"; then
-        echo "RPM Build Dir: ${rpmbuild_dir}, Docker image: ${rpmbuild_docker_image}"
-
-        docker run --rm -v ${rpmbuild_dir}/SOURCES:/home/builder/rpm/SOURCES -v ${rpms_dir}:/home/builder/rpm/RPMS ${rpmbuild_docker_image} sh -c "
-            set -o errexit
-            set -o xtrace
-            echo \"spec_name: ${spec_name}\"
-            echo \"repo_name: ${repo_name}\"
-            echo \"rpm_verison: ${rpm_version}\"
+    if ! is_build_needed "${spec_name}" "${rpm_version}"; then
+        return
+    fi
 
-            sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES
+    if [ -d "${root_dir}/go-build" ]; then
+        volume_mounts+=" -v ${root_dir}/go-build:/home/builder/.cache/go-build"
+    fi
+    if [ -d "${root_dir}/go-path" ]; then
+        volume_mounts+=" -v ${root_dir}/go-path:/home/builder/go/pkg/mod"
+    fi
 
-            # Add 'Epoch' to spec file to prevent update of rpms which are built in PR build
-            if [ -n \"$RPM_EPOCH\" ]; then
-                sed -i '/^Version:.*/i Epoch: 1' /home/builder/rpm/SOURCES/${spec_name}.spec
-            fi
+    echo "spec_name: ${spec_name}"
+    echo "repo_name: ${repo_name}"
+    echo "rpm_verison: ${rpm_version}"
 
-            rm -rf /home/builder/rpm/RPMS/${spec_name}-*
+    docker run --rm ${volume_mounts} ${rpmbuild_docker_image} sh -c "
+        set -o errexit
+        set -o xtrace
 
-            printf '[local]\nname=local\nbaseurl=file:///home/builder/rpm/RPMS\ngpgcheck=0\nenabled=1\n' \
-                | sudo tee /etc/yum.repos.d/local.repo
+        sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES
 
-            until /usr/bin/createrepo_c --update /home/builder/rpm/RPMS; do
-              echo waiting
-              sleep 1
-            done
+        # Add 'Epoch' to spec file to prevent update of rpms which are built in PR build
+        if [ \"$RPM_EPOCH\" = 1 ]; then
+            sed -i '/^Version:.*/i Epoch: 1' /home/builder/rpm/SOURCES/${spec_name}.spec
+        fi
 
-            sudo yum-builddep --randomwait=1 -y SOURCES/${spec_name}.spec
+        rm -rf /home/builder/rpm/RPMS/${spec_name}-*
 
-            spectool -C SOURCES -g SOURCES/${spec_name}.spec
-            rpmbuild  --define '_rpmdir %{_topdir}/RPMS/${spec_name}-${rpm_version}' \
-                      --define 'dist .${rpmbuild_dist}' \
-                      --define 'debug_package %{nil}' \
-                      -ba SOURCES/${spec_name}.spec
+        printf '[local]\nname=local\nbaseurl=file:///home/builder/rpm/RPMS\ngpgcheck=0\nenabled=1\n' \
+            | sudo tee /etc/yum.repos.d/local.repo
 
-            rm -f SOURCES/${spec_name}.spec*
-            sudo chown -R $(id -u):$(id -g) /home/builder/rpm/RPMS /home/builder/rpm/SOURCES
-        "
+        until /usr/bin/createrepo_c --update /home/builder/rpm/RPMS; do
+          echo waiting
+          sleep 1
+        done
 
-        if [ -n "$RPM_EPOCH" ]; then
-            aws s3 sync \
-                --region us-east-2 \
-                ${rpms_dir}/${spec_name}-${rpm_version} \
-                s3://pmm-build-cache/PR-BUILDS/${rpmbuild_dist}/${spec_name}-${rpm_version} \
-                || :
-        else
-            aws s3 sync \
-                --region us-east-2 \
-                ${rpms_dir}/${spec_name}-${rpm_version} \
-                s3://pmm-build-cache/RELEASE/${rpmbuild_dist}/${spec_name}-${rpm_version} \
-                || :
+        if grep ${spec_name}.spec <<< ${build_deps[*]} &>/dev/null; then
+          sudo yum-builddep --randomwait=1 -y SOURCES/${spec_name}.spec
         fi
 
-        cp ${rpms_dir}/${spec_name}-${rpm_version}/*/*.rpm ${rpms_dir}
+        spectool -C SOURCES -g SOURCES/${spec_name}.spec
+        rpmbuild  --define '_rpmdir %{_topdir}/RPMS/${spec_name}-${rpm_version}' \
+                  --define 'dist .${rpmbuild_dist}' \
+                  --define 'debug_package %{nil}' \
+                  -ba SOURCES/${spec_name}.spec
+
+        rm -f SOURCES/${spec_name}.spec*
+        sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES
+    "
+
+    # TODO: see if this is even needed
+    # cp ${rpms_dir}/${spec_name}-${rpm_version}/*/*.rpm ${rpms_dir}
+
+    if ! command -v aws &> /dev/null; then
+        echo "AWS CLI is not installed. Skipping upload to S3."
+        return
+    fi
 
+    if [ "$RPM_EPOCH" = 1 ]; then
+      cache_dir=PR-BUILDS
     fi
+    aws s3 sync \
+        --region us-east-2 \
+        ${rpms_dir}/${spec_name}-${rpm_version} \
+        s3://pmm-build-cache/${cache_dir}/${rpmbuild_dist}/${spec_name}-${rpm_version} \
+        || :
 }
 
 build "$1" "$2"
diff --git a/build/scripts/build-submodules b/build/scripts/build-submodules
index 34134e794b..e32f4b05a8 100755
--- a/build/scripts/build-submodules
+++ b/build/scripts/build-submodules
@@ -1,42 +1,30 @@
 #!/bin/bash
 set -o errexit
 
+git submodule update --init --jobs 10
+git submodule status
+
 if [ -s ci.yml ]; then
-    sudo rm -rf results tmp || :
     python3 ci.py
+    cat .git-sources
     . ./.git-sources
-    echo $pmm_commit > apiCommitSha
-    echo $pmm_branch > apiBranch
-    echo $pmm_url > apiURL
-    echo $pmm_qa_branch > pmmQABranch
-    echo $pmm_qa_commit > pmmQACommitSha
-    echo $pmm_ui_tests_branch > pmmUITestBranch
-    echo $pmm_ui_tests_commit > pmmUITestsCommitSha
 else
-    sudo rm -rf results tmp || :
-    git reset --hard
-    git clean -fdx
-    git submodule foreach --recursive git reset --hard
-    git submodule foreach --recursive git clean -fdx
-    git submodule status
-    export commit_sha=$(git submodule status | grep 'pmm-managed' | awk -F ' ' '{print $1}')
-    export api_tests_commit_sha=$(git submodule status | grep 'pmm' | awk -F ' ' '{print $1}')
-    export api_tests_branch=$(git config -f .gitmodules submodule.pmm.branch)
-    export api_tests_url=$(git config -f .gitmodules submodule.pmm.url)
-    echo $api_tests_commit_sha > apiCommitSha
-    echo $api_tests_branch > apiBranch
-    echo $api_tests_url > apiURL
-    cat apiBranch
-    cat apiURL
-    export pmm_qa_commit_sha=$(git submodule status | grep 'pmm-qa' | awk -F ' ' '{print $1}')
-    export pmm_qa_branch=$(git config -f .gitmodules submodule.pmm-qa.branch)
-    echo $pmm_qa_branch > pmmQABranch
-    echo $pmm_qa_commit_sha > pmmQACommitSha
-    export pmm_ui_tests_commit_sha=$(git submodule status | grep 'pmm-ui-tests' | awk -F ' ' '{print $1}')
-    export pmm_ui_tests_branch=$(git config -f .gitmodules submodule.pmm-ui-tests.branch)
-    echo $pmm_ui_tests_branch > pmmUITestBranch
-    echo $pmm_ui_tests_commit_sha > pmmUITestsCommitSha
+    # Define variables
+    pmm_commit=$(git submodule status | grep 'sources/pmm/src' | awk -F ' ' '{print $1}')
+    pmm_branch=$(git config -f .gitmodules submodule.pmm.branch)
+    pmm_url=$(git config -f .gitmodules submodule.pmm.url)
+    pmm_qa_branch=$(git config -f .gitmodules submodule.pmm-qa.branch)
+    pmm_qa_commit=$(git submodule status | grep 'pmm-qa' | awk -F ' ' '{print $1}')
+    pmm_ui_tests_branch=$(git config -f .gitmodules submodule.pmm-ui-tests.branch)
+    pmm_ui_tests_commit=$(git submodule status | grep 'pmm-ui-tests' | awk -F ' ' '{print $1}')
 fi
 
-export fb_commit_sha=$(git rev-parse HEAD)
+fb_commit_sha=$(git rev-parse HEAD)
 echo $fb_commit_sha > fbCommitSha
+echo $pmm_commit > apiCommitSha
+echo $pmm_branch > apiBranch
+echo $pmm_url > apiURL
+echo $pmm_qa_branch > pmmQABranch
+echo $pmm_qa_commit > pmmQACommitSha
+echo $pmm_ui_tests_branch > pmmUITestBranch
+echo $pmm_ui_tests_commit > pmmUITestsCommitSha
diff --git a/build/scripts/vars b/build/scripts/vars
index af854a8ffe..ea72294352 100644
--- a/build/scripts/vars
+++ b/build/scripts/vars
@@ -19,11 +19,7 @@ full_pmm_version=${pmm_base_version}-${pmm_branch}-$(git rev-parse --short HEAD)
 # Replace '/' with '-' to prevent sed from failing on dependabot-authored PRs
 full_pmm_version=${full_pmm_version//\//-}
 
-# TODO Maybe it makes sense to use variable from job here
-if [[ ${pmm_branch} =~ release-* || \
-      ${pmm_branch} =~ pmm-3.* || \
-      ${pmm_branch} == v3 || \
-      ${pmm_branch} == "main" ]]; then
+if [[ "${pmm_branch}" =~ ^pmm-3.|^v3$|^main$ ]]; then
     full_pmm_version=${pmm_base_version}
 fi
 

From 32c8fa8f90063b01efcda089c32d34b7b65eef4f Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Thu, 21 Mar 2024 01:41:46 +0000
Subject: [PATCH 02/35] PMM-12899 update grafana spec

---
 build/packages/rpm/server/SPECS/grafana.spec | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/build/packages/rpm/server/SPECS/grafana.spec b/build/packages/rpm/server/SPECS/grafana.spec
index 85976d18a1..da293bbc1f 100644
--- a/build/packages/rpm/server/SPECS/grafana.spec
+++ b/build/packages/rpm/server/SPECS/grafana.spec
@@ -22,9 +22,6 @@ Source0:        https://github.com/percona/grafana/archive/%{commit}.tar.gz
 ExclusiveArch:  %{ix86} x86_64 %{arm}
 
 BuildRequires: fontconfig
-%if 0%{?rhel} < 9
-BuildRequires: nodejs-grunt-cli
-%endif
 
 %description
 Grafana is an open source, feature rich metrics dashboard and graph editor for
@@ -34,9 +31,7 @@ Graphite, InfluxDB & OpenTSDB.
 %setup -q -n grafana-%{commit}
 rm -rf Godeps
 sed -i "s/unknown-dev/%{grafana_version}/" pkg/build/git.go
-%if 0%{?rhel} >= 9
-    sudo npm install -g grunt-cli
-%endif
+sudo npm install -g grunt-cli
 
 %build
 mkdir -p _build/src
@@ -82,6 +77,9 @@ getent passwd pmm >/dev/null || echo "User pmm does not exist. Please create it
 exit 0
 
 %changelog
+* Tue Mar 20 2024 Alex Demidoff <alexander.demidoff@percona.com> - 10.4.0-2
+- PMM-12899 Use module and build cache
+
 * Tue Mar 12 2024 Matej Kubinec <matej.kubinec@ext.percona.com> - 10.4.0-1
 - PMM-12991 Grafana 10.4.0
 

From 37e005a07ee7c730bbb6f3e42c55620f6522a184 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Thu, 21 Mar 2024 02:11:03 +0000
Subject: [PATCH 03/35] PMM-12899 fix the build_command

---
 build/scripts/build-client-binary | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build/scripts/build-client-binary b/build/scripts/build-client-binary
index 295a5a5b02..f580e3f5d8 100755
--- a/build/scripts/build-client-binary
+++ b/build/scripts/build-client-binary
@@ -51,7 +51,7 @@ gobuild_component() {
     elif [ ${component} = 'percona-toolkit' ]; then
         build_command="cd /tmp/go/src/${extract_path}/${component_path} && go build -o ${bin_dir}/${result_file} ."
     else
-        build_command="cd /tmp/go/src/${extract_path}/${component_path} go build -o ${bin_dir}/${result_file} ."
+        build_command="cd /tmp/go/src/${extract_path}/${component_path} && go build -o ${bin_dir}/${result_file} ."
     fi
 
     docker run --rm \

From c863f50812f8c198d15610526f9f24c9026113ab Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Thu, 21 Mar 2024 03:47:06 +0000
Subject: [PATCH 04/35] PMM-12899 fix builddeps

---
 build/local/build.sh                         |  3 ---
 build/packages/rpm/server/SPECS/grafana.spec |  2 +-
 build/scripts/build-client-packages          |  2 +-
 build/scripts/build-server-rpm               | 12 ++++++------
 4 files changed, 8 insertions(+), 11 deletions(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index f27ec569a8..938d12f720 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -82,15 +82,12 @@ export RPMBUILD_DIST="el9"
 "$PATH_TO_SCRIPTS/build-server-rpm" pmm-managed pmm
 "$PATH_TO_SCRIPTS/build-server-rpm" percona-qan-api2 pmm
 "$PATH_TO_SCRIPTS/build-server-rpm" pmm-update pmm
-"$PATH_TO_SCRIPTS/build-server-rpm" dbaas-controller
-"$PATH_TO_SCRIPTS/build-server-rpm" dbaas-tools
 "$PATH_TO_SCRIPTS/build-server-rpm" pmm-dump
 "$PATH_TO_SCRIPTS/build-server-rpm" grafana-db-migrator
 "$PATH_TO_SCRIPTS/build-server-rpm" vmproxy pmm
 
 # 3rd-party
 "$PATH_TO_SCRIPTS/build-server-rpm" victoriametrics
-"$PATH_TO_SCRIPTS/build-server-rpm" alertmanager
 "$PATH_TO_SCRIPTS/build-server-rpm" grafana
 
 export DOCKER_TAG=local/pmm-server:${GIT_COMMIT}
diff --git a/build/packages/rpm/server/SPECS/grafana.spec b/build/packages/rpm/server/SPECS/grafana.spec
index da293bbc1f..95d5b956f4 100644
--- a/build/packages/rpm/server/SPECS/grafana.spec
+++ b/build/packages/rpm/server/SPECS/grafana.spec
@@ -77,7 +77,7 @@ getent passwd pmm >/dev/null || echo "User pmm does not exist. Please create it
 exit 0
 
 %changelog
-* Tue Mar 20 2024 Alex Demidoff <alexander.demidoff@percona.com> - 10.4.0-2
+* Wed Mar 20 2024 Alex Demidoff <alexander.demidoff@percona.com> - 10.4.0-2
 - PMM-12899 Use module and build cache
 
 * Tue Mar 12 2024 Matej Kubinec <matej.kubinec@ext.percona.com> - 10.4.0-1
diff --git a/build/scripts/build-client-packages b/build/scripts/build-client-packages
index 97865555b3..e7205ae7bb 100755
--- a/build/scripts/build-client-packages
+++ b/build/scripts/build-client-packages
@@ -437,7 +437,7 @@ build_deb(){
 #main
 
 CURDIR=$(pwd)
-PACKAGE_NAME=pmm3-client
+PACKAGE_NAME=pmm-client
 VERSION_FILE=$CURDIR/pmm-client.properties
 args=
 WORKDIR=
diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index 9be0b05f3c..295924c54f 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -104,7 +104,7 @@ build() {
     local CH_TAG=${rpm_version#*-}
     local cache_dir=RELEASE
     local volume_mounts="-v ${rpmbuild_dir}/SOURCES:/home/builder/rpm/SOURCES -v ${rpms_dir}:/home/builder/rpm/RPMS"
-    local build_deps=("dbaas-tools.spec" "grafana.spec" "percona-dashboards.spec")
+    # local build_deps=("grafana.spec" "percona-dashboards.spec")
 
     if ! is_build_needed "${spec_name}" "${rpm_version}"; then
         return
@@ -128,7 +128,7 @@ build() {
         sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES
 
         # Add 'Epoch' to spec file to prevent update of rpms which are built in PR build
-        if [ \"$RPM_EPOCH\" = 1 ]; then
+        if [ ${RPM_EPOCH} = 1 ]; then
             sed -i '/^Version:.*/i Epoch: 1' /home/builder/rpm/SOURCES/${spec_name}.spec
         fi
 
@@ -138,12 +138,12 @@ build() {
             | sudo tee /etc/yum.repos.d/local.repo
 
         until /usr/bin/createrepo_c --update /home/builder/rpm/RPMS; do
-          echo waiting
-          sleep 1
+            echo waiting
+            sleep 1
         done
 
-        if grep ${spec_name}.spec <<< ${build_deps[*]} &>/dev/null; then
-          sudo yum-builddep --randomwait=1 -y SOURCES/${spec_name}.spec
+        if [[ ${spec_name} =~ ^grafana$|^percona-dashboards$ ]]; then
+            sudo yum-builddep --randomwait=1 -y SOURCES/${spec_name}.spec
         fi
 
         spectool -C SOURCES -g SOURCES/${spec_name}.spec

From 853be6ed9076267f039ed860dce8ef1c30168c42 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Thu, 21 Mar 2024 04:07:25 +0000
Subject: [PATCH 05/35] PMM-12899 use rpmbuild:3

---
 build/local/build.sh | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index 938d12f720..03da4e51eb 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -43,7 +43,7 @@ test -d "${root_dir}/go-path" || mkdir -p "go-path"
 test -d "${root_dir}/go-build" || mkdir -p "go-build"
 
 PATH_TO_SCRIPTS="sources/pmm/src/github.com/percona/pmm/build/scripts"
-export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:ol9 
+export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:3
 
 # Local reference test environment
 # CPU: 4 cores
@@ -83,7 +83,6 @@ export RPMBUILD_DIST="el9"
 "$PATH_TO_SCRIPTS/build-server-rpm" percona-qan-api2 pmm
 "$PATH_TO_SCRIPTS/build-server-rpm" pmm-update pmm
 "$PATH_TO_SCRIPTS/build-server-rpm" pmm-dump
-"$PATH_TO_SCRIPTS/build-server-rpm" grafana-db-migrator
 "$PATH_TO_SCRIPTS/build-server-rpm" vmproxy pmm
 
 # 3rd-party
@@ -91,7 +90,6 @@ export RPMBUILD_DIST="el9"
 "$PATH_TO_SCRIPTS/build-server-rpm" grafana
 
 export DOCKER_TAG=local/pmm-server:${GIT_COMMIT}
-export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:ol9
 export RPMBUILD_DIST=el9
 export DOCKERFILE=Dockerfile.el9.local
 ${PATH_TO_SCRIPTS}/build-server-docker

From d57ec55633f133d0fc74f543eca0a67e708e93d8 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 26 Mar 2024 09:39:53 +0300
Subject: [PATCH 06/35] PMM-12641 Fix the dashboard upgrade (#2926)

* PMM-12641 Fix the dashboard upgrade

This fixes a bug where, due to a wrong ansible formula, PMM would upgrade the dashboards no matter what :)

* PMM-12641 follow up on review
---
 .../roles/initialization/tasks/main.yml       | 95 ++++++++++---------
 1 file changed, 50 insertions(+), 45 deletions(-)

diff --git a/build/ansible/roles/initialization/tasks/main.yml b/build/ansible/roles/initialization/tasks/main.yml
index be42baf2e8..4cb85312f7 100644
--- a/build/ansible/roles/initialization/tasks/main.yml
+++ b/build/ansible/roles/initialization/tasks/main.yml
@@ -27,56 +27,61 @@
 
 - name: Set need_upgrade fact
   set_fact:
-    need_upgrade: not pmm_current_version is version(pmm_image_version, '>=')
+    need_upgrade: "{{ pmm_current_version is version(pmm_image_version, '<') }}"
 
 - name: Print current PMM and image versions
   debug:
     msg: "Current version: {{ pmm_current_version }} Image Version: {{ pmm_image_version }}"
 
-- name: Enable maintenance mode before upgrade
-  copy:
-    src: maintenance.html
-    dest: /usr/share/pmm-server/maintenance/
-    owner: pmm
-    group: pmm
-    mode: 0644
-
-- name: Upgrade dashboards
-  include_role:
-    name: dashboards
-  when: need_upgrade
-
-- name: Copy file with image version
-  copy:
-    src: /usr/share/percona-dashboards/VERSION
-    dest: /srv/grafana/PERCONA_DASHBOARDS_VERSION
-    owner: pmm
-    group: pmm
-    mode: 0644
-    remote_src: yes
-  when: need_upgrade
-
-- name: Create a backup directory
-  file:
-    path: /srv/backup
-    state: directory
-    owner: pmm
-    group: pmm
-    mode: 0775
+- name: Print need_upgrade fact
+  debug:
+    msg: "Need upgrade: {{ need_upgrade }}"
 
-# Note: we want to leave this for some time until we achieve stable builds
-- name: Output pmm-managed logs
-  shell: sleep 10 && tail -n 300 /srv/logs/pmm-managed.log
+- name: Perform upgrade tasks
+  block:
+    - name: Enable maintenance mode before upgrade
+      copy:
+        src: maintenance.html
+        dest: /usr/share/pmm-server/maintenance/
+        owner: pmm
+        group: pmm
+        mode: 0644
+    
+    - name: Upgrade dashboards
+      include_role:
+        name: dashboards
+    
+    - name: Copy file with image version
+      copy:
+        src: /usr/share/percona-dashboards/VERSION
+        dest: /srv/grafana/PERCONA_DASHBOARDS_VERSION
+        owner: pmm
+        group: pmm
+        mode: 0644
+        remote_src: yes
+    
+    - name: Create a backup directory
+      file:
+        path: /srv/backup
+        state: directory
+        owner: pmm
+        group: pmm
+        mode: 0775
 
-- name: Wait for PMM to be ready
-  ansible.builtin.uri:
-    url: "http://127.0.0.1:7772/v1/readyz"
-    status_code: 200
-    method: GET
-  retries: 20
-  delay: 5
+    # Note: we want to leave this for some time until we achieve stable builds
+    - name: Output pmm-managed logs
+      shell: sleep 10 && tail -n 300 /srv/logs/pmm-managed.log
 
-- name: Disable maintenance mode
-  file:
-    state: absent
-    path: /usr/share/pmm-server/maintenance/maintenance.html
+    - name: Wait for PMM to be ready
+      ansible.builtin.uri:
+        url: "http://127.0.0.1:7772/v1/readyz"
+        status_code: 200
+        method: GET
+      retries: 20
+      delay: 5
+    
+    - name: Disable maintenance mode
+      file:
+        state: absent
+        path: /usr/share/pmm-server/maintenance/maintenance.html
+  when: need_upgrade

From 5c7821fc83520d31455ef6ad20a405f162a7669c Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Sun, 31 Mar 2024 23:03:42 +0000
Subject: [PATCH 07/35] PMM-12899 prevent grafana from using own GOPATH

---
 build/packages/rpm/server/SPECS/grafana.spec |  4 ++--
 build/scripts/build-server-rpm               | 15 +++++++++------
 2 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/build/packages/rpm/server/SPECS/grafana.spec b/build/packages/rpm/server/SPECS/grafana.spec
index 95d5b956f4..a535cf69b1 100644
--- a/build/packages/rpm/server/SPECS/grafana.spec
+++ b/build/packages/rpm/server/SPECS/grafana.spec
@@ -34,8 +34,8 @@ sed -i "s/unknown-dev/%{grafana_version}/" pkg/build/git.go
 sudo npm install -g grunt-cli
 
 %build
-mkdir -p _build/src
-export GOPATH="$(pwd)/_build"
+# mkdir -p _build/src
+# export GOPATH="$(pwd)/_build"
 
 make build-go
 
diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index 295924c54f..6aa60e4388 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -104,7 +104,6 @@ build() {
     local CH_TAG=${rpm_version#*-}
     local cache_dir=RELEASE
     local volume_mounts="-v ${rpmbuild_dir}/SOURCES:/home/builder/rpm/SOURCES -v ${rpms_dir}:/home/builder/rpm/RPMS"
-    # local build_deps=("grafana.spec" "percona-dashboards.spec")
 
     if ! is_build_needed "${spec_name}" "${rpm_version}"; then
         return
@@ -117,6 +116,7 @@ build() {
         volume_mounts+=" -v ${root_dir}/go-path:/home/builder/go/pkg/mod"
     fi
 
+    echo "Start building Server RPMs..."
     echo "spec_name: ${spec_name}"
     echo "repo_name: ${repo_name}"
     echo "rpm_verison: ${rpm_version}"
@@ -125,7 +125,7 @@ build() {
         set -o errexit
         set -o xtrace
 
-        sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES
+        sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES /home/builder/.cache
 
         # Add 'Epoch' to spec file to prevent update of rpms which are built in PR build
         if [ ${RPM_EPOCH} = 1 ]; then
@@ -142,8 +142,10 @@ build() {
             sleep 1
         done
 
+        # Only these two specs have build dependencies
         if [[ ${spec_name} =~ ^grafana$|^percona-dashboards$ ]]; then
-            sudo yum-builddep --randomwait=1 -y SOURCES/${spec_name}.spec
+            sleep 5s
+            sudo yum-builddep -y SOURCES/${spec_name}.spec
         fi
 
         spectool -C SOURCES -g SOURCES/${spec_name}.spec
@@ -160,7 +162,7 @@ build() {
     # cp ${rpms_dir}/${spec_name}-${rpm_version}/*/*.rpm ${rpms_dir}
 
     if ! command -v aws &> /dev/null; then
-        echo "AWS CLI is not installed. Skipping upload to S3."
+        echo "AWS CLI is not installed. Upload to S3 will be skipped."
         return
     fi
 
@@ -172,9 +174,10 @@ build() {
         ${rpms_dir}/${spec_name}-${rpm_version} \
         s3://pmm-build-cache/${cache_dir}/${rpmbuild_dist}/${spec_name}-${rpm_version} \
         || :
+
+    echo "Finished building Server RPMs, spec_name: ${spec_name}.spec, repo_name: ${repo_name}"
 }
 
-build "$1" "$2"
-echo "build-server-rpm finished, spec_name: $1, repo_name: ${2:-$1}"
+build "$@"
 
 # vim: expandtab shiftwidth=4 tabstop=4

From 42a02a9217d204fac5d490e808b39ae9878c8cbc Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Sun, 31 Mar 2024 23:52:37 +0000
Subject: [PATCH 08/35] PMM-12899 use a dedicated folder for yarn cache

---
 build/local/build.sh           | 1 +
 build/scripts/build-server-rpm | 3 +++
 2 files changed, 4 insertions(+)

diff --git a/build/local/build.sh b/build/local/build.sh
index 03da4e51eb..cba72f27df 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -41,6 +41,7 @@ echo $fb_commit_sha > fbCommitSha
 # Create cache directories. 
 test -d "${root_dir}/go-path" || mkdir -p "go-path"
 test -d "${root_dir}/go-build" || mkdir -p "go-build"
+test -d "${root_dir}/yarn-cache" || mkdir -p "yarn-cache"
 
 PATH_TO_SCRIPTS="sources/pmm/src/github.com/percona/pmm/build/scripts"
 export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:3
diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index 6aa60e4388..06ae02326e 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -112,6 +112,9 @@ build() {
     if [ -d "${root_dir}/go-build" ]; then
         volume_mounts+=" -v ${root_dir}/go-build:/home/builder/.cache/go-build"
     fi
+    if [ -d "${root_dir}/yarn-cache" ]; then
+        volume_mounts+=" -v ${root_dir}/yarn-cache:/home/builder/.yarn"
+    fi
     if [ -d "${root_dir}/go-path" ]; then
         volume_mounts+=" -v ${root_dir}/go-path:/home/builder/go/pkg/mod"
     fi

From 6d0b02bdcab1d18ef4a91591a31683c8270ebc00 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Mon, 1 Apr 2024 01:19:42 +0000
Subject: [PATCH 09/35] PMM-12899 extract server rpm script

---
 build/local/build.sh                  | 82 ++++++++++++++++++++-------
 build/scripts/build-rpmbuild-docker   | 11 ----
 build/scripts/build-server-docker     | 18 +++---
 build/scripts/build-server-rpm        | 12 ++--
 build/scripts/build-server-rpm-script | 40 +++++++++++++
 5 files changed, 115 insertions(+), 48 deletions(-)
 delete mode 100755 build/scripts/build-rpmbuild-docker
 create mode 100755 build/scripts/build-server-rpm-script

diff --git a/build/local/build.sh b/build/local/build.sh
index cba72f27df..80caaa0d2d 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -1,7 +1,38 @@
 #!/bin/bash -e
 
-# Update submodules and PR branches
-/bin/bash $(dirname $0)/update.sh
+# Define global variables
+NO_UPDATE=0
+NO_CLIENT=0
+NO_CLIENT_DOCKER=0
+NO_SERVER_RPM=0
+
+while test "$#" -gt 0; do
+  case "$1" in
+    --no-update)
+      NO_UPDATE=1
+      ;;
+    --no-client)
+      NO_CLIENT=1
+      ;;
+    --no-client-docker)
+      NO_CLIENT_DOCKER=1
+      ;;
+    --no-server-rpm)
+      NO_SERVER_RPM=1
+      ;;
+    *)
+      echo "Unknown argument: $1"
+      echo "Usage: $0 [--no-update] [--no-client] [--no-client-docker] [--no-server-rpm]"
+      exit 1
+      ;;
+  esac
+  shift
+done
+
+if [ "$NO_UPDATE" -eq 0 ]; then
+  # Update submodules and PR branches
+  /bin/bash $(dirname $0)/update.sh
+fi
 
 get_branch_name() {
   local module="${1:-}"
@@ -33,7 +64,7 @@ echo $pmm_ui_tests_commit > pmmUITestsCommitSha
 fb_commit_sha=$(git rev-parse HEAD)
 echo $fb_commit_sha > fbCommitSha
 
-# We use a special docker image to build various PMM artifacts - `perconalab/rpmbuild:ol9`.
+# We use a special docker image to build various PMM artifacts - `perconalab/rpmbuild:3`.
 # Important: the docker container's user need to be able to write to these directories.
 # The docker container's user is `builder` with uid 1000 and gid 1000. You need to make sure
 # that the directories we create on the host are owned by a user with same uid and gid.
@@ -51,22 +82,26 @@ export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:3
 # RAM: 16GB
 # OS: Ubuntu 22.04.1 LTS
 
-# Build client source: 4m39s from scratch, 0m27s using cache
-"$PATH_TO_SCRIPTS/build-client-source"
+if [ "$NO_CLIENT" -eq 0 ]; then
+  # Build client source: 4m39s from scratch, 0m27s using cache
+  "$PATH_TO_SCRIPTS/build-client-source"
 
-# Build client binary: ??? from scratch, 0m20s using cache
-"$PATH_TO_SCRIPTS/build-client-binary"
+  # Build client binary: ??? from scratch, 0m20s using cache
+  "$PATH_TO_SCRIPTS/build-client-binary"
 
-# Building client source rpm takes 13s (caching does not apply)
-"$PATH_TO_SCRIPTS/build-client-srpm"
+  # Building client source rpm takes 13s (caching does not apply)
+  "$PATH_TO_SCRIPTS/build-client-srpm"
 
-# Building client rpm takes 1m40s
-"$PATH_TO_SCRIPTS/build-client-rpm"
+  # Building client rpm takes 1m40s
+  "$PATH_TO_SCRIPTS/build-client-rpm"
+fi
 
 # Building client docker image takes 17s
 GIT_COMMIT=$(git rev-parse HEAD | head -c 8)
 export DOCKER_CLIENT_TAG=local/pmm-client:${GIT_COMMIT}
-"$PATH_TO_SCRIPTS/build-client-docker"
+if [ "$NO_CLIENT_DOCKER" -eq 0 ] || [ "$NO_CLIENT" -eq 0 ]; then
+  "$PATH_TO_SCRIPTS/build-client-docker"
+fi
 
 # Building PMM CLient locally (non-CI, i.e. non-Jenkins)
 # total time: 6m26s - build from scratch, no initial cache
@@ -79,20 +114,23 @@ export DOCKER_CLIENT_TAG=local/pmm-client:${GIT_COMMIT}
 
 export RPM_EPOCH=1
 export RPMBUILD_DIST="el9"
-"$PATH_TO_SCRIPTS/build-server-rpm" percona-dashboards grafana-dashboards
-"$PATH_TO_SCRIPTS/build-server-rpm" pmm-managed pmm
-"$PATH_TO_SCRIPTS/build-server-rpm" percona-qan-api2 pmm
-"$PATH_TO_SCRIPTS/build-server-rpm" pmm-update pmm
-"$PATH_TO_SCRIPTS/build-server-rpm" pmm-dump
-"$PATH_TO_SCRIPTS/build-server-rpm" vmproxy pmm
 
-# 3rd-party
-"$PATH_TO_SCRIPTS/build-server-rpm" victoriametrics
-"$PATH_TO_SCRIPTS/build-server-rpm" grafana
+if [ "$NO_SERVER_RPM" -eq 0 ]; then
+  "$PATH_TO_SCRIPTS/build-server-rpm" percona-dashboards grafana-dashboards
+  "$PATH_TO_SCRIPTS/build-server-rpm" pmm-managed pmm
+  "$PATH_TO_SCRIPTS/build-server-rpm" percona-qan-api2 pmm
+  "$PATH_TO_SCRIPTS/build-server-rpm" pmm-update pmm
+  "$PATH_TO_SCRIPTS/build-server-rpm" pmm-dump
+  "$PATH_TO_SCRIPTS/build-server-rpm" vmproxy pmm
+
+  # 3rd-party
+  "$PATH_TO_SCRIPTS/build-server-rpm" victoriametrics
+  "$PATH_TO_SCRIPTS/build-server-rpm" grafana
+fi
 
 export DOCKER_TAG=local/pmm-server:${GIT_COMMIT}
 export RPMBUILD_DIST=el9
-export DOCKERFILE=Dockerfile.el9.local
+export DOCKERFILE=Dockerfile.el9
 ${PATH_TO_SCRIPTS}/build-server-docker
 
 # Clean up temporary files
diff --git a/build/scripts/build-rpmbuild-docker b/build/scripts/build-rpmbuild-docker
deleted file mode 100755
index 77706f3621..0000000000
--- a/build/scripts/build-rpmbuild-docker
+++ /dev/null
@@ -1,11 +0,0 @@
-#!/bin/bash
-
-set -o errexit
-set -o xtrace
-
-. $(dirname $0)/vars
-
-root_dir=$(cd $(dirname $0)/../..; pwd -P)
-docker build --squash -f ${docker_file} -t rpmbuild:local ${root_dir}/build/docker/rpmbuild/
-
-# vim: expandtab shiftwidth=4 tabstop=4
diff --git a/build/scripts/build-server-docker b/build/scripts/build-server-docker
index 421a4dd688..8cb2f70bfc 100755
--- a/build/scripts/build-server-docker
+++ b/build/scripts/build-server-docker
@@ -32,15 +32,15 @@ if [ -z "${DOCKER_TAG}" ]; then
     DOCKER_TAG=perconalab/pmm-server-fb:${full_pmm_version}
 fi
 
-IMAGE_VERSION=`echo $DOCKER_TAG | cut -d ':' -f2`
-
-docker build --build-arg BUILD_DATE="`date --rfc-3339=seconds`" \
-             --build-arg VERSION="$IMAGE_VERSION" \
-             --squash \
-             --no-cache \
-             -f ${docker_root}/${docker_file} \
-             -t ${DOCKER_TAG} \
-             ${docker_root}/
+IMAGE_VERSION=$(echo "$DOCKER_TAG" | cut -d ':' -f2)
+
+docker buildx build \
+    --build-arg BUILD_DATE=$(date --rfc-3339=seconds) \
+    --build-arg VERSION="$IMAGE_VERSION" \
+    --progress plain \
+    -f ${docker_root}/${docker_file} \
+    -t ${DOCKER_TAG} \
+    ${docker_root}
 
 if [ -n "${PUSH_DOCKER}" ]; then
     mkdir -p $(dirname ${docker_tag_file})
diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index 06ae02326e..031aa7707e 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -24,7 +24,7 @@ is_build_needed() {
     local spec_name=$1
     local rpm_version=$2
     local packages=
-    local cache_dir=RELEASE
+    local s3_cache_dir=RELEASE
 
     # Structure of S3 build cache
     # s3://pmm-build-cache/PR-BUILDS/7 - el7
@@ -36,13 +36,13 @@ is_build_needed() {
 
     # RPM_EPOCH is set to 1 for Feature Builds
     if [ "$RPM_EPOCH" = 1 ]; then
-        cache_dir=PR-BUILDS
+        s3_cache_dir=PR-BUILDS
     fi
     if command -v aws &> /dev/null; then
         aws s3 sync \
             --region us-east-2 \
             --no-sign-request \
-            s3://pmm-build-cache/${cache_dir}/${rpmbuild_dist}/${spec_name}-${rpm_version} \
+            s3://pmm-build-cache/${s3_cache_dir}/${rpmbuild_dist}/${spec_name}-${rpm_version} \
             ${rpms_dir}/${spec_name}-${rpm_version}
     fi
 
@@ -102,7 +102,7 @@ build() {
     local rpm_version=$(get_rpm_version "${spec_name}")
     local CH_VERSION=${rpm_version%-*}
     local CH_TAG=${rpm_version#*-}
-    local cache_dir=RELEASE
+    local s3_cache_dir=RELEASE
     local volume_mounts="-v ${rpmbuild_dir}/SOURCES:/home/builder/rpm/SOURCES -v ${rpms_dir}:/home/builder/rpm/RPMS"
 
     if ! is_build_needed "${spec_name}" "${rpm_version}"; then
@@ -170,12 +170,12 @@ build() {
     fi
 
     if [ "$RPM_EPOCH" = 1 ]; then
-      cache_dir=PR-BUILDS
+      s3_cache_dir=PR-BUILDS
     fi
     aws s3 sync \
         --region us-east-2 \
         ${rpms_dir}/${spec_name}-${rpm_version} \
-        s3://pmm-build-cache/${cache_dir}/${rpmbuild_dist}/${spec_name}-${rpm_version} \
+        s3://pmm-build-cache/${s3_cache_dir}/${rpmbuild_dist}/${spec_name}-${rpm_version} \
         || :
 
     echo "Finished building Server RPMs, spec_name: ${spec_name}.spec, repo_name: ${repo_name}"
diff --git a/build/scripts/build-server-rpm-script b/build/scripts/build-server-rpm-script
new file mode 100755
index 0000000000..cece634cdc
--- /dev/null
+++ b/build/scripts/build-server-rpm-script
@@ -0,0 +1,40 @@
+#!/bin/bash
+
+set -o errexit
+set -o xtrace
+
+. $(dirname $0)/vars
+
+sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES /home/builder/.cache
+
+# Add 'Epoch' to spec file to prevent update of rpms which are built in PR build
+if [ ${RPM_EPOCH} = 1 ]; then
+    sed -i '/^Version:.*/i Epoch: 1' /home/builder/rpm/SOURCES/${spec_name}.spec
+fi
+
+rm -rf /home/builder/rpm/RPMS/${spec_name}-*
+
+printf '[local]\nname=local\nbaseurl=file:///home/builder/rpm/RPMS\ngpgcheck=0\nenabled=1\n' \
+    | sudo tee /etc/yum.repos.d/local.repo
+
+until /usr/bin/createrepo_c --update /home/builder/rpm/RPMS; do
+    echo waiting
+    sleep 1
+done
+
+# Only these two specs have build dependencies
+if [[ ${spec_name} =~ ^grafana$|^percona-dashboards$ ]]; then
+    sleep 5s
+    sudo yum-builddep -y SOURCES/${spec_name}.spec
+fi
+
+spectool -C SOURCES -g SOURCES/${spec_name}.spec
+rpmbuild  --define '_rpmdir %{_topdir}/RPMS/${spec_name}-${rpm_version}' \
+          --define 'dist .${rpmbuild_dist}' \
+          --define 'debug_package %{nil}' \
+          -ba SOURCES/${spec_name}.spec
+
+rm -f SOURCES/${spec_name}.spec*
+sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES
+
+# vim: expandtab shiftwidth=4 tabstop=4

From 098e894628c160cd4805ac5f5feb7bdf743ac024 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Mon, 1 Apr 2024 01:31:54 +0000
Subject: [PATCH 10/35] PMM-12899 fix server param evaluation

---
 build/scripts/build-server-docker | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build/scripts/build-server-docker b/build/scripts/build-server-docker
index 8cb2f70bfc..16027c3beb 100755
--- a/build/scripts/build-server-docker
+++ b/build/scripts/build-server-docker
@@ -35,7 +35,7 @@ fi
 IMAGE_VERSION=$(echo "$DOCKER_TAG" | cut -d ':' -f2)
 
 docker buildx build \
-    --build-arg BUILD_DATE=$(date --rfc-3339=seconds) \
+    --build-arg BUILD_DATE="$(date --rfc-3339=seconds)" \
     --build-arg VERSION="$IMAGE_VERSION" \
     --progress plain \
     -f ${docker_root}/${docker_file} \

From aa5729fb413c226baaa908cb05ac7ab19dd69e52 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Mon, 1 Apr 2024 20:08:11 +0000
Subject: [PATCH 11/35] PMM-12899 clean up server RPM specs

---
 build/local/build.sh                          |  6 +--
 build/packages/rpm/server/SPECS/grafana.spec  | 15 +++----
 .../rpm/server/SPECS/percona-dashboards.spec  | 44 ++++++++++--------
 .../rpm/server/SPECS/percona-qan-api2.spec    | 41 +++++++++--------
 build/packages/rpm/server/SPECS/pmm-dump.spec | 16 ++++---
 .../rpm/server/SPECS/pmm-managed.spec         | 45 ++++++++++---------
 .../packages/rpm/server/SPECS/pmm-update.spec | 32 ++++++-------
 .../rpm/server/SPECS/victoriametrics.spec     | 20 ++++-----
 build/packages/rpm/server/SPECS/vmproxy.spec  | 44 +++++++++---------
 build/scripts/vars                            |  2 +-
 10 files changed, 139 insertions(+), 126 deletions(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index 80caaa0d2d..bd174fb965 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -12,7 +12,7 @@ while test "$#" -gt 0; do
       NO_UPDATE=1
       ;;
     --no-client)
-      NO_CLIENT=1
+      NO_CLIENT=1; NO_CLIENT_DOCKER=1
       ;;
     --no-client-docker)
       NO_CLIENT_DOCKER=1
@@ -99,7 +99,7 @@ fi
 # Building client docker image takes 17s
 GIT_COMMIT=$(git rev-parse HEAD | head -c 8)
 export DOCKER_CLIENT_TAG=local/pmm-client:${GIT_COMMIT}
-if [ "$NO_CLIENT_DOCKER" -eq 0 ] || [ "$NO_CLIENT" -eq 0 ]; then
+if [ "$NO_CLIENT_DOCKER" -eq 0 ]; then
   "$PATH_TO_SCRIPTS/build-client-docker"
 fi
 
@@ -113,7 +113,6 @@ fi
 # total time: ??? - subsequent build, using cache from prior builds
 
 export RPM_EPOCH=1
-export RPMBUILD_DIST="el9"
 
 if [ "$NO_SERVER_RPM" -eq 0 ]; then
   "$PATH_TO_SCRIPTS/build-server-rpm" percona-dashboards grafana-dashboards
@@ -129,7 +128,6 @@ if [ "$NO_SERVER_RPM" -eq 0 ]; then
 fi
 
 export DOCKER_TAG=local/pmm-server:${GIT_COMMIT}
-export RPMBUILD_DIST=el9
 export DOCKERFILE=Dockerfile.el9
 ${PATH_TO_SCRIPTS}/build-server-docker
 
diff --git a/build/packages/rpm/server/SPECS/grafana.spec b/build/packages/rpm/server/SPECS/grafana.spec
index a535cf69b1..11e7b246a0 100644
--- a/build/packages/rpm/server/SPECS/grafana.spec
+++ b/build/packages/rpm/server/SPECS/grafana.spec
@@ -1,15 +1,13 @@
-%global debug_package   %{nil}
 %global commit          f283b87d773da4ad61ca65660e72c084ac8f8716
 %global shortcommit     %(c=%{commit}; echo ${c:0:7})
-%define build_timestamp %(date -u +"%y%m%d%H%M")
-%define release         104
+%define release         105
 %define grafana_version 10.4.0
 %define full_pmm_version 2.0.0
 %define full_version    v%{grafana_version}-%{full_pmm_version}
-%define rpm_release     %{release}.%{build_timestamp}.%{shortcommit}%{?dist}
+%define rpm_release     %{release}.%{shortcommit}%{?dist}
 
 %if ! 0%{?gobuild:1}
-%define gobuild(o:) go build -ldflags "${LDFLAGS:-} -B 0x$(head -c20 /dev/urandom|od -An -tx1|tr -d ' \\n')" -a -v -x %{?**};
+%define gobuild(o:) go build -ldflags "${LDFLAGS:-} -B 0x$(head -c20 /dev/urandom | od -An -tx1 | tr -d ' \\n')" -a -v -x %{?**};
 %endif
 
 Name:           percona-grafana
@@ -24,8 +22,9 @@ ExclusiveArch:  %{ix86} x86_64 %{arm}
 BuildRequires: fontconfig
 
 %description
-Grafana is an open source, feature rich metrics dashboard and graph editor for
-Graphite, InfluxDB & OpenTSDB.
+Grafana is an open source observability and data visualization platform.
+Visualize metrics, logs, and traces from multiple sources like
+Prometheus, Loki, Elasticsearch, InfluxDB, Postgres and many more.
 
 %prep
 %setup -q -n grafana-%{commit}
@@ -77,7 +76,7 @@ getent passwd pmm >/dev/null || echo "User pmm does not exist. Please create it
 exit 0
 
 %changelog
-* Wed Mar 20 2024 Alex Demidoff <alexander.demidoff@percona.com> - 10.4.0-2
+* Wed Mar 20 2024 Alex Demidoff <alexander.demidoff@percona.com> - 10.4.0-105
 - PMM-12899 Use module and build cache
 
 * Tue Mar 12 2024 Matej Kubinec <matej.kubinec@ext.percona.com> - 10.4.0-1
diff --git a/build/packages/rpm/server/SPECS/percona-dashboards.spec b/build/packages/rpm/server/SPECS/percona-dashboards.spec
index 5a1a895ed3..e356038179 100644
--- a/build/packages/rpm/server/SPECS/percona-dashboards.spec
+++ b/build/packages/rpm/server/SPECS/percona-dashboards.spec
@@ -1,27 +1,30 @@
-%global debug_package   %{nil}
 %global __strip         /bin/true
 
-%global repo		grafana-dashboards
-%global provider	github.com/percona/%{repo}
-%global import_path	%{provider}
-%global commit		ad4af6808bcd361284e8eb8cd1f36b1e98e32bce
-%global shortcommit	%(c=%{commit}; echo ${c:0:7})
-%define build_timestamp %(date -u +"%y%m%d%H%M")
-%define release         21
-%define rpm_release     %{release}.%{build_timestamp}.%{shortcommit}%{?dist}
-
-Name:		percona-dashboards
-Version:	%{version}
-Release:	%{rpm_release}
-Summary:	Grafana dashboards for monitoring
-
-License:	AGPLv3
-URL:		https://%{provider}
-Source0:	https://%{provider}/archive/%{commit}/%{repo}-%{shortcommit}.tar.gz
+%global repo            grafana-dashboards
+%global provider        github.com/percona/%{repo}
+%global import_path     %{provider}
+%global commit          ad4af6808bcd361284e8eb8cd1f36b1e98e32bce
+%global shortcommit     %(c=%{commit}; echo ${c:0:7})
+%define release         22
+%define rpm_release     %{release}.%{shortcommit}%{?dist}
+
+%if ! 0%{?gobuild:1}
+# https://github.com/rpm-software-management/rpm/issues/367
+# https://fedoraproject.org/wiki/PackagingDrafts/Go#Build_ID
+%define gobuild(o:) go build -ldflags "${LDFLAGS:-} -B 0x$(head -c20 /dev/urandom | od -An -tx1 | tr -d ' \\n')" -a -v -x %{?**};
+%endif
+
+Name:     percona-dashboards
+Version:  %{version}
+Release:  %{rpm_release}
+Summary:  Grafana dashboards for monitoring
+
+License:  AGPLv3
+URL:      https://%{provider}
+Source0:  https://%{provider}/archive/%{commit}/%{repo}-%{shortcommit}.tar.gz
 
 BuildRequires:	nodejs
 Requires:	percona-grafana
-Provides:	percona-grafana-dashboards = %{version}-%{release}
 
 %description
 This is a set of Grafana dashboards for database and system monitoring
@@ -57,6 +60,9 @@ echo %{version} > %{buildroot}%{_datadir}/%{name}/VERSION
 
 
 %changelog
+* Wed Mar 20 2024 Alex Demidoff <alexander.demidoff@percona.com> - 3.0.0-22
+- PMM-12899 Use module and build cache
+
 * Wed Nov 29 2023 Alex Demidoff <alexander.demidoff@percona.com> - 3.0.0-21
 - PMM-12693 Run Grafana as non-root user
 
diff --git a/build/packages/rpm/server/SPECS/percona-qan-api2.spec b/build/packages/rpm/server/SPECS/percona-qan-api2.spec
index 14e89677b5..ae33484d71 100644
--- a/build/packages/rpm/server/SPECS/percona-qan-api2.spec
+++ b/build/packages/rpm/server/SPECS/percona-qan-api2.spec
@@ -1,8 +1,3 @@
-# Go build id is not supported for now.
-# https://github.com/rpm-software-management/rpm/issues/367
-# https://bugzilla.redhat.com/show_bug.cgi?id=1295951
-%undefine _missing_build_ids_terminate_build
-
 %global repo            pmm
 %global provider        github.com/percona/%{repo}
 %global import_path     %{provider}
@@ -10,17 +5,22 @@
 # see: https://github.com/percona/pmm/blob/main/build/scripts/build-server-rpm#L58
 %global commit          0000000000000000000000000000000000000000
 %global shortcommit     %(c=%{commit}; echo ${c:0:7})
-%define build_timestamp %(date -u +"%y%m%d%H%M")
-%define release         17
-%define rpm_release     %{release}.%{build_timestamp}.%{shortcommit}%{?dist}
+%define release         18
+%define rpm_release     %{release}.%{shortcommit}%{?dist}
 
 # the line below is sed'ed by build/bin/build-server-rpm to set a correct version
 %define full_pmm_version 2.0.0
 
-Name:           percona-qan-api2
+%if ! 0%{?gobuild:1}
+# https://github.com/rpm-software-management/rpm/issues/367
+# https://fedoraproject.org/wiki/PackagingDrafts/Go#Build_ID
+%define gobuild(o:) go build -ldflags "${LDFLAGS:-} -B 0x$(head -c20 /dev/urandom | od -An -tx1 | tr -d ' \\n')" -a -v -x %{?**};
+%endif
+
+Name:           pmm-qan
 Version:        %{version}
 Release:        %{rpm_release}
-Summary:        Query Analytics API v2 for PMM
+Summary:        Query Analytics API for PMM
 
 License:        AGPLv3
 URL:            https://%{provider}
@@ -32,10 +32,7 @@ See PMM docs for more information - https://docs.percona.com/percona-monitoring-
 
 
 %prep
-%setup -T -c -n %{repo}-%{version}
-%setup -q -c -a 0 -n %{repo}-%{version}
-mkdir -p src/github.com/percona
-mv %{repo}-%{commit} src/%{provider}
+%setup -q -n %{repo}-%{commit}
 
 
 %build
@@ -43,29 +40,31 @@ export PMM_RELEASE_VERSION=%{full_pmm_version}
 export PMM_RELEASE_FULLCOMMIT=%{commit}
 export PMM_RELEASE_BRANCH=""
 
-cd src/%{provider}/qan-api2
-make release
+make -C qan-api2 release
 
 
 %install
 
 install -d -p %{buildroot}%{_sbindir}
-install -p -m 0755 src/%{provider}/bin/qan-api2 %{buildroot}%{_sbindir}/%{name}
+install -p -m 0755 ./bin/qan-api2 %{buildroot}%{_sbindir}/%{name}
 
 
 %files
 %attr(0755, root, root) %{_sbindir}/%{name}
-%license src/%{provider}/qan-api2/LICENSE
-%doc src/%{provider}/qan-api2/README.md
+%license qan-api2/LICENSE
+%doc qan-api2/README.md
 
 %changelog
-* Mon Nov  7 2022 Alexander Tymchuk <alexander.tymchuk@percona.com> - 2.0.0-17
+* Wed Apr 1 2024 Alex Demidoff <alexander.demidoff@percona.com> - 3.0.0-18
+- PMM-12899 Use module and build cache
+
+* Mon Nov 7 2022 Alexander Tymchuk <alexander.tymchuk@percona.com> - 2.0.0-17
 - PMM-10117 migrate QAN API to monorepo
 
 * Mon May 16 2022 Nikita Beletskii <nikita.beletskii@percona.com> - 2.0.0-16
 - PMM-10027 remove useless packages
 
-* Thu Jul  2 2020 Mykyta Solomko <mykyta.solomko@percona.com> - 2.0.0-15
+* Thu Jul 2 2020 Mykyta Solomko <mykyta.solomko@percona.com> - 2.0.0-15
 - PMM-5645 built using Golang 1.14
 
 * Tue Mar 19 2019 Vadim Yalovets <vadim.yalovets@percona.com> - 2.0.0-4
diff --git a/build/packages/rpm/server/SPECS/pmm-dump.spec b/build/packages/rpm/server/SPECS/pmm-dump.spec
index c8882aa25b..e5ddbee367 100644
--- a/build/packages/rpm/server/SPECS/pmm-dump.spec
+++ b/build/packages/rpm/server/SPECS/pmm-dump.spec
@@ -1,12 +1,15 @@
-%undefine _missing_build_ids_terminate_build
-
 %global repo            pmm-dump
 %global provider        github.com/percona/%{repo}
 %global commit          0d49b27729506dc62950f9fa59147d63df194db2
 %global shortcommit     %(c=%{commit}; echo ${c:0:7})
-%define build_timestamp %(date -u +"%y%m%d%H%M")
-%define release         1
-%define rpm_release     %{release}.%{build_timestamp}.%{shortcommit}%{?dist}
+%define release         2
+%define rpm_release     %{release}.%{shortcommit}%{?dist}
+
+%if ! 0%{?gobuild:1}
+# https://github.com/rpm-software-management/rpm/issues/367
+# https://fedoraproject.org/wiki/PackagingDrafts/Go#Build_ID
+%define gobuild(o:) go build -ldflags "${LDFLAGS:-} -B 0x$(head -c20 /dev/urandom | od -An -tx1 | tr -d ' \\n')" -a -v -x %{?**};
+%endif
 
 Name:		pmm-dump
 Version:	0.7.0
@@ -37,6 +40,9 @@ install -p -m 0755 pmm-dump %{buildroot}%{_sbindir}/pmm-dump
 
 
 %changelog
+* Mon Apr 1 2024 Alex Demidoff <alexander.demidoff@percona.com> - 0.7.0-2
+- PMM-12899 Use module and build cache
+
 * Tue Nov 23 2023 Artem Gavrilov <artem.gavrilov@percona.com> - 0.7.0-ga
 - PMM-12460 Update pmm-dump to v0.7.0-ga version
 
diff --git a/build/packages/rpm/server/SPECS/pmm-managed.spec b/build/packages/rpm/server/SPECS/pmm-managed.spec
index b313be8ba8..d9cc53019c 100644
--- a/build/packages/rpm/server/SPECS/pmm-managed.spec
+++ b/build/packages/rpm/server/SPECS/pmm-managed.spec
@@ -1,25 +1,29 @@
-%undefine _missing_build_ids_terminate_build
 %global _dwz_low_mem_die_limit 0
 
 %global repo            pmm
 %global provider        github.com/percona/%{repo}
 %global commit          8f3d007617941033867aea6a134c48b39142427f
 %global shortcommit     %(c=%{commit}; echo ${c:0:7})
-%define build_timestamp %(date -u +"%y%m%d%H%M")
-%define release         20
-%define rpm_release     %{release}.%{build_timestamp}.%{shortcommit}%{?dist}
+%define release         21
+%define rpm_release     %{release}.%{shortcommit}%{?dist}
 
 # the line below is sed'ed by build/bin/build-server-rpm to set a correct version
 %define full_pmm_version 2.0.0
 
-Name:		pmm-managed
-Version:	%{version}
-Release:	%{rpm_release}
-Summary:	Percona Monitoring and Management management daemon
+%if ! 0%{?gobuild:1}
+# https://github.com/rpm-software-management/rpm/issues/367
+# https://fedoraproject.org/wiki/PackagingDrafts/Go#Build_ID
+%define gobuild(o:) go build -ldflags "${LDFLAGS:-} -B 0x$(head -c20 /dev/urandom | od -An -tx1 | tr -d ' \\n')" -a -v -x %{?**};
+%endif
 
-License:	AGPLv3
-URL:		  https://%{provider}
-Source0:	https://%{provider}/archive/%{commit}/%{repo}-%{shortcommit}.tar.gz
+Name:     pmm-managed
+Version:  %{version}
+Release:  %{rpm_release}
+Summary:  Percona Monitoring and Management management daemon
+
+License:  AGPLv3
+URL:      https://%{provider}
+Source0:  https://%{provider}/archive/%{commit}/%{repo}-%{shortcommit}.tar.gz
 
 %description
 pmm-managed manages configuration of PMM server components (VictoriaMetrics,
@@ -28,19 +32,14 @@ See PMM docs for more information.
 
 
 %prep
-%setup -q -n pmm-%{commit}
-mkdir -p src/github.com/percona
-ln -s $(pwd) src/%{provider}
-
+%setup -q -n %{repo}-%{commit}
 
 %build
-
 export PMM_RELEASE_VERSION=%{full_pmm_version}
 export PMM_RELEASE_FULLCOMMIT=%{commit}
 export PMM_RELEASE_BRANCH=""
 
-cd src/github.com/percona/pmm/managed
-make release
+make -C managed release
 
 
 %install
@@ -51,13 +50,12 @@ install -p -m 0755 bin/pmm-managed %{buildroot}%{_sbindir}/pmm-managed
 install -p -m 0755 bin/pmm-managed-init %{buildroot}%{_sbindir}/pmm-managed-init
 install -p -m 0755 bin/pmm-managed-starlark %{buildroot}%{_sbindir}/pmm-managed-starlark
 
-cd src/github.com/percona/pmm
-cp -pa ./api/swagger %{buildroot}%{_datadir}/%{name}
+cp -pa api/swagger %{buildroot}%{_datadir}/%{name}
 
 
 %files
-%license src/%{provider}/LICENSE
-%doc src/%{provider}/README.md
+%license managed/LICENSE
+%doc managed/README.md
 %{_sbindir}/pmm-managed
 %{_sbindir}/pmm-managed-init
 %{_sbindir}/pmm-managed-starlark
@@ -65,6 +63,9 @@ cp -pa ./api/swagger %{buildroot}%{_datadir}/%{name}
 
 
 %changelog
+* Mon Apr 1 2024 Alex Demidoff <alexander.demidoff@percona.com> - 3.0.0-21
+- PMM-12899 Use module and build cache
+
 * Thu Jul 28 2022 Alex Tymchuk <alexander.tymchuk@percona.com> - 2.30.0-1
 - PMM-10036 migrate to monorepo
 
diff --git a/build/packages/rpm/server/SPECS/pmm-update.spec b/build/packages/rpm/server/SPECS/pmm-update.spec
index e68fc9f55d..9a9a5eac21 100644
--- a/build/packages/rpm/server/SPECS/pmm-update.spec
+++ b/build/packages/rpm/server/SPECS/pmm-update.spec
@@ -1,5 +1,3 @@
-%undefine _missing_build_ids_terminate_build
-
 # TODO: remove it as soon as we remove all noarch pmm-update rpms
 # from 'pmm3-components/yum/laboratory'
 %define _binaries_in_noarch_packages_terminate_build   0
@@ -10,20 +8,25 @@
 %global import_path     %{provider}
 %global commit	        592eddf656bce32a11bd958af0a32c62bd5ea34c
 %global shortcommit	    %(c=%{commit}; echo ${c:0:7})
-%define build_timestamp %(date -u +"%y%m%d%H%M")
-%define release         67
-%define rpm_release     %{release}.%{build_timestamp}.%{shortcommit}%{?dist}
+%define release         68
+%define rpm_release     %{release}.%{shortcommit}%{?dist}
 
 # the line below is sed'ed by build/bin/build-server-rpm to set a correct version
 %define full_pmm_version 2.0.0
 
-Name:		pmm-update
+%if ! 0%{?gobuild:1}
+# https://github.com/rpm-software-management/rpm/issues/367
+# https://fedoraproject.org/wiki/PackagingDrafts/Go#Build_ID
+%define gobuild(o:) go build -ldflags "${LDFLAGS:-} -B 0x$(head -c20 /dev/urandom | od -An -tx1 | tr -d ' \\n')" -a -v -x %{?**};
+%endif
+
+Name:		  pmm-update
 Version:	%{full_pmm_version}
 Release:	%{rpm_release}
 Summary:	Tool for updating packages and OS configuration for PMM Server
 
 License:	AGPLv3
-URL:		https://%{provider}
+URL:		  https://%{provider}
 Source0:	https://%{provider}/archive/%{commit}/%{repo}-%{shortcommit}.tar.gz
 
 BuildArch:	noarch
@@ -34,16 +37,13 @@ BuildArch:	noarch
 
 %prep
 %setup -q -n %{repo}-%{commit}
-mkdir -p src/github.com/percona
-ln -s $(pwd) src/%{provider}
 
 %build
 export PMM_RELEASE_VERSION=%{full_pmm_version}
 export PMM_RELEASE_FULLCOMMIT=%{commit}
 export PMM_RELEASE_BRANCH=""
 
-cd src/github.com/percona/pmm/update
-make release
+make -C update release
 
 
 %install
@@ -51,18 +51,20 @@ install -d %{buildroot}%{_datadir}/%{name}
 cp -pav ./update/ansible %{buildroot}%{_datadir}/%{name}
 
 install -d %{buildroot}%{_sbindir}
-cd src/github.com/percona/pmm/update
-install -p -m 0755 bin/pmm-update %{buildroot}%{_sbindir}/
+install -p -m 0755 update/bin/pmm-update %{buildroot}%{_sbindir}/
 
 
 %files
-%license LICENSE
-%doc README.md
+%license update/LICENSE
+%doc update/README.md
 %{_sbindir}/pmm-update
 %{_datadir}/%{name}
 
 
 %changelog
+* Mon Apr 1 2024 Alex Demidoff <alexander.demidoff@percona.com> - 3.0.0-68
+- PMM-12899 Use module and build cache
+
 * Thu Dec 8 2022 Michal Kralik <michal.kralik@percona.com> - 2.34.0-67
 - PMM-11207 Migrate pmm-update to monorepo
 
diff --git a/build/packages/rpm/server/SPECS/victoriametrics.spec b/build/packages/rpm/server/SPECS/victoriametrics.spec
index ec55d239d1..585a35954a 100644
--- a/build/packages/rpm/server/SPECS/victoriametrics.spec
+++ b/build/packages/rpm/server/SPECS/victoriametrics.spec
@@ -1,19 +1,16 @@
-%undefine _missing_build_ids_terminate_build
-
-%define copying() \
-%if 0%{?fedora} >= 21 || 0%{?rhel} >= 7 \
-%license %{*} \
-%else \
-%doc %{*} \
-%endif
-
 %global repo            VictoriaMetrics
 %global provider        github.com/VictoriaMetrics/%{repo}
 %global commit          pmm-6401-v1.93.4
 
+%if ! 0%{?gobuild:1}
+# https://github.com/rpm-software-management/rpm/issues/367
+# https://fedoraproject.org/wiki/PackagingDrafts/Go#Build_ID
+%define gobuild(o:) go build -ldflags "${LDFLAGS:-} -B 0x$(head -c20 /dev/urandom | od -An -tx1 | tr -d ' \\n')" -a -v -x %{?**};
+%endif
+
 Name:           percona-victoriametrics
 Version:        1.93.4
-Release:        1%{?dist}
+Release:        2%{?dist}
 Summary:        VictoriaMetrics monitoring solution and time series database
 License:        Apache-2.0
 URL:            https://%{provider}
@@ -50,6 +47,9 @@ install -D -p -m 0755 ./bin/vmalert-pure %{buildroot}%{_sbindir}/vmalert
 
 
 %changelog
+* Mon Apr 1 2024 Alex Demidoff <alexander.demidoff@percona.com> - 1.93.4-2
+- PMM-12899 Use module and build cache
+
 * Thu Sep 14 2023 Alex Tymchuk <alexander.tymchuk@percona.com> - 1.93.4-1
 - upgrade victoriametrics to 1.93.4 release
 
diff --git a/build/packages/rpm/server/SPECS/vmproxy.spec b/build/packages/rpm/server/SPECS/vmproxy.spec
index be5cdca4ca..fe4100e2d8 100644
--- a/build/packages/rpm/server/SPECS/vmproxy.spec
+++ b/build/packages/rpm/server/SPECS/vmproxy.spec
@@ -1,25 +1,29 @@
-%undefine _missing_build_ids_terminate_build
 %global _dwz_low_mem_die_limit 0
 
 %global repo            pmm
 %global provider        github.com/percona/%{repo}
 %global commit          8f74cea10d85e441ee88ef4b12bc47bc05165ba9
 %global shortcommit     %(c=%{commit}; echo ${c:0:7})
-%define build_timestamp %(date -u +"%y%m%d%H%M")
-%define release         1
-%define rpm_release     %{release}.%{build_timestamp}.%{shortcommit}%{?dist}
+%define release         2
+%define rpm_release     %{release}.%{shortcommit}%{?dist}
 
 # the line below is sed'ed by build/bin/build-server-rpm to set a correct version
 %define full_pmm_version 2.0.0
 
-Name:		vmproxy
-Version:	%{full_pmm_version}
-Release:	%{rpm_release}
-Summary:	Percona VMProxy stateless reverse proxy for VictoriaMetrics
+%if ! 0%{?gobuild:1}
+# https://github.com/rpm-software-management/rpm/issues/367
+# https://fedoraproject.org/wiki/PackagingDrafts/Go#Build_ID
+%define gobuild(o:) go build -ldflags "${LDFLAGS:-} -B 0x$(head -c20 /dev/urandom | od -An -tx1 | tr -d ' \\n')" -a -v -x %{?**};
+%endif
 
-License:	AGPLv3
-URL:		https://%{provider}
-Source0:	https://%{provider}/archive/%{commit}/%{repo}-%{shortcommit}.tar.gz
+Name:     vmproxy
+Version:  %{full_pmm_version}
+Release:  %{rpm_release}
+Summary:  Percona VMProxy stateless reverse proxy for VictoriaMetrics
+
+License:  AGPLv3
+URL:      https://%{provider}
+Source0:  https://%{provider}/archive/%{commit}/%{repo}-%{shortcommit}.tar.gz
 
 %description
 VMProxy is a stateless reverse proxy which proxies requests to VictoriaMetrics and
@@ -27,33 +31,31 @@ optionally adds `extra_filters` query based on the provided configuration.
 
 
 %prep
-%setup -q -n pmm-%{commit}
-mkdir -p src/github.com/percona
-ln -s $(pwd) src/%{provider}
+%setup -q -n %{repo}-%{commit}
 
 
 %build
-
 export PMM_RELEASE_VERSION=%{full_pmm_version}
 export PMM_RELEASE_FULLCOMMIT=%{commit}
 export PMM_RELEASE_BRANCH=""
 
-cd src/github.com/percona/pmm/vmproxy
-make release
+make -C vmproxy release
 
 
 %install
-install -d -p %{buildroot}%{_bindir}
 install -d -p %{buildroot}%{_sbindir}
-install -p -m 0755 bin/vmproxy %{buildroot}%{_sbindir}/vmproxy
+install -p -m 0755 vmproxy/bin/vmproxy %{buildroot}%{_sbindir}/vmproxy
 
 
 %files
-%license src/%{provider}/vmproxy/LICENSE
-%doc src/%{provider}/vmproxy/README.md
+%license vmproxy/LICENSE
+%doc vmproxy/README.md
 %{_sbindir}/vmproxy
 
 
 %changelog
+* Mon Apr 1 2024 Alex Demidoff <alexander.demidoff@percona.com> - 3.0.0-2
+- PMM-12899 Use module and build cache
+
 * Mon Dec 5 2022 Michal Kralik <michal.kralik@percona.com> - 2.34.0-1
 - Initial release of VMProxy
diff --git a/build/scripts/vars b/build/scripts/vars
index ea72294352..de57bfb8a4 100644
--- a/build/scripts/vars
+++ b/build/scripts/vars
@@ -41,7 +41,7 @@ echo -e "\n\n\n>>> full_pmm_version=${full_pmm_version} pmm_version=${pmm_versio
 rpmbuild_docker_image=${RPMBUILD_DOCKER_IMAGE:-public.ecr.aws/e7j3v3n0/rpmbuild:2}
 rpms_dir=${root_dir}/tmp/pmm-server/RPMS
 rpmbuild_dir=${root_dir}/sources/pmm/src/github.com/percona/pmm/build/packages/rpm/server
-rpmbuild_dist=${RPMBUILD_DIST:-"el7"}
+rpmbuild_dist=${RPMBUILD_DIST:-"el9"}
 source_dir=${root_dir}/tmp/source/pmm-client-${pmm_version}
 binary_dir=${root_dir}/tmp/binary/pmm-client-${pmm_version}
 client_properties=${root_dir}/results/pmm-client.properties

From aa5e8b82cd3cfa621be875d28df3ee6ab05d5df7 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Mon, 1 Apr 2024 20:16:41 +0000
Subject: [PATCH 12/35] PMM-12899 fix directory permissions

---
 build/scripts/build-server-rpm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index 031aa7707e..ff5ab06553 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -128,7 +128,7 @@ build() {
         set -o errexit
         set -o xtrace
 
-        sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES /home/builder/.cache
+        sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES /home/builder/.cache /home/builder/go
 
         # Add 'Epoch' to spec file to prevent update of rpms which are built in PR build
         if [ ${RPM_EPOCH} = 1 ]; then

From ee45fd6aadb70d8bc59d4aa5b9eee695b4723572 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Mon, 1 Apr 2024 21:49:29 +0000
Subject: [PATCH 13/35] PMM-12899 remove unnecessary macro

---
 build/packages/rpm/server/SPECS/grafana.spec  |  2 ++
 .../rpm/server/SPECS/percona-dashboards.spec  |  6 -----
 .../rpm/server/SPECS/percona-qan-api2.spec    |  6 -----
 build/packages/rpm/server/SPECS/pmm-dump.spec | 24 ++++++++-----------
 .../rpm/server/SPECS/pmm-managed.spec         |  7 +-----
 .../packages/rpm/server/SPECS/pmm-update.spec |  7 +-----
 .../rpm/server/SPECS/victoriametrics.spec     |  8 ++-----
 build/packages/rpm/server/SPECS/vmproxy.spec  |  9 ++-----
 8 files changed, 18 insertions(+), 51 deletions(-)

diff --git a/build/packages/rpm/server/SPECS/grafana.spec b/build/packages/rpm/server/SPECS/grafana.spec
index 11e7b246a0..e4c3e1558d 100644
--- a/build/packages/rpm/server/SPECS/grafana.spec
+++ b/build/packages/rpm/server/SPECS/grafana.spec
@@ -7,6 +7,8 @@
 %define rpm_release     %{release}.%{shortcommit}%{?dist}
 
 %if ! 0%{?gobuild:1}
+# https://github.com/rpm-software-management/rpm/issues/367
+# https://fedoraproject.org/wiki/PackagingDrafts/Go#Build_ID
 %define gobuild(o:) go build -ldflags "${LDFLAGS:-} -B 0x$(head -c20 /dev/urandom | od -An -tx1 | tr -d ' \\n')" -a -v -x %{?**};
 %endif
 
diff --git a/build/packages/rpm/server/SPECS/percona-dashboards.spec b/build/packages/rpm/server/SPECS/percona-dashboards.spec
index e356038179..d52adfdddf 100644
--- a/build/packages/rpm/server/SPECS/percona-dashboards.spec
+++ b/build/packages/rpm/server/SPECS/percona-dashboards.spec
@@ -8,12 +8,6 @@
 %define release         22
 %define rpm_release     %{release}.%{shortcommit}%{?dist}
 
-%if ! 0%{?gobuild:1}
-# https://github.com/rpm-software-management/rpm/issues/367
-# https://fedoraproject.org/wiki/PackagingDrafts/Go#Build_ID
-%define gobuild(o:) go build -ldflags "${LDFLAGS:-} -B 0x$(head -c20 /dev/urandom | od -An -tx1 | tr -d ' \\n')" -a -v -x %{?**};
-%endif
-
 Name:     percona-dashboards
 Version:  %{version}
 Release:  %{rpm_release}
diff --git a/build/packages/rpm/server/SPECS/percona-qan-api2.spec b/build/packages/rpm/server/SPECS/percona-qan-api2.spec
index ae33484d71..46995118b1 100644
--- a/build/packages/rpm/server/SPECS/percona-qan-api2.spec
+++ b/build/packages/rpm/server/SPECS/percona-qan-api2.spec
@@ -11,12 +11,6 @@
 # the line below is sed'ed by build/bin/build-server-rpm to set a correct version
 %define full_pmm_version 2.0.0
 
-%if ! 0%{?gobuild:1}
-# https://github.com/rpm-software-management/rpm/issues/367
-# https://fedoraproject.org/wiki/PackagingDrafts/Go#Build_ID
-%define gobuild(o:) go build -ldflags "${LDFLAGS:-} -B 0x$(head -c20 /dev/urandom | od -An -tx1 | tr -d ' \\n')" -a -v -x %{?**};
-%endif
-
 Name:           pmm-qan
 Version:        %{version}
 Release:        %{rpm_release}
diff --git a/build/packages/rpm/server/SPECS/pmm-dump.spec b/build/packages/rpm/server/SPECS/pmm-dump.spec
index e5ddbee367..b5b4edad32 100644
--- a/build/packages/rpm/server/SPECS/pmm-dump.spec
+++ b/build/packages/rpm/server/SPECS/pmm-dump.spec
@@ -1,3 +1,5 @@
+%undefine _missing_build_ids_terminate_build
+
 %global repo            pmm-dump
 %global provider        github.com/percona/%{repo}
 %global commit          0d49b27729506dc62950f9fa59147d63df194db2
@@ -5,20 +7,14 @@
 %define release         2
 %define rpm_release     %{release}.%{shortcommit}%{?dist}
 
-%if ! 0%{?gobuild:1}
-# https://github.com/rpm-software-management/rpm/issues/367
-# https://fedoraproject.org/wiki/PackagingDrafts/Go#Build_ID
-%define gobuild(o:) go build -ldflags "${LDFLAGS:-} -B 0x$(head -c20 /dev/urandom | od -An -tx1 | tr -d ' \\n')" -a -v -x %{?**};
-%endif
-
-Name:		pmm-dump
-Version:	0.7.0
-Release:	%{rpm_release}
-Summary:	Percona PMM Dump allows to export and import monitoring metrics and query analytics.
+Name:     pmm-dump
+Version:  0.7.0
+Release:  %{rpm_release}
+Summary:  Percona PMM Dump allows to export and import monitoring metrics and query analytics.
 
-License:	AGPLv3
-URL:		https://%{provider}
-Source0:	https://%{provider}/archive/%{commit}.tar.gz
+License:  AGPLv3
+URL:      https://%{provider}
+Source0:  https://%{provider}/archive/%{commit}.tar.gz
 
 %description
 %{summary}
@@ -43,7 +39,7 @@ install -p -m 0755 pmm-dump %{buildroot}%{_sbindir}/pmm-dump
 * Mon Apr 1 2024 Alex Demidoff <alexander.demidoff@percona.com> - 0.7.0-2
 - PMM-12899 Use module and build cache
 
-* Tue Nov 23 2023 Artem Gavrilov <artem.gavrilov@percona.com> - 0.7.0-ga
+* Thu Nov 23 2023 Artem Gavrilov <artem.gavrilov@percona.com> - 0.7.0-ga
 - PMM-12460 Update pmm-dump to v0.7.0-ga version
 
 * Tue Mar 29 2022 Alex Tymchuk <alexander.tymchuk@percona.com> - 0.6.0-1
diff --git a/build/packages/rpm/server/SPECS/pmm-managed.spec b/build/packages/rpm/server/SPECS/pmm-managed.spec
index d9cc53019c..02f1578043 100644
--- a/build/packages/rpm/server/SPECS/pmm-managed.spec
+++ b/build/packages/rpm/server/SPECS/pmm-managed.spec
@@ -1,3 +1,4 @@
+%undefine _missing_build_ids_terminate_build
 %global _dwz_low_mem_die_limit 0
 
 %global repo            pmm
@@ -10,12 +11,6 @@
 # the line below is sed'ed by build/bin/build-server-rpm to set a correct version
 %define full_pmm_version 2.0.0
 
-%if ! 0%{?gobuild:1}
-# https://github.com/rpm-software-management/rpm/issues/367
-# https://fedoraproject.org/wiki/PackagingDrafts/Go#Build_ID
-%define gobuild(o:) go build -ldflags "${LDFLAGS:-} -B 0x$(head -c20 /dev/urandom | od -An -tx1 | tr -d ' \\n')" -a -v -x %{?**};
-%endif
-
 Name:     pmm-managed
 Version:  %{version}
 Release:  %{rpm_release}
diff --git a/build/packages/rpm/server/SPECS/pmm-update.spec b/build/packages/rpm/server/SPECS/pmm-update.spec
index 9a9a5eac21..cb048070c3 100644
--- a/build/packages/rpm/server/SPECS/pmm-update.spec
+++ b/build/packages/rpm/server/SPECS/pmm-update.spec
@@ -1,3 +1,4 @@
+%undefine _missing_build_ids_terminate_build
 # TODO: remove it as soon as we remove all noarch pmm-update rpms
 # from 'pmm3-components/yum/laboratory'
 %define _binaries_in_noarch_packages_terminate_build   0
@@ -14,12 +15,6 @@
 # the line below is sed'ed by build/bin/build-server-rpm to set a correct version
 %define full_pmm_version 2.0.0
 
-%if ! 0%{?gobuild:1}
-# https://github.com/rpm-software-management/rpm/issues/367
-# https://fedoraproject.org/wiki/PackagingDrafts/Go#Build_ID
-%define gobuild(o:) go build -ldflags "${LDFLAGS:-} -B 0x$(head -c20 /dev/urandom | od -An -tx1 | tr -d ' \\n')" -a -v -x %{?**};
-%endif
-
 Name:		  pmm-update
 Version:	%{full_pmm_version}
 Release:	%{rpm_release}
diff --git a/build/packages/rpm/server/SPECS/victoriametrics.spec b/build/packages/rpm/server/SPECS/victoriametrics.spec
index 585a35954a..97e2b39a78 100644
--- a/build/packages/rpm/server/SPECS/victoriametrics.spec
+++ b/build/packages/rpm/server/SPECS/victoriametrics.spec
@@ -1,13 +1,9 @@
+%undefine _missing_build_ids_terminate_build
+
 %global repo            VictoriaMetrics
 %global provider        github.com/VictoriaMetrics/%{repo}
 %global commit          pmm-6401-v1.93.4
 
-%if ! 0%{?gobuild:1}
-# https://github.com/rpm-software-management/rpm/issues/367
-# https://fedoraproject.org/wiki/PackagingDrafts/Go#Build_ID
-%define gobuild(o:) go build -ldflags "${LDFLAGS:-} -B 0x$(head -c20 /dev/urandom | od -An -tx1 | tr -d ' \\n')" -a -v -x %{?**};
-%endif
-
 Name:           percona-victoriametrics
 Version:        1.93.4
 Release:        2%{?dist}
diff --git a/build/packages/rpm/server/SPECS/vmproxy.spec b/build/packages/rpm/server/SPECS/vmproxy.spec
index fe4100e2d8..11ca103785 100644
--- a/build/packages/rpm/server/SPECS/vmproxy.spec
+++ b/build/packages/rpm/server/SPECS/vmproxy.spec
@@ -1,3 +1,4 @@
+%undefine _missing_build_ids_terminate_build
 %global _dwz_low_mem_die_limit 0
 
 %global repo            pmm
@@ -10,12 +11,6 @@
 # the line below is sed'ed by build/bin/build-server-rpm to set a correct version
 %define full_pmm_version 2.0.0
 
-%if ! 0%{?gobuild:1}
-# https://github.com/rpm-software-management/rpm/issues/367
-# https://fedoraproject.org/wiki/PackagingDrafts/Go#Build_ID
-%define gobuild(o:) go build -ldflags "${LDFLAGS:-} -B 0x$(head -c20 /dev/urandom | od -An -tx1 | tr -d ' \\n')" -a -v -x %{?**};
-%endif
-
 Name:     vmproxy
 Version:  %{full_pmm_version}
 Release:  %{rpm_release}
@@ -44,7 +39,7 @@ make -C vmproxy release
 
 %install
 install -d -p %{buildroot}%{_sbindir}
-install -p -m 0755 vmproxy/bin/vmproxy %{buildroot}%{_sbindir}/vmproxy
+install -p -m 0755 ./bin/vmproxy %{buildroot}%{_sbindir}/vmproxy
 
 
 %files

From afde65ad8151a097eec42d8a80f4f1671c58686f Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Mon, 1 Apr 2024 22:24:15 +0000
Subject: [PATCH 14/35] PMM-12899 revert QAN package name

---
 build/packages/rpm/server/SPECS/percona-qan-api2.spec | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build/packages/rpm/server/SPECS/percona-qan-api2.spec b/build/packages/rpm/server/SPECS/percona-qan-api2.spec
index 46995118b1..3f851cb36f 100644
--- a/build/packages/rpm/server/SPECS/percona-qan-api2.spec
+++ b/build/packages/rpm/server/SPECS/percona-qan-api2.spec
@@ -11,7 +11,7 @@
 # the line below is sed'ed by build/bin/build-server-rpm to set a correct version
 %define full_pmm_version 2.0.0
 
-Name:           pmm-qan
+Name:           percona-qan-api2
 Version:        %{version}
 Release:        %{rpm_release}
 Summary:        Query Analytics API for PMM

From f28cbccca37377a86d6abff3b6f08a9993955551 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 2 Apr 2024 11:03:47 +0000
Subject: [PATCH 15/35] PMM-12899 merge update.sh to build.sh

---
 build/local/build.sh              | 316 ++++++++++++++++++++++--------
 build/local/update.sh             | 101 ----------
 build/scripts/build-client-binary |   4 +
 build/scripts/build-client-docker |   6 +-
 build/scripts/build-client-rpm    |   6 +-
 build/scripts/build-client-source |   4 +
 build/scripts/build-client-srpm   |   4 +
 build/scripts/build-server-docker |   9 +-
 build/scripts/build-server-rpm    |   6 +-
 9 files changed, 269 insertions(+), 187 deletions(-)
 delete mode 100755 build/local/update.sh

diff --git a/build/local/build.sh b/build/local/build.sh
index bd174fb965..6e9985e706 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -2,12 +2,116 @@
 
 # Define global variables
 NO_UPDATE=0
+UPDATE_ONLY=0
 NO_CLIENT=0
 NO_CLIENT_DOCKER=0
 NO_SERVER_RPM=0
+OUTPUT_LOG_FILE="/tmp/build.log"
+
+needs-to-pull() {
+  local UPSTREAM=${1:-'@{u}'}
+  local LOCAL=$(git rev-parse @)
+  local BASE=$(git merge-base @ "$UPSTREAM")
+  local REMOTE=$(git rev-parse "$UPSTREAM")
+
+  if [ "$LOCAL" = "$REMOTE" ]; then
+    return 1 # false, we are up-to-date
+  fi
+
+  if [ "$LOCAL" = "$BASE" ]; then
+    return 0 # true, we are behind upstream
+  fi
+}
+
+rewind() {
+  local DIR="$1"
+  local BRANCH="$2"
+
+  cd "$DIR"
+  CURRENT=$(git branch --show-current)
+  git fetch
+
+  if [ "$CURRENT" != "$BRANCH" ]; then
+    echo "Currently on $CURRENT, checking out $BRANCH"
+    git checkout "$BRANCH"
+  fi
+
+  if needs-to-pull; then
+    git pull origin
+    echo "Submodule has pulled from upstream"
+    git logs -n 2
+    cd - >/dev/null
+    git add "$DIR"
+  else
+    cd - >/dev/null
+    echo "Submodule is up-to-date with upstream"
+  fi
+}
+
+check-files() {
+  local DIR="$1"
+
+  test -z "DIR" && exit 1
+
+  if [ -d "$DIR/sources" ] && [ -f "$DIR/ci-default.yml" ] && [ -f "$DIR/ci.yml" ]; then
+    return 0
+  fi
+
+  return 1
+}
+
+update() {
+  local DEPS=
+  local CURDIR="$PWD"
+  local DIR=pmm-submodules
+
+  # Thouroughly verify the presence of known files, otherwise bail out
+  if check-files "."; then # pwd is pmm-submodules
+    DIR="."
+  elif [ -d "$DIR" ]; then # pwd is outside pmm-submodules
+    if ! check-files "$DIR"; then
+      echo "FATAL: could not locate known files in ${PWD}/${DIR}"
+      exit 1
+    fi
+  else
+    echo "FATAL: could not locate known files in $PWD"
+    exit 1
+  fi
+
+  cd "$DIR"
+
+  # Join the dependencies from ci-default.yml and ci.yml
+  DEPS=$(yq -o=json eval-all '. as $item ireduce ({}; . *d $item )' ci-default.yml ci.yml | jq '.deps')
+
+  echo "This script rewinds submodule branches as per the joint config of 'ci-default.yml' and 'ci.yml'"
+
+  echo "$DEPS" | jq -c '.[]' | while read -r item; do
+    branch=$(echo "$item" | jq -r '.branch')
+    path=$(echo "$item" | jq -r '.path')
+    name=$(echo "$item" | jq -r '.name')
+    echo
+    echo "Rewinding submodule '$name' ..."
+    echo "path: ${path}, branch: ${branch}"
+
+    rewind "$path" "$branch"
+  done
+
+  echo
+  echo "Printing git status..."
+  git status --short
+  echo
+  echo "Printing git submodule status..."
+  git submodule status
+
+  cd "$CURDIR" > /dev/null
+}
+
 
 while test "$#" -gt 0; do
   case "$1" in
+    --update-only)
+      UPDATE_ONLY=1; NO_UPDATE=0
+      ;;
     --no-update)
       NO_UPDATE=1
       ;;
@@ -22,7 +126,7 @@ while test "$#" -gt 0; do
       ;;
     *)
       echo "Unknown argument: $1"
-      echo "Usage: $0 [--no-update] [--no-client] [--no-client-docker] [--no-server-rpm]"
+      echo "Usage: $0 [--no-update | --update-only] [--no-client] [--no-client-docker] [--no-server-rpm]"
       exit 1
       ;;
   esac
@@ -30,8 +134,18 @@ while test "$#" -gt 0; do
 done
 
 if [ "$NO_UPDATE" -eq 0 ]; then
+  MD5SUM=$(md5sum $(dirname $0)/build.sh)
+  
   # Update submodules and PR branches
-  /bin/bash $(dirname $0)/update.sh
+  update
+
+  test "$UPDATE_ONLY" -eq 1 && return
+
+  if [ "$MD5SUM" != "$(md5sum $(dirname $0)/build.sh)" ]; then
+    echo "The updated version of this script has been fetched from the repository, exiting..."
+    echo "Please run it again, i.e. '/bin/bash $(dirname $0)/build.sh --no-update'"
+    return
+  fi
 fi
 
 get_branch_name() {
@@ -46,97 +160,141 @@ get_branch_name() {
   echo $branch_name
 }
 
-# Define global variables
-pmm_commit=$(git submodule status | grep 'sources/pmm/src' | awk -F ' ' '{print $1}')
-echo $pmm_commit > apiCommitSha
-pmm_branch=$(get_branch_name pmm)
-echo $pmm_branch > apiBranch
-pmm_url=$(git config -f .gitmodules submodule.pmm.url)
-echo $pmm_url > apiURL
-pmm_qa_branch=$(get_branch_name pmm-qa)
-echo $pmm_qa_branch > pmmQABranch
-pmm_qa_commit=$(git submodule status | grep 'pmm-qa' | awk -F ' ' '{print $1}')
-echo $pmm_qa_commit > pmmQACommitSha
-pmm_ui_tests_branch=$(get_branch_name pmm-ui-tests)
-echo $pmm_ui_tests_branch > pmmUITestBranch
-pmm_ui_tests_commit=$(git submodule status | grep 'pmm-ui-tests' | awk -F ' ' '{print $1}')
-echo $pmm_ui_tests_commit > pmmUITestsCommitSha
-fb_commit_sha=$(git rev-parse HEAD)
-echo $fb_commit_sha > fbCommitSha
-
-# We use a special docker image to build various PMM artifacts - `perconalab/rpmbuild:3`.
-# Important: the docker container's user need to be able to write to these directories.
-# The docker container's user is `builder` with uid 1000 and gid 1000. You need to make sure
-# that the directories we create on the host are owned by a user with same uid and gid.
-
-# Create cache directories. 
-test -d "${root_dir}/go-path" || mkdir -p "go-path"
-test -d "${root_dir}/go-build" || mkdir -p "go-build"
-test -d "${root_dir}/yarn-cache" || mkdir -p "yarn-cache"
-
-PATH_TO_SCRIPTS="sources/pmm/src/github.com/percona/pmm/build/scripts"
-export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:3
+build_with_logs() {
+  local script="$1"
+  local start_time
+  local end_time
+
+  if [ ! -f "$script" ]; then
+    echo "Fatal: script $script does not exist"
+    exit 1
+  fi
+
+  start_time=$(date +%s)
+  if [ "$#" -gt 1 ]; then
+    shift
+    $script "$@" | tee -a $OUTPUT_LOG_FILE
+  else
+    $script | tee -a $OUTPUT_LOG_FILE
+  fi
+  end_time=$(date +%s)
+
+  echo "Execution time for $script: $((end_time - start_time)) seconds"
+}
+
+init() {
+  # Remove the temp directory
+  if [ -d tmp ]; then
+    echo "Removing tmp directory..."
+    rm -rf tmp
+  fi
+  if [ -f "$OUTPUT_LOG_FILE" ]; then
+    echo "Removing the log file..."
+    rm -f $OUTPUT_LOG_FILE
+  fi
+
+  # Define global variables
+  pmm_commit=$(git submodule status | grep 'sources/pmm/src' | awk -F ' ' '{print $1}')
+  echo $pmm_commit > apiCommitSha
+  pmm_branch=$(get_branch_name pmm)
+  echo $pmm_branch > apiBranch
+  pmm_url=$(git config -f .gitmodules submodule.pmm.url)
+  echo $pmm_url > apiURL
+  pmm_qa_branch=$(get_branch_name pmm-qa)
+  echo $pmm_qa_branch > pmmQABranch
+  pmm_qa_commit=$(git submodule status | grep 'pmm-qa' | awk -F ' ' '{print $1}')
+  echo $pmm_qa_commit > pmmQACommitSha
+  pmm_ui_tests_branch=$(get_branch_name pmm-ui-tests)
+  echo $pmm_ui_tests_branch > pmmUITestBranch
+  pmm_ui_tests_commit=$(git submodule status | grep 'pmm-ui-tests' | awk -F ' ' '{print $1}')
+  echo $pmm_ui_tests_commit > pmmUITestsCommitSha
+  fb_commit_sha=$(git rev-parse HEAD)
+  echo $fb_commit_sha > fbCommitSha
+
+  PATH_TO_SCRIPTS="sources/pmm/src/github.com/percona/pmm/build/scripts"
+  export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:3
+
+  # We use a special docker image to build various PMM artifacts - `perconalab/rpmbuild:3`.
+  # Important: the docker container's user needs to be able to write to these directories.
+  # The docker container's user is `builder` with uid 1000 and gid 1000. You need to make sure
+  # that the directories we create on the host are owned by a user with the same uid and gid.
+
+  # Create cache directories.
+  test -d "${root_dir}/go-path" || mkdir -p "go-path"
+  test -d "${root_dir}/go-build" || mkdir -p "go-build"
+  test -d "${root_dir}/yarn-cache" || mkdir -p "yarn-cache"
+  test -d "${root_dir}/yum-cache" || mkdir -p "yum-cache"
+
+}
+
+cleanup() {
+  # Clean up temporary files
+  rm -f apiBranch \
+    apiCommitSha \
+    apiURL \
+    fbCommitSha \
+    pmmQABranch \
+    pmmQACommitSha \
+    pmmUITestBranch \
+    pmmUITestsCommitSha
+}
 
 # Local reference test environment
 # CPU: 4 cores
 # RAM: 16GB
 # OS: Ubuntu 22.04.1 LTS
 
-if [ "$NO_CLIENT" -eq 0 ]; then
-  # Build client source: 4m39s from scratch, 0m27s using cache
-  "$PATH_TO_SCRIPTS/build-client-source"
+main() {
 
-  # Build client binary: ??? from scratch, 0m20s using cache
-  "$PATH_TO_SCRIPTS/build-client-binary"
+  if [ "$NO_CLIENT" -eq 0 ]; then
+    # Build client source: 4m39s from scratch, 0m27s using cache
+    build_with_logs "$PATH_TO_SCRIPTS/build-client-source"
 
-  # Building client source rpm takes 13s (caching does not apply)
-  "$PATH_TO_SCRIPTS/build-client-srpm"
+    # Build client binary: ??? from scratch, 0m20s using cache
+    build_with_logs "$PATH_TO_SCRIPTS/build-client-binary"
 
-  # Building client rpm takes 1m40s
-  "$PATH_TO_SCRIPTS/build-client-rpm"
-fi
+    # Building client source rpm takes 13s (caching does not apply)
+    build_with_logs "$PATH_TO_SCRIPTS/build-client-srpm"
 
-# Building client docker image takes 17s
-GIT_COMMIT=$(git rev-parse HEAD | head -c 8)
-export DOCKER_CLIENT_TAG=local/pmm-client:${GIT_COMMIT}
-if [ "$NO_CLIENT_DOCKER" -eq 0 ]; then
-  "$PATH_TO_SCRIPTS/build-client-docker"
-fi
+    # Building client rpm takes 1m40s
+    build_with_logs "$PATH_TO_SCRIPTS/build-client-rpm"
+  fi
 
-# Building PMM CLient locally (non-CI, i.e. non-Jenkins)
-# total time: 6m26s - build from scratch, no initial cache
-# total time: 2m49s - subsequent build, using cache from prior builds
+  # Building client docker image takes 17s
+  GIT_COMMIT=$(git rev-parse HEAD | head -c 8)
+  export DOCKER_CLIENT_TAG=local/pmm-client:${GIT_COMMIT}
+  if [ "$NO_CLIENT_DOCKER" -eq 0 ]; then
+    build_with_logs "$PATH_TO_SCRIPTS/build-client-docker"
+  fi
 
+  # Building PMM CLient locally (non-CI, i.e. non-Jenkins)
+  # total time: 6m26s - build from scratch, no initial cache
+  # total time: 2m49s - subsequent build, using cache from prior builds
 
-# Building PMM CLient in a CI environment, i.e. Jenkins running on AWS
-# total time: 8m45s - build from scratch, no initial cache
-# total time: ??? - subsequent build, using cache from prior builds
 
-export RPM_EPOCH=1
+  # Building PMM CLient in a CI environment, i.e. Jenkins running on AWS
+  # total time: 8m45s - build from scratch, no initial cache
+  # total time: ??? - subsequent build, using cache from prior builds
 
-if [ "$NO_SERVER_RPM" -eq 0 ]; then
-  "$PATH_TO_SCRIPTS/build-server-rpm" percona-dashboards grafana-dashboards
-  "$PATH_TO_SCRIPTS/build-server-rpm" pmm-managed pmm
-  "$PATH_TO_SCRIPTS/build-server-rpm" percona-qan-api2 pmm
-  "$PATH_TO_SCRIPTS/build-server-rpm" pmm-update pmm
-  "$PATH_TO_SCRIPTS/build-server-rpm" pmm-dump
-  "$PATH_TO_SCRIPTS/build-server-rpm" vmproxy pmm
+  export RPM_EPOCH=1
+  if [ "$NO_SERVER_RPM" -eq 0 ]; then
+    build_with_logs "$PATH_TO_SCRIPTS/build-server-rpm" percona-dashboards grafana-dashboards
+    build_with_logs "$PATH_TO_SCRIPTS/build-server-rpm" pmm-managed pmm
+    build_with_logs "$PATH_TO_SCRIPTS/build-server-rpm" percona-qan-api2 pmm
+    build_with_logs "$PATH_TO_SCRIPTS/build-server-rpm" pmm-update pmm
+    build_with_logs "$PATH_TO_SCRIPTS/build-server-rpm" pmm-dump
+    build_with_logs "$PATH_TO_SCRIPTS/build-server-rpm" vmproxy pmm
 
-  # 3rd-party
-  "$PATH_TO_SCRIPTS/build-server-rpm" victoriametrics
-  "$PATH_TO_SCRIPTS/build-server-rpm" grafana
-fi
+    # 3rd-party
+    build_with_logs "$PATH_TO_SCRIPTS/build-server-rpm" victoriametrics
+    build_with_logs "$PATH_TO_SCRIPTS/build-server-rpm" grafana
+  fi
+
+  export DOCKER_TAG=local/pmm-server:${GIT_COMMIT}
+  export DOCKERFILE=Dockerfile.el9
+  build_with_logs "${PATH_TO_SCRIPTS}/build-server-docker"
+}
 
-export DOCKER_TAG=local/pmm-server:${GIT_COMMIT}
-export DOCKERFILE=Dockerfile.el9
-${PATH_TO_SCRIPTS}/build-server-docker
-
-# Clean up temporary files
-rm -f apiBranch \
-  apiCommitSha \
-	apiURL \
-	fbCommitSha \
-	pmmQABranch \
-	pmmQACommitSha \
-	pmmUITestBranch \
-	pmmUITestsCommitSha
+init
+main
+cleanup
diff --git a/build/local/update.sh b/build/local/update.sh
deleted file mode 100755
index 4922dd9bda..0000000000
--- a/build/local/update.sh
+++ /dev/null
@@ -1,101 +0,0 @@
-#!/bin/bash -e
-
-needs-to-pull() {
-  local UPSTREAM=${1:-'@{u}'}
-  local LOCAL=$(git rev-parse @)
-  local BASE=$(git merge-base @ "$UPSTREAM")
-  local REMOTE=$(git rev-parse "$UPSTREAM")
-
-  if [ "$LOCAL" = "$REMOTE" ]; then
-    return 1 # false, we are up-to-date
-  fi
-
-  if [ "$LOCAL" = "$BASE" ]; then
-    return 0 # true, we are behind upstream
-  fi
-}
-
-rewind() {
-  local DIR="$1"
-  local BRANCH="$2"
-
-  cd "$DIR"
-  CURRENT=$(git branch --show-current)
-  git fetch
-
-  if [ "$CURRENT" != "$BRANCH" ]; then
-    echo "Currently on $CURRENT, checking out $BRANCH"
-    git checkout "$BRANCH"
-  fi
-
-  if needs-to-pull; then
-    git pull origin
-    echo "Submodule has pulled from upstream"
-    git logs -n 2
-    cd - >/dev/null
-    git add "$DIR"
-  else
-    cd - >/dev/null
-    echo "Submodule is up-to-date with upstream"
-  fi
-}
-
-check-files() {
-  local DIR="$1"
-
-  test -z "DIR" && exit 1
-
-  if [ -d "$DIR/sources" ] && [ -f "$DIR/ci-default.yml" ] && [ -f "$DIR/ci.yml" ]; then
-    return 0
-  fi
-
-  return 1
-}
-
-main() {
-  local DEPS=
-  local CURDIR="$PWD"
-  local DIR=pmm-submodules
-
-  # Thouroughly verify the presence of known files, otherwise bail out
-  if check-files "."; then # pwd is pmm-submodules
-    DIR="."
-  elif [ -d "$DIR" ]; then # pwd is outside pmm-submodules
-    if ! check-files "$DIR"; then
-      echo "FATAL: could not locate known files in ${PWD}/${DIR}"
-      exit 1
-    fi
-  else
-    echo "FATAL: could not locate known files in $PWD"
-    exit 1
-  fi
-
-  cd "$DIR"
-
-  # Join the dependencies from ci-default.yml and ci.yml
-  DEPS=$(yq -o=json eval-all '. as $item ireduce ({}; . *d $item )' ci-default.yml ci.yml | jq '.deps')
-
-  echo "This script rewinds submodule branches as per the joint config of 'ci-default.yml' and 'ci.yml'"
-
-  echo "$DEPS" | jq -c '.[]' | while read -r item; do
-    branch=$(echo "$item" | jq -r '.branch')
-    path=$(echo "$item" | jq -r '.path')
-    name=$(echo "$item" | jq -r '.name')
-    echo
-    echo "Rewinding submodule '$name' ..."
-    echo "path: ${path}, branch: ${branch}"
-
-    rewind "$path" "$branch"
-  done
-
-  echo
-  echo "Printing git status..."
-  git status --short
-  echo
-  echo "Printing git submodule status..."
-  git submodule status
-
-  cd "$CURDIR" > /dev/null
-}
-
-main
diff --git a/build/scripts/build-client-binary b/build/scripts/build-client-binary
index f580e3f5d8..6b9460baf4 100755
--- a/build/scripts/build-client-binary
+++ b/build/scripts/build-client-binary
@@ -108,6 +108,10 @@ copy_component() {
 }
 
 main() {
+    echo -----------------------------------------
+    echo "Building PMM Client binary files..."
+    echo -----------------------------------------
+
     extract_source_tarball
 
     gobuild_component "vmagent" "" "github.com/VictoriaMetrics/VictoriaMetrics" "app/vmagent"
diff --git a/build/scripts/build-client-docker b/build/scripts/build-client-docker
index 119a47f7a3..af6cbae9b7 100755
--- a/build/scripts/build-client-docker
+++ b/build/scripts/build-client-docker
@@ -6,10 +6,14 @@ set -o xtrace
 . $(dirname $0)/vars
 
 if [ -f "${docker_client_tarball}" ]; then
-    echo skip docker build
+    echo Docker client tarball found, skipping docker build...
     exit 0
 fi
 
+echo -----------------------------------------
+echo "Building docker image for PMM Client..."
+echo -----------------------------------------
+
 DOCKER_FILE_LOCATION=tmp/source/pmm/build/docker/client # relative to `root_dir`
 cp ${root_dir}/results/tarball/pmm-client-*.tar.gz ${root_dir}/${DOCKER_FILE_LOCATION}/pmm-client.tar.gz
 
diff --git a/build/scripts/build-client-rpm b/build/scripts/build-client-rpm
index ae5780ae24..e11e691ae9 100755
--- a/build/scripts/build-client-rpm
+++ b/build/scripts/build-client-rpm
@@ -6,6 +6,10 @@ set -o errexit
 set -o xtrace
 
 main() {
+    echo -----------------------------------------
+    echo "Building PMM Client RPM files..."
+    echo -----------------------------------------
+
     local IMAGE=${1:-${rpmbuild_docker_image}}
     docker run --rm -v ${bin_dir}:/home/builder/bin -v ${root_dir}/results:/home/builder/results ${IMAGE} sh -c "
         set -o errexit
@@ -13,7 +17,7 @@ main() {
 
         export pmm_version=$pmm_version
         export pmm_release=$pmm_release
-        export sudo_path=\$(ls /usr/bin/sudo)
+        export sudo_path=\$$(command -v sudo)
         mkdir -p /tmp/pmm
         cd /home/builder/results
         /home/builder/bin/build-client-packages \
diff --git a/build/scripts/build-client-source b/build/scripts/build-client-source
index 978f09a1b0..a1112413ad 100755
--- a/build/scripts/build-client-source
+++ b/build/scripts/build-client-source
@@ -59,6 +59,10 @@ prepare_vmagent_tarball() {
 }
 
 main() {
+    echo -----------------------------------------
+    echo "Building PMM Client source files..."
+    echo -----------------------------------------
+
     rm -rf ${source_tarball}
     mkdir -p ${source_dir}/ $(dirname ${source_tarball}) || :
 
diff --git a/build/scripts/build-client-srpm b/build/scripts/build-client-srpm
index 20747d0779..8e4fe8e315 100755
--- a/build/scripts/build-client-srpm
+++ b/build/scripts/build-client-srpm
@@ -6,6 +6,10 @@ set -o errexit
 set -o xtrace
 
 main() {
+    echo -----------------------------------------
+    echo "Building PMM Client source RPM files..."
+    echo -----------------------------------------
+
     local IMAGE=${1:-${rpmbuild_docker_image}}
     docker run --rm -v ${bin_dir}:/home/builder/bin -v ${root_dir}/results:/home/builder/results ${IMAGE} sh -c "
         set -o errexit
diff --git a/build/scripts/build-server-docker b/build/scripts/build-server-docker
index 16027c3beb..e6bffbaa20 100755
--- a/build/scripts/build-server-docker
+++ b/build/scripts/build-server-docker
@@ -6,15 +6,16 @@ set -o xtrace
 . $(dirname $0)/vars
 
 if [ -f "${docker_tarball}" ]; then
-    echo skip docker build
+    echo Docker tarball found, skipping docker build...
     exit 0
 fi
 
+echo -----------------------------------------
+echo "Building docker image for PMM Server..."
+echo -----------------------------------------
+
 docker_root=$(realpath ${rpms_dir}/..)
 cp -r ${root_dir}/tmp/source/pmm/build/ansible ${docker_root}/ansible
-ls ${docker_root}/ansible
-ls ${docker_root}/ansible/roles/pmm-images
-ls ${docker_root}/ansible/roles/pmm-images/tasks
 cp ${root_dir}/tmp/source/pmm/build/docker/server/* ${docker_root}/
 cp ${root_dir}/results/tarball/pmm-client-*.tar.gz ${docker_root}/pmm-client.tar.gz
 
diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index ff5ab06553..4ada6b52d1 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -43,7 +43,7 @@ is_build_needed() {
             --region us-east-2 \
             --no-sign-request \
             s3://pmm-build-cache/${s3_cache_dir}/${rpmbuild_dist}/${spec_name}-${rpm_version} \
-            ${rpms_dir}/${spec_name}-${rpm_version}
+            ${rpms_dir}/${spec_name}-${rpm_version} || :
     fi
 
     packages=$(find ${rpms_dir}/${spec_name}-${rpm_version} -name "*.${rpmbuild_dist}.*.rpm" | wc -l)
@@ -109,6 +109,10 @@ build() {
         return
     fi
 
+    echo -------------------------------------------------------
+    echo "Building PMM Server RPM for ${spec_name} component..."
+    echo -------------------------------------------------------
+
     if [ -d "${root_dir}/go-build" ]; then
         volume_mounts+=" -v ${root_dir}/go-build:/home/builder/.cache/go-build"
     fi

From 093179afc763d58df0a9c7445e6fab5ebbbbd6ec Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 2 Apr 2024 11:37:36 +0000
Subject: [PATCH 16/35] PMM-12899 reorganize the script structure

---
 build/local/build.sh | 93 +++++++++++++++++++++-----------------------
 1 file changed, 45 insertions(+), 48 deletions(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index 6e9985e706..e59c9d8174 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -8,6 +8,32 @@ NO_CLIENT_DOCKER=0
 NO_SERVER_RPM=0
 OUTPUT_LOG_FILE="/tmp/build.log"
 
+while test "$#" -gt 0; do
+  case "$1" in
+    --update-only)
+      UPDATE_ONLY=1; NO_UPDATE=0
+      ;;
+    --no-update)
+      NO_UPDATE=1
+      ;;
+    --no-client)
+      NO_CLIENT=1; NO_CLIENT_DOCKER=1
+      ;;
+    --no-client-docker)
+      NO_CLIENT_DOCKER=1
+      ;;
+    --no-server-rpm)
+      NO_SERVER_RPM=1
+      ;;
+    *)
+      echo "Unknown argument: $1"
+      echo "Usage: $0 [--no-update | --update-only] [--no-client] [--no-client-docker] [--no-server-rpm]"
+      exit 1
+      ;;
+  esac
+  shift
+done
+
 needs-to-pull() {
   local UPSTREAM=${1:-'@{u}'}
   local LOCAL=$(git rev-parse @)
@@ -106,48 +132,6 @@ update() {
   cd "$CURDIR" > /dev/null
 }
 
-
-while test "$#" -gt 0; do
-  case "$1" in
-    --update-only)
-      UPDATE_ONLY=1; NO_UPDATE=0
-      ;;
-    --no-update)
-      NO_UPDATE=1
-      ;;
-    --no-client)
-      NO_CLIENT=1; NO_CLIENT_DOCKER=1
-      ;;
-    --no-client-docker)
-      NO_CLIENT_DOCKER=1
-      ;;
-    --no-server-rpm)
-      NO_SERVER_RPM=1
-      ;;
-    *)
-      echo "Unknown argument: $1"
-      echo "Usage: $0 [--no-update | --update-only] [--no-client] [--no-client-docker] [--no-server-rpm]"
-      exit 1
-      ;;
-  esac
-  shift
-done
-
-if [ "$NO_UPDATE" -eq 0 ]; then
-  MD5SUM=$(md5sum $(dirname $0)/build.sh)
-  
-  # Update submodules and PR branches
-  update
-
-  test "$UPDATE_ONLY" -eq 1 && return
-
-  if [ "$MD5SUM" != "$(md5sum $(dirname $0)/build.sh)" ]; then
-    echo "The updated version of this script has been fetched from the repository, exiting..."
-    echo "Please run it again, i.e. '/bin/bash $(dirname $0)/build.sh --no-update'"
-    return
-  fi
-fi
-
 get_branch_name() {
   local module="${1:-}"
   local branch_name
@@ -224,7 +208,6 @@ init() {
   test -d "${root_dir}/go-build" || mkdir -p "go-build"
   test -d "${root_dir}/yarn-cache" || mkdir -p "yarn-cache"
   test -d "${root_dir}/yum-cache" || mkdir -p "yum-cache"
-
 }
 
 cleanup() {
@@ -239,12 +222,21 @@ cleanup() {
     pmmUITestsCommitSha
 }
 
-# Local reference test environment
-# CPU: 4 cores
-# RAM: 16GB
-# OS: Ubuntu 22.04.1 LTS
-
 main() {
+  if [ "$NO_UPDATE" -eq 0 ]; then
+    MD5SUM=$(md5sum $(dirname $0)/build.sh)
+    
+    # Update submodules and PR branches
+    update
+
+    test "$UPDATE_ONLY" -eq 1 && return
+
+    if [ "$MD5SUM" != "$(md5sum $(dirname $0)/build.sh)" ]; then
+      echo "The updated version of this script has been fetched from the repository, exiting..."
+      echo "Please run it again, i.e. '/bin/bash $(dirname $0)/build.sh --no-update'"
+      return
+    fi
+  fi
 
   if [ "$NO_CLIENT" -eq 0 ]; then
     # Build client source: 4m39s from scratch, 0m27s using cache
@@ -295,6 +287,11 @@ main() {
   build_with_logs "${PATH_TO_SCRIPTS}/build-server-docker"
 }
 
+# Local reference test environment
+# CPU: 4 cores
+# RAM: 16GB
+# OS: Ubuntu 22.04.1 LTS
+
 init
 main
 cleanup

From 12fd453a88a2898ea97728209be8e7f06b11409c Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 2 Apr 2024 13:20:14 +0000
Subject: [PATCH 17/35] PMM-12899 get rid of `exit 0` in callee scripts

---
 build/local/build.sh                         | 23 +++--
 build/packages/rpm/server/SPECS/grafana.spec |  2 +-
 build/scripts/build-client-binary            |  1 -
 build/scripts/build-client-deb               |  1 -
 build/scripts/build-client-docker            | 76 ++++++++-------
 build/scripts/build-client-rpm               |  1 -
 build/scripts/build-client-sdeb              |  1 -
 build/scripts/build-client-source            |  1 -
 build/scripts/build-client-srpm              |  1 -
 build/scripts/build-server-docker            | 98 ++++++++++----------
 build/scripts/vars                           |  4 +-
 11 files changed, 110 insertions(+), 99 deletions(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index e59c9d8174..32c8caa52f 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -6,7 +6,7 @@ UPDATE_ONLY=0
 NO_CLIENT=0
 NO_CLIENT_DOCKER=0
 NO_SERVER_RPM=0
-OUTPUT_LOG_FILE="/tmp/build.log"
+LOG_FILE="/tmp/build.log"
 
 while test "$#" -gt 0; do
   case "$1" in
@@ -25,9 +25,18 @@ while test "$#" -gt 0; do
     --no-server-rpm)
       NO_SERVER_RPM=1
       ;;
+    --log-file)
+      shift
+      if [ -z "$1" ]; then
+        echo "Missing argument for --log-file"
+        exit 1
+      fi
+      LOG_FILE="$1"
+      ;;
     *)
       echo "Unknown argument: $1"
-      echo "Usage: $0 [--no-update | --update-only] [--no-client] [--no-client-docker] [--no-server-rpm]"
+      echo "Usage: $0 [--no-update | --update-only] [--no-client] [--no-client-docker] [--no-server-rpm] [--log-file <path>]"
+      echo
       exit 1
       ;;
   esac
@@ -150,16 +159,16 @@ build_with_logs() {
   local end_time
 
   if [ ! -f "$script" ]; then
-    echo "Fatal: script $script does not exist"
+    echo "FATAL: script $script does not exist"
     exit 1
   fi
 
   start_time=$(date +%s)
   if [ "$#" -gt 1 ]; then
     shift
-    $script "$@" | tee -a $OUTPUT_LOG_FILE
+    $script "$@" | tee -a $LOG_FILE
   else
-    $script | tee -a $OUTPUT_LOG_FILE
+    $script | tee -a $LOG_FILE
   fi
   end_time=$(date +%s)
 
@@ -172,9 +181,9 @@ init() {
     echo "Removing tmp directory..."
     rm -rf tmp
   fi
-  if [ -f "$OUTPUT_LOG_FILE" ]; then
+  if [ -f "$LOG_FILE" ]; then
     echo "Removing the log file..."
-    rm -f $OUTPUT_LOG_FILE
+    rm -f $LOG_FILE
   fi
 
   # Define global variables
diff --git a/build/packages/rpm/server/SPECS/grafana.spec b/build/packages/rpm/server/SPECS/grafana.spec
index e4c3e1558d..94e993bd3c 100644
--- a/build/packages/rpm/server/SPECS/grafana.spec
+++ b/build/packages/rpm/server/SPECS/grafana.spec
@@ -75,7 +75,7 @@ install -d -p %{buildroot}%{_sharedstatedir}/grafana
 %pre
 getent group pmm >/dev/null || echo "Group pmm does not exist. Please create it manually."
 getent passwd pmm >/dev/null || echo "User pmm does not exist. Please create it manually."
-exit 0
+
 
 %changelog
 * Wed Mar 20 2024 Alex Demidoff <alexander.demidoff@percona.com> - 10.4.0-105
diff --git a/build/scripts/build-client-binary b/build/scripts/build-client-binary
index 6b9460baf4..c26344bb6c 100755
--- a/build/scripts/build-client-binary
+++ b/build/scripts/build-client-binary
@@ -157,6 +157,5 @@ main() {
 }
 
 main
-exit 0
 
 # vim: expandtab shiftwidth=4 tabstop=4
diff --git a/build/scripts/build-client-deb b/build/scripts/build-client-deb
index 2a94002557..ccedf8e497 100755
--- a/build/scripts/build-client-deb
+++ b/build/scripts/build-client-deb
@@ -35,6 +35,5 @@ main() {
 }
 
 main $*
-exit 0
 
 # vim: expandtab shiftwidth=4 tabstop=4
diff --git a/build/scripts/build-client-docker b/build/scripts/build-client-docker
index af6cbae9b7..9494d41f5d 100755
--- a/build/scripts/build-client-docker
+++ b/build/scripts/build-client-docker
@@ -5,41 +5,45 @@ set -o xtrace
 
 . $(dirname $0)/vars
 
-if [ -f "${docker_client_tarball}" ]; then
-    echo Docker client tarball found, skipping docker build...
-    exit 0
-fi
-
-echo -----------------------------------------
-echo "Building docker image for PMM Client..."
-echo -----------------------------------------
-
-DOCKER_FILE_LOCATION=tmp/source/pmm/build/docker/client # relative to `root_dir`
-cp ${root_dir}/results/tarball/pmm-client-*.tar.gz ${root_dir}/${DOCKER_FILE_LOCATION}/pmm-client.tar.gz
-
-if [ -z "${DOCKER_CLIENT_TAG}" ]; then
-    DOCKER_CLIENT_TAG=perconalab/pmm-client-fb:${full_pmm_version}
-fi
-
-
-CLIENT_IMAGE_VERSION=`echo $DOCKER_CLIENT_TAG | cut -d ':' -f2`
-
-docker buildx build \
-    --build-arg BUILD_DATE="`date --rfc-3339=seconds`" \
-    --build-arg VERSION="$CLIENT_IMAGE_VERSION" \
-    --progress plain \
-    -f ${DOCKER_FILE_LOCATION}/${docker_file} \
-    -t ${DOCKER_CLIENT_TAG} \
-    ${DOCKER_FILE_LOCATION}
-
-if [ -n "${PUSH_DOCKER}" ]; then
-    mkdir -p $(dirname ${docker_client_tag_file})
-    echo ${DOCKER_CLIENT_TAG} > ${docker_client_tag_file}
-    docker push ${DOCKER_CLIENT_TAG}
-fi
-if [ -n "${SAVE_DOCKER}" ]; then
-    mkdir -p $(dirname ${docker_client_tarball})
-    docker save ${DOCKER_CLIENT_TAG} | xz > ${docker_client_tarball}
-fi
+main() {
+    echo -----------------------------------------
+    echo "Building docker image for PMM Client..."
+    echo -----------------------------------------
+
+    if [ -f "${docker_client_tarball}" ]; then
+        echo Docker client tarball found, skipping docker build...
+        return
+    fi
+
+    DOCKER_FILE_LOCATION=tmp/source/pmm/build/docker/client # relative to `root_dir`
+    cp ${root_dir}/results/tarball/pmm-client-*.tar.gz ${root_dir}/${DOCKER_FILE_LOCATION}/pmm-client.tar.gz
+
+    if [ -z "${DOCKER_CLIENT_TAG}" ]; then
+        DOCKER_CLIENT_TAG=perconalab/pmm-client-fb:${full_pmm_version}
+    fi
+
+
+    CLIENT_IMAGE_VERSION=`echo $DOCKER_CLIENT_TAG | cut -d ':' -f2`
+
+    docker buildx build \
+        --build-arg BUILD_DATE="`date --rfc-3339=seconds`" \
+        --build-arg VERSION="$CLIENT_IMAGE_VERSION" \
+        --progress plain \
+        -f ${DOCKER_FILE_LOCATION}/${docker_file} \
+        -t ${DOCKER_CLIENT_TAG} \
+        ${DOCKER_FILE_LOCATION}
+
+    if [ -n "${PUSH_DOCKER}" ]; then
+        mkdir -p $(dirname ${docker_client_tag_file})
+        echo ${DOCKER_CLIENT_TAG} > ${docker_client_tag_file}
+        docker push ${DOCKER_CLIENT_TAG}
+    fi
+    if [ -n "${SAVE_DOCKER}" ]; then
+        mkdir -p $(dirname ${docker_client_tarball})
+        docker save ${DOCKER_CLIENT_TAG} | xz > ${docker_client_tarball}
+    fi
+}
+
+main
 
 # vim: expandtab shiftwidth=4 tabstop=4
diff --git a/build/scripts/build-client-rpm b/build/scripts/build-client-rpm
index e11e691ae9..6cf2291207 100755
--- a/build/scripts/build-client-rpm
+++ b/build/scripts/build-client-rpm
@@ -29,6 +29,5 @@ main() {
 }
 
 main $*
-exit 0
 
 # vim: expandtab shiftwidth=4 tabstop=4
diff --git a/build/scripts/build-client-sdeb b/build/scripts/build-client-sdeb
index f52cff6577..fb6bb50a43 100755
--- a/build/scripts/build-client-sdeb
+++ b/build/scripts/build-client-sdeb
@@ -32,6 +32,5 @@ main() {
 }
 
 main $*
-exit 0
 
 # vim: expandtab shiftwidth=4 tabstop=4
diff --git a/build/scripts/build-client-source b/build/scripts/build-client-source
index a1112413ad..7b0b62fb7a 100755
--- a/build/scripts/build-client-source
+++ b/build/scripts/build-client-source
@@ -84,6 +84,5 @@ main() {
 }
 
 main
-exit 0
 
 # vim: expandtab shiftwidth=4 tabstop=4
diff --git a/build/scripts/build-client-srpm b/build/scripts/build-client-srpm
index 8e4fe8e315..ff4671659b 100755
--- a/build/scripts/build-client-srpm
+++ b/build/scripts/build-client-srpm
@@ -28,6 +28,5 @@ main() {
 }
 
 main $*
-exit 0
 
 # vim: expandtab shiftwidth=4 tabstop=4
diff --git a/build/scripts/build-server-docker b/build/scripts/build-server-docker
index e6bffbaa20..005fbee466 100755
--- a/build/scripts/build-server-docker
+++ b/build/scripts/build-server-docker
@@ -5,52 +5,56 @@ set -o xtrace
 
 . $(dirname $0)/vars
 
-if [ -f "${docker_tarball}" ]; then
-    echo Docker tarball found, skipping docker build...
-    exit 0
-fi
-
-echo -----------------------------------------
-echo "Building docker image for PMM Server..."
-echo -----------------------------------------
-
-docker_root=$(realpath ${rpms_dir}/..)
-cp -r ${root_dir}/tmp/source/pmm/build/ansible ${docker_root}/ansible
-cp ${root_dir}/tmp/source/pmm/build/docker/server/* ${docker_root}/
-cp ${root_dir}/results/tarball/pmm-client-*.tar.gz ${docker_root}/pmm-client.tar.gz
-
-git -C ${root_dir} rev-parse HEAD > ${docker_root}/gitCommit
-ls -la ${rpms_dir}
-docker run --rm -v ${rpms_dir}:/home/builder/rpm/RPMS ${rpmbuild_docker_image} sh -c "
-    sudo chown -R builder /home/builder/rpm/RPMS
-    until /usr/bin/createrepo_c --update /home/builder/rpm/RPMS; do
-        echo "waiting"
-        sleep 1
-    done
-"
-
-if [ -z "${DOCKER_TAG}" ]; then
-    DOCKER_TAG=perconalab/pmm-server-fb:${full_pmm_version}
-fi
-
-IMAGE_VERSION=$(echo "$DOCKER_TAG" | cut -d ':' -f2)
-
-docker buildx build \
-    --build-arg BUILD_DATE="$(date --rfc-3339=seconds)" \
-    --build-arg VERSION="$IMAGE_VERSION" \
-    --progress plain \
-    -f ${docker_root}/${docker_file} \
-    -t ${DOCKER_TAG} \
-    ${docker_root}
-
-if [ -n "${PUSH_DOCKER}" ]; then
-    mkdir -p $(dirname ${docker_tag_file})
-    echo ${DOCKER_TAG} > ${docker_tag_file}
-    docker push ${DOCKER_TAG}
-fi
-if [ -n "${SAVE_DOCKER}" ]; then
-    mkdir -p $(dirname ${docker_tarball})
-    docker save ${DOCKER_TAG} > ${docker_tarball}
-fi
+main() {
+  echo -----------------------------------------
+  echo "Building docker image for PMM Server..."
+  echo -----------------------------------------
+
+  if [ -f "${docker_tarball}" ]; then
+      echo Docker tarball found, skipping docker build...
+      return
+  fi
+
+  docker_root=$(realpath ${rpms_dir}/..)
+  cp -r ${root_dir}/tmp/source/pmm/build/ansible ${docker_root}/ansible
+  cp ${root_dir}/tmp/source/pmm/build/docker/server/* ${docker_root}/
+  cp ${root_dir}/results/tarball/pmm-client-*.tar.gz ${docker_root}/pmm-client.tar.gz
+
+  git -C ${root_dir} rev-parse HEAD > ${docker_root}/gitCommit
+  ls -la ${rpms_dir}
+  docker run --rm -v ${rpms_dir}:/home/builder/rpm/RPMS ${rpmbuild_docker_image} sh -c "
+      sudo chown -R builder /home/builder/rpm/RPMS
+      until /usr/bin/createrepo_c --update /home/builder/rpm/RPMS; do
+          echo "waiting"
+          sleep 1
+      done
+  "
+
+  if [ -z "${DOCKER_TAG}" ]; then
+      DOCKER_TAG=perconalab/pmm-server-fb:${full_pmm_version}
+  fi
+
+  IMAGE_VERSION=$(echo "$DOCKER_TAG" | cut -d ':' -f2)
+
+  docker buildx build \
+      --build-arg BUILD_DATE="$(date --rfc-3339=seconds)" \
+      --build-arg VERSION="$IMAGE_VERSION" \
+      --progress plain \
+      -f ${docker_root}/${docker_file} \
+      -t ${DOCKER_TAG} \
+      ${docker_root}
+
+  if [ -n "${PUSH_DOCKER}" ]; then
+      mkdir -p $(dirname ${docker_tag_file})
+      echo ${DOCKER_TAG} > ${docker_tag_file}
+      docker push ${DOCKER_TAG}
+  fi
+  if [ -n "${SAVE_DOCKER}" ]; then
+      mkdir -p $(dirname ${docker_tarball})
+      docker save ${DOCKER_TAG} > ${docker_tarball}
+  fi
+}
+
+main
 
 # vim: expandtab shiftwidth=4 tabstop=4
diff --git a/build/scripts/vars b/build/scripts/vars
index de57bfb8a4..338e6512e2 100644
--- a/build/scripts/vars
+++ b/build/scripts/vars
@@ -2,8 +2,8 @@ bin_dir=$(cd $(dirname $0); pwd -P)
 # TODO: refactor to pass ${WORKSPACE} as ROOT_DIR from either Jenkins or, if relevant, from GH actions
 # NOTE: in most cases, this evaluates to Jenkins ${WORKSPACE} directory when called from pipelines
 # Example: /home/ec2-user/workspace/ol9-build-server/, see how it's used in ol9-build-server.groovy
-root_dir_tmp=$(cd $(dirname $0)/../../../../../../../..; pwd -P)
-root_dir=${ROOT_DIR:-$root_dir_tmp}
+root_dir=$(cd $(dirname $0)/../../../../../../../..; pwd -P)
+root_dir=${ROOT_DIR:-$root_dir}
 tmp_dir=${root_dir}/tmp
 
 # In VERSION file we can have numeric value like '2.0.0' as well as

From 9587f21bcf70678c278c53f85c6b3b352ab11506 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 2 Apr 2024 13:39:18 +0000
Subject: [PATCH 18/35] PMM-12899 clean up pmm-update.spec

---
 build/packages/rpm/server/SPECS/pmm-update.spec | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/build/packages/rpm/server/SPECS/pmm-update.spec b/build/packages/rpm/server/SPECS/pmm-update.spec
index cb048070c3..5e93dea47a 100644
--- a/build/packages/rpm/server/SPECS/pmm-update.spec
+++ b/build/packages/rpm/server/SPECS/pmm-update.spec
@@ -42,9 +42,6 @@ make -C update release
 
 
 %install
-install -d %{buildroot}%{_datadir}/%{name}
-cp -pav ./update/ansible %{buildroot}%{_datadir}/%{name}
-
 install -d %{buildroot}%{_sbindir}
 install -p -m 0755 update/bin/pmm-update %{buildroot}%{_sbindir}/
 
@@ -53,7 +50,6 @@ install -p -m 0755 update/bin/pmm-update %{buildroot}%{_sbindir}/
 %license update/LICENSE
 %doc update/README.md
 %{_sbindir}/pmm-update
-%{_datadir}/%{name}
 
 
 %changelog

From 13b4f6517e670e24d1dd906922ec50f586808099 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 2 Apr 2024 14:23:50 +0000
Subject: [PATCH 19/35] PMM-12899 send exec time to the log file

---
 build/local/build.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index 32c8caa52f..7f46190b8e 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -172,7 +172,7 @@ build_with_logs() {
   fi
   end_time=$(date +%s)
 
-  echo "Execution time for $script: $((end_time - start_time)) seconds"
+  echo "Execution time for $script: $((end_time - start_time)) seconds" | tee -a $LOG_FILE
 }
 
 init() {

From 6f9eac8ea978c432ad15f16a7f9a0525d3f0373b Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 2 Apr 2024 17:07:58 +0000
Subject: [PATCH 20/35] PMM-12899 remove a redundant directory

---
 build/local/build.sh    | 4 +++-
 update/ansible/.gitkeep | 0
 2 files changed, 3 insertions(+), 1 deletion(-)
 delete mode 100644 update/ansible/.gitkeep

diff --git a/build/local/build.sh b/build/local/build.sh
index 7f46190b8e..9dcef31aec 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -157,6 +157,7 @@ build_with_logs() {
   local script="$1"
   local start_time
   local end_time
+  local script_name=$(basename $script)
 
   if [ ! -f "$script" ]; then
     echo "FATAL: script $script does not exist"
@@ -166,13 +167,14 @@ build_with_logs() {
   start_time=$(date +%s)
   if [ "$#" -gt 1 ]; then
     shift
+    script_name="${script_name}:($1)"
     $script "$@" | tee -a $LOG_FILE
   else
     $script | tee -a $LOG_FILE
   fi
   end_time=$(date +%s)
 
-  echo "Execution time for $script: $((end_time - start_time)) seconds" | tee -a $LOG_FILE
+  echo "Execution time (in sec) for $script_name: $((end_time - start_time))" | tee 
 }
 
 init() {
diff --git a/update/ansible/.gitkeep b/update/ansible/.gitkeep
deleted file mode 100644
index e69de29bb2..0000000000

From 3eb50f8b04721030c5be7778da3dd595cff496d4 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 2 Apr 2024 17:42:18 +0000
Subject: [PATCH 21/35] PMM-12899 add total exec time

---
 build/local/build.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/build/local/build.sh b/build/local/build.sh
index 9dcef31aec..3c156e9349 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -6,6 +6,7 @@ UPDATE_ONLY=0
 NO_CLIENT=0
 NO_CLIENT_DOCKER=0
 NO_SERVER_RPM=0
+START_TIME=$(date +%s)
 LOG_FILE="/tmp/build.log"
 
 while test "$#" -gt 0; do
@@ -296,6 +297,9 @@ main() {
   export DOCKER_TAG=local/pmm-server:${GIT_COMMIT}
   export DOCKERFILE=Dockerfile.el9
   build_with_logs "${PATH_TO_SCRIPTS}/build-server-docker"
+
+  echo "Done building PMM artifacts."
+  echo "Total execution time, sec: $(($(date +%s) - $START_TIME))" | tee -a $LOG_FILE
 }
 
 # Local reference test environment

From 36869b7a500ec7190e5af805468d57e50ec12f5f Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 2 Apr 2024 23:23:06 +0000
Subject: [PATCH 22/35] PMM-12899 reorg some variables

---
 build/local/build.sh                | 58 +++++++++++++++++------------
 build/scripts/build-client-packages | 24 ++++++------
 build/scripts/build-client-source   |  2 +-
 3 files changed, 48 insertions(+), 36 deletions(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index 3c156e9349..ca1ac4a9dd 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -106,11 +106,11 @@ update() {
     DIR="."
   elif [ -d "$DIR" ]; then # pwd is outside pmm-submodules
     if ! check-files "$DIR"; then
-      echo "FATAL: could not locate known files in ${PWD}/${DIR}"
+      echo "Fatal: could not locate known files in ${PWD}/${DIR}"
       exit 1
     fi
   else
-    echo "FATAL: could not locate known files in $PWD"
+    echo "Fatal: could not locate known files in $PWD"
     exit 1
   fi
 
@@ -155,13 +155,13 @@ get_branch_name() {
 }
 
 build_with_logs() {
-  local script="$1"
+  local script="$PATH_TO_SCRIPTS/$1"
   local start_time
   local end_time
-  local script_name=$(basename $script)
+  local script_name="$1"
 
   if [ ! -f "$script" ]; then
-    echo "FATAL: script $script does not exist"
+    echo "Fatal: script $script does not exist"
     exit 1
   fi
 
@@ -175,14 +175,23 @@ build_with_logs() {
   fi
   end_time=$(date +%s)
 
-  echo "Execution time (in sec) for $script_name: $((end_time - start_time))" | tee 
+  echo ---
+  echo "Execution time (in sec) for $script_name: $((end_time - start_time))" | tee -a $LOG_FILE
+  echo ---
 }
 
 init() {
-  # Remove the temp directory
+  local tmp_files
+  # Remove stale files and directories
   if [ -d tmp ]; then
-    echo "Removing tmp directory..."
-    rm -rf tmp
+    echo "Removing stale files and directories..."
+    if [ -d "tmp/pmm-server" ]; then
+      tmp_files=$(find tmp/pmm-server | grep -v "RPMS")
+      tmp_files=($tmp_files)
+      for f in "${tmp_files[@]}"; do
+        rm -rf "$f"
+      done
+    fi
   fi
   if [ -f "$LOG_FILE" ]; then
     echo "Removing the log file..."
@@ -252,23 +261,23 @@ main() {
 
   if [ "$NO_CLIENT" -eq 0 ]; then
     # Build client source: 4m39s from scratch, 0m27s using cache
-    build_with_logs "$PATH_TO_SCRIPTS/build-client-source"
+    build_with_logs build-client-source
 
     # Build client binary: ??? from scratch, 0m20s using cache
-    build_with_logs "$PATH_TO_SCRIPTS/build-client-binary"
+    build_with_logs build-client-binary
 
     # Building client source rpm takes 13s (caching does not apply)
-    build_with_logs "$PATH_TO_SCRIPTS/build-client-srpm"
+    build_with_logs build-client-srpm
 
     # Building client rpm takes 1m40s
-    build_with_logs "$PATH_TO_SCRIPTS/build-client-rpm"
+    build_with_logs build-client-rpm
   fi
 
   # Building client docker image takes 17s
   GIT_COMMIT=$(git rev-parse HEAD | head -c 8)
   export DOCKER_CLIENT_TAG=local/pmm-client:${GIT_COMMIT}
   if [ "$NO_CLIENT_DOCKER" -eq 0 ]; then
-    build_with_logs "$PATH_TO_SCRIPTS/build-client-docker"
+    build_with_logs build-client-docker
   fi
 
   # Building PMM CLient locally (non-CI, i.e. non-Jenkins)
@@ -282,24 +291,27 @@ main() {
 
   export RPM_EPOCH=1
   if [ "$NO_SERVER_RPM" -eq 0 ]; then
-    build_with_logs "$PATH_TO_SCRIPTS/build-server-rpm" percona-dashboards grafana-dashboards
-    build_with_logs "$PATH_TO_SCRIPTS/build-server-rpm" pmm-managed pmm
-    build_with_logs "$PATH_TO_SCRIPTS/build-server-rpm" percona-qan-api2 pmm
-    build_with_logs "$PATH_TO_SCRIPTS/build-server-rpm" pmm-update pmm
-    build_with_logs "$PATH_TO_SCRIPTS/build-server-rpm" pmm-dump
-    build_with_logs "$PATH_TO_SCRIPTS/build-server-rpm" vmproxy pmm
+    build_with_logs build-server-rpm percona-dashboards grafana-dashboards
+    build_with_logs build-server-rpm pmm-managed pmm
+    build_with_logs build-server-rpm percona-qan-api2 pmm
+    build_with_logs build-server-rpm pmm-update pmm
+    build_with_logs build-server-rpm pmm-dump
+    build_with_logs build-server-rpm vmproxy pmm
 
     # 3rd-party
-    build_with_logs "$PATH_TO_SCRIPTS/build-server-rpm" victoriametrics
-    build_with_logs "$PATH_TO_SCRIPTS/build-server-rpm" grafana
+    build_with_logs build-server-rpm victoriametrics
+    build_with_logs build-server-rpm grafana
   fi
 
   export DOCKER_TAG=local/pmm-server:${GIT_COMMIT}
   export DOCKERFILE=Dockerfile.el9
-  build_with_logs "${PATH_TO_SCRIPTS}/build-server-docker"
+  build_with_logs build-server-docker
 
+  echo
   echo "Done building PMM artifacts."
+  echo ---
   echo "Total execution time, sec: $(($(date +%s) - $START_TIME))" | tee -a $LOG_FILE
+  echo ---
 }
 
 # Local reference test environment
diff --git a/build/scripts/build-client-packages b/build/scripts/build-client-packages
index e7205ae7bb..6f80554bc0 100755
--- a/build/scripts/build-client-packages
+++ b/build/scripts/build-client-packages
@@ -174,26 +174,26 @@ get_system(){
 }
 
 install_deps() {
-    local RUN_CMD
+    local PACKAGES
     if [ $INSTALL = 0 ]; then
-        echo "Dependencies will not be installed"
+        echo "Dependencies will not be installed."
         return;
     fi
     if [ ! $( id -u ) -eq 0 ]; then
-        echo "It is not possible to instal dependencies. Please run as root"
+        echo "It is not possible to install dependencies. Please run as root."
         exit 1
     fi
     if [ "$OS" = "rpm" ]; then
         # yum -y install git curl wget rpmdevtools bison yum-utils rpm-build
-        command -v git || RUN_CMD+=" git"
-        command -v curl || RUN_CMD+=" curl"
-        command -v wget || RUN_CMD+=" wget"
-        command -v spectool || RUN_CMD+=" rpmdevtools"
-        command -v bison || RUN_CMD+=" bison"
-        command -v repoquery || RUN_CMD+=" yum-utils"
-        command -v rpmbuild || RUN_CMD+=" rpm-build"
-        if [ -n "$RUN_CMD" ]; then
-            yum -y install "$RUN_CMD"
+        command -v git || PACKAGES+=" git"
+        command -v curl || PACKAGES+=" curl"
+        command -v wget || PACKAGES+=" wget"
+        command -v spectool || PACKAGES+=" rpmdevtools"
+        command -v bison || PACKAGES+=" bison"
+        command -v repoquery || PACKAGES+=" yum-utils"
+        command -v rpmbuild || PACKAGES+=" rpm-build"
+        if [ -n "$PACKAGES" ]; then
+            yum -y install "$PACKAGES"
         fi
     else
         apt-get update
diff --git a/build/scripts/build-client-source b/build/scripts/build-client-source
index 7b0b62fb7a..0786be9cd1 100755
--- a/build/scripts/build-client-source
+++ b/build/scripts/build-client-source
@@ -54,7 +54,7 @@ prepare_vmagent_tarball() {
     if [ -f "${tarball}" ]; then
         echo ${tarball} already exists, skipping download
     else
-        curl -o ${tarball} -fSL https://github.com/VictoriaMetrics/VictoriaMetrics/archive/${commit_hash}.tar.gz
+        curl -o ${tarball} -fsSL https://github.com/VictoriaMetrics/VictoriaMetrics/archive/${commit_hash}.tar.gz
     fi
 }
 

From bf640caa27e5eb9fb0a0670b643704232d382511 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Wed, 3 Apr 2024 00:02:13 +0000
Subject: [PATCH 23/35] PMM-12899 fix the regexp

---
 build/local/build.sh | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index ca1ac4a9dd..999691aade 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -186,12 +186,15 @@ init() {
   if [ -d tmp ]; then
     echo "Removing stale files and directories..."
     if [ -d "tmp/pmm-server" ]; then
-      tmp_files=$(find tmp/pmm-server | grep -v "RPMS")
+      tmp_files=$(find tmp/pmm-server | grep -v "RPMS" | grep -Ev "^tmp/pmm-server$")
       tmp_files=($tmp_files)
       for f in "${tmp_files[@]}"; do
         rm -rf "$f"
       done
     fi
+    if [ -d "tmp/source/pmm" ]; then
+      rm -rf tmp/source/pmm
+    fi
   fi
   if [ -f "$LOG_FILE" ]; then
     echo "Removing the log file..."

From f28202aad1f4df24d3cff9ed50f6dfd65d6433d5 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Wed, 3 Apr 2024 07:50:16 +0000
Subject: [PATCH 24/35] PMM-12899 structure the code

---
 build/local/build.sh | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index 999691aade..0779cbf53c 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -180,7 +180,7 @@ build_with_logs() {
   echo ---
 }
 
-init() {
+purge_files() {
   local tmp_files
   # Remove stale files and directories
   if [ -d tmp ]; then
@@ -200,7 +200,9 @@ init() {
     echo "Removing the log file..."
     rm -f $LOG_FILE
   fi
+}
 
+init() {
   # Define global variables
   pmm_commit=$(git submodule status | grep 'sources/pmm/src' | awk -F ' ' '{print $1}')
   echo $pmm_commit > apiCommitSha
@@ -262,6 +264,9 @@ main() {
     fi
   fi
 
+  init
+  purge_files
+
   if [ "$NO_CLIENT" -eq 0 ]; then
     # Build client source: 4m39s from scratch, 0m27s using cache
     build_with_logs build-client-source
@@ -315,6 +320,8 @@ main() {
   echo ---
   echo "Total execution time, sec: $(($(date +%s) - $START_TIME))" | tee -a $LOG_FILE
   echo ---
+
+  cleanup
 }
 
 # Local reference test environment
@@ -322,6 +329,4 @@ main() {
 # RAM: 16GB
 # OS: Ubuntu 22.04.1 LTS
 
-init
 main
-cleanup

From a0f7b5011f3661b4597c960fee4ef57386940272 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Wed, 3 Apr 2024 08:36:11 +0000
Subject: [PATCH 25/35] PMM-12899 add a flag to skip building server docker

---
 build/local/build.sh | 20 +++++++++++++++++---
 1 file changed, 17 insertions(+), 3 deletions(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index 0779cbf53c..84c071d851 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -6,6 +6,7 @@ UPDATE_ONLY=0
 NO_CLIENT=0
 NO_CLIENT_DOCKER=0
 NO_SERVER_RPM=0
+NO_SERVER_DOCKER=0
 START_TIME=$(date +%s)
 LOG_FILE="/tmp/build.log"
 
@@ -24,8 +25,19 @@ while test "$#" -gt 0; do
       NO_CLIENT_DOCKER=1
       ;;
     --no-server-rpm)
+      if [ "$NO_SERVER_DOCKER" -eq 1 ]; then
+        echo "Cannot disable both server RPM and server Docker"
+        exit 1
+      fi
       NO_SERVER_RPM=1
       ;;
+    --no-server-docker)
+      if [ "$NO_SERVER_RPM" -eq 1 ]; then
+        echo "Cannot disable both server RPM and server Docker"
+        exit 1
+      fi
+      NO_SERVER_DOCKER=1
+      ;;
     --log-file)
       shift
       if [ -z "$1" ]; then
@@ -311,9 +323,11 @@ main() {
     build_with_logs build-server-rpm grafana
   fi
 
-  export DOCKER_TAG=local/pmm-server:${GIT_COMMIT}
-  export DOCKERFILE=Dockerfile.el9
-  build_with_logs build-server-docker
+  if [ "$NO_SERVER_DOCKER" -eq 0 ]; then
+    export DOCKER_TAG=local/pmm-server:${GIT_COMMIT}
+    export DOCKERFILE=Dockerfile.el9
+    build_with_logs build-server-docker
+  fi
 
   echo
   echo "Done building PMM artifacts."

From 988cfb0c32ef52158bfc05898362567171578df8 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Wed, 3 Apr 2024 09:16:40 +0000
Subject: [PATCH 26/35] PMM-12899 fix a failure of find when no files are found

---
 build/local/build.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index 84c071d851..81cd1cddba 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -198,7 +198,7 @@ purge_files() {
   if [ -d tmp ]; then
     echo "Removing stale files and directories..."
     if [ -d "tmp/pmm-server" ]; then
-      tmp_files=$(find tmp/pmm-server | grep -v "RPMS" | grep -Ev "^tmp/pmm-server$")
+      tmp_files=$(find tmp/pmm-server | grep -v "RPMS" | grep -Ev "^tmp/pmm-server$" || :)
       tmp_files=($tmp_files)
       for f in "${tmp_files[@]}"; do
         rm -rf "$f"

From 0582998db6ff3244ba69fd1335a9ab324b107e68 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Wed, 10 Apr 2024 16:05:51 +0300
Subject: [PATCH 27/35] PMM-12899 update the documentation

---
 build/local/README.md | 51 +++++++++++++++++++++++++++++++++++++++----
 build/local/build.sh  | 17 +++++++++------
 2 files changed, 57 insertions(+), 11 deletions(-)

diff --git a/build/local/README.md b/build/local/README.md
index 1517e9da3f..5f9051784c 100644
--- a/build/local/README.md
+++ b/build/local/README.md
@@ -12,6 +12,7 @@ The build process is mostly based on bash scripts, which control the build flow.
 
 The build process is designed to be run on a Linux host. We believe it can be run on other flavors of Linux, including MacOS, with little to no modification (TBC).
 
+
 ## Prerequisites
 
 Below is a list of prerequisites that are required to build PMM locally.
@@ -26,9 +27,51 @@ Below is a list of prerequisites that are required to build PMM locally.
 - yq: 4.42.0+ (tested on 4.42.1)
 - jq: 1.6+ (tested on 1.6)
 
-## Build Steps
+
+## How to build PMM
 
 1. Install the prerequisites
-2. Clone the PMM repository
-3. Change to the `build/local` directory
-4. Run the `build.sh` script
+2. Clone the PMM repository to the user's home directory, e.g.: `git clone https://github.com/percona/pmm /home/user/pmm`. 
+3. Change to the `build/local` directory in the cloned repo.
+4. Run `build.sh --help` to print the help message and check the usage.
+5. Run `build.sh` with parameters of your choice to build PMM v3.
+
+Usually, you will want to rebuild PMM whenever there are changes in at least one of its components. All components of PMM are gathered together in one repository - `github.com/percona-lab/pmm-submodules` (or `pmm-submodules`). Therefore, you can run `build.sh` as often as those changes need to be factored in to the next build.
+
+Once the build is finished, you can proceed with launching a new instance of PMM Server, or installing a freshly built PMM Client, and testing the changes.
+
+
+## The `rpmbuild` image and docker cache
+
+We use a special docker image to build various PMM artifacts - `perconalab/rpmbuild:3`. It comes provisioned with tools which are required to build PMM artifacts, for example RPM packages. As a build tool, it offers a number of benefits, two most obvious of which are:
+
+- it frees the user from installing dependencies on their host machine
+- it leverages a very powerful docker caching system, which results in reduced build times
+
+During the first run, `build.sh` will create a few directories on the host machine, which are necessary to make use of docker cache. Please be aware, that the docker container's user needs to be able to write to these directories. The docker container's user is `builder` with uid 1000 and gid 1000. You need to make sure that the directories we create on the host are owned by a user with the same uid and gid. If the build fails, this is the first thing to check.
+
+
+## Avoiding unnecessary builds
+
+Sometimes, the changes you make affect only PMM Client. Other times, they affect only PMM Server. Therefore, you may want to skip building parts of PMM. The `build.sh` script offers several parameters to help control what you want to build.
+
+* --no-update: run the build tasks without pulling the changes from `pmm-submodules` repository
+* --update-only: pull changes from the repo without building PMM
+* --no-client: do not build the client, use the cached PMM Client artifacts
+* --no-client-docker: skip building PMM Client docker container
+* --no-server-rpm: skip building PMM Server RPM artifacts
+* --log-file <path>: change the path of the build log file
+
+It's important to note, however, that once all changes are made and tested, you most probably want to re-build both PMM Client and Server to test them together.
+
+
+## Target environments
+
+Currently, local builds target the following environments:
+- PMM Client
+  - tarball - virtually any amd64 Linux environment
+  - RPM - RHEL9-compatible environments
+  - docker image - docker and Kubernetes environments (amd64)
+- PMM Server
+  - docker image - docker and Kubernetes environments (amd64)
+
diff --git a/build/local/build.sh b/build/local/build.sh
index 81cd1cddba..a821dad1e7 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -9,6 +9,8 @@ NO_SERVER_RPM=0
 NO_SERVER_DOCKER=0
 START_TIME=$(date +%s)
 LOG_FILE="/tmp/build.log"
+BASE_NAME=$(basename $0)
+USAGE="Usage: $BASE_NAME [--no-update | --update-only] [--no-client] [--no-client-docker] [--no-server-rpm] [--log-file <path>] [--help | -h]"
 
 while test "$#" -gt 0; do
   case "$1" in
@@ -46,9 +48,15 @@ while test "$#" -gt 0; do
       fi
       LOG_FILE="$1"
       ;;
+    --help | -h)
+      shift
+      echo "$USAGE"
+      echo
+      exit 0
+      ;;
     *)
       echo "Unknown argument: $1"
-      echo "Usage: $0 [--no-update | --update-only] [--no-client] [--no-client-docker] [--no-server-rpm] [--log-file <path>]"
+      echo "$USAGE"
       echo
       exit 1
       ;;
@@ -236,12 +244,7 @@ init() {
   PATH_TO_SCRIPTS="sources/pmm/src/github.com/percona/pmm/build/scripts"
   export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:3
 
-  # We use a special docker image to build various PMM artifacts - `perconalab/rpmbuild:3`.
-  # Important: the docker container's user needs to be able to write to these directories.
-  # The docker container's user is `builder` with uid 1000 and gid 1000. You need to make sure
-  # that the directories we create on the host are owned by a user with the same uid and gid.
-
-  # Create cache directories.
+  # Create cache directories. Read more in the section about `rpmbuild`.
   test -d "${root_dir}/go-path" || mkdir -p "go-path"
   test -d "${root_dir}/go-build" || mkdir -p "go-build"
   test -d "${root_dir}/yarn-cache" || mkdir -p "yarn-cache"

From ea42fe7904ab925058749c62d568e1830e854b98 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Sat, 12 Oct 2024 11:34:28 +0300
Subject: [PATCH 28/35] PMM-12899 update README.md

---
 build/local/README.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/build/local/README.md b/build/local/README.md
index 5f9051784c..b775030f66 100644
--- a/build/local/README.md
+++ b/build/local/README.md
@@ -6,11 +6,11 @@ This directory contains a set of scripts aimed at providing a simple way to buil
 
 Historically, PMM used to be built using Jenkins. This worked well for the team, but not for the community. The learning curve was, and still is, rather steep, and it is hard for folks, even internally, to contribute to.
 
-Therefore, we decided to make it possible to build PMM locally. This is a work in progress, but we are definitely committed to making it easier to build PMM locally.
+Therefore, we decided to make it possible to build PMM locally. This is a work in progress, but we are definitely committed to bring the developer experience to an acceptable level.
 
-The build process is mostly based on bash scripts, which control the build flow. This was an intentional decision early on to make the build process easy to understand and contribute to. Apart from bash and a few other well-known utilitites like `curl` or `make`, it also uses Docker for environment isolation and caching.
+The build process is mostly based on bash scripts, which control the build flow. This was an intentional decision early on, since every developer should have at least a basic command of bash. Apart from bash and a few other well-known utilitites like `curl` or `make`, it also uses Docker for environment isolation and caching.
 
-The build process is designed to be run on a Linux host. We believe it can be run on other flavors of Linux, including MacOS, with little to no modification (TBC).
+The build process is designed to run on a Linux host. We believe it can run on other flavors of Linux, including MacOS, with little to no modification (TBC).
 
 
 ## Prerequisites

From fc248de6e988937af6e7085fe7bfa7b5146e18ef Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Sat, 12 Oct 2024 14:20:03 +0300
Subject: [PATCH 29/35] PMM-12899 fix duplicate labels in Dockerfile

---
 build/docker/server/Dockerfile.el9 | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/build/docker/server/Dockerfile.el9 b/build/docker/server/Dockerfile.el9
index 656ab8615c..9131565bb4 100644
--- a/build/docker/server/Dockerfile.el9
+++ b/build/docker/server/Dockerfile.el9
@@ -33,12 +33,6 @@ RUN install -T -p -m 644 /opt/ansible/ansible.cfg /etc/ansible/ansible.cfg && \
     ansible-playbook -vvv -i 'localhost,' -c local /opt/ansible/pmm/post-build-actions.yml && \
     sed -i '/^assumeyes/d' /etc/dnf/dnf.conf
 
-LABEL org.opencontainers.image.created ${BUILD_DATE}
-LABEL org.opencontainers.image.licenses AGPL-3.0
-LABEL org.opencontainers.image.title Percona Monitoring and Management
-LABEL org.opencontainers.image.vendor Percona LLC
-LABEL org.opencontainers.image.version ${VERSION}
-
 LABEL org.opencontainers.image.created ${BUILD_DATE}
 LABEL org.opencontainers.image.licenses AGPL-3.0
 LABEL org.opencontainers.image.title Percona Monitoring and Management

From 4b469bda90ddc03f14b072020225b5b09157c94b Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 15 Oct 2024 00:45:07 +0300
Subject: [PATCH 30/35] PMM-12899 add a config file for all PMM components

---
 build/local/ci-default.yml | 89 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 89 insertions(+)
 create mode 100644 build/local/ci-default.yml

diff --git a/build/local/ci-default.yml b/build/local/ci-default.yml
new file mode 100644
index 0000000000..c344eeac13
--- /dev/null
+++ b/build/local/ci-default.yml
@@ -0,0 +1,89 @@
+deps:
+  # COMMON
+  - name: pmm
+    branch: v3
+    path: sources/pmm/src/github.com/percona/pmm
+    url: https://github.com/percona/pmm
+    component: common
+
+  # CLIENT
+  - name: node_exporter
+    branch: main
+    path: sources/node_exporter/src/github.com/prometheus/node_exporter
+    url: https://github.com/percona/node_exporter
+    component: client
+
+  - name: mysqld_exporter
+    branch: main
+    path: sources/mysqld_exporter/src/github.com/percona/mysqld_exporter
+    url: https://github.com/percona/mysqld_exporter
+    component: client
+
+  - name: mongodb_exporter
+    branch: main
+    path: sources/mongodb_exporter/src/github.com/percona/mongodb_exporter
+    url: https://github.com/percona/mongodb_exporter
+    component: client
+
+  - name: postgres_exporter
+    branch: main
+    path: sources/postgres_exporter/src/github.com/percona/postgres_exporter
+    url: https://github.com/percona/postgres_exporter
+    component: client
+
+  - name: proxysql_exporter
+    branch: main
+    path: sources/proxysql_exporter/src/github.com/percona/proxysql_exporter
+    url: https://github.com/percona/proxysql_exporter
+    component: client
+
+  - name: rds_exporter
+    branch: main
+    path: sources/rds_exporter/src/github.com/percona/rds_exporter
+    url: https://github.com/percona/rds_exporter
+    component: client
+
+  - name: azure_metrics_exporter
+    branch: main
+    path: sources/azure_metrics_exporter/src/github.com/percona/azure_metrics_exporter
+    url: https://github.com/percona/azure_metrics_exporter
+    component: client
+
+  - name: percona-toolkit
+    branch: release-v3.5.2
+    path: sources/percona-toolkit/src/github.com/percona/percona-toolkit
+    url: https://github.com/percona/percona-toolkit
+    component: client
+
+  # SERVER
+
+  - name: grafana-dashboards
+    branch: v3
+    path: sources/grafana-dashboards
+    url: https://github.com/percona/grafana-dashboards
+    component: server
+
+  - name: grafana
+    branch: v3
+    path: sources/grafana/src/github.com/grafana/grafana
+    url: https://github.com/percona/grafana
+    component: server
+
+  - name: pmm-dump
+    branch: main
+    path: sources/pmm-dump
+    url: https://github.com/percona/pmm-dump
+    component: server
+
+  # QA
+  - name: pmm-qa
+    branch: v3
+    path: sources/pmm-qa/src/github.com/percona/pmm-qa
+    url: https://github.com/percona/pmm-qa
+    component: qa
+
+  - name: pmm-ui-tests
+    branch: v3
+    path: sources/pmm-ui-tests/src/github.com/percona/pmm-ui-tests
+    url: https://github.com/percona/pmm-ui-tests
+    component: qa
\ No newline at end of file

From f35a85f16afa21f964837605f5194f0094f0ed99 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Sat, 19 Oct 2024 18:52:16 +0300
Subject: [PATCH 31/35] PMM-12899 simplify init, add usage

---
 build/local/build.sh | 50 +++++++++++++++++++-------------------------
 1 file changed, 22 insertions(+), 28 deletions(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index a821dad1e7..76b0327185 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -50,20 +50,31 @@ while test "$#" -gt 0; do
       ;;
     --help | -h)
       shift
-      echo "$USAGE"
-      echo
+      usage
       exit 0
       ;;
     *)
       echo "Unknown argument: $1"
-      echo "$USAGE"
-      echo
+      usage
       exit 1
       ;;
   esac
   shift
 done
 
+usage() {
+  cat <<-EOF
+Usage: $BASE_NAME [--no-update | --update-only] [--no-client] [--no-client-docker] [--no-server-rpm] [--log-file <path>] [--help | -h]
+--no-update              Do not fetch the latest changes from the repo
+--update-only            Only fetch the latest changes from the repo
+--no-client              Do not (re)build PMM client
+--no-client-docker       Do not (re)build PMM Client docker image
+--no-server-rpm          Do not (re)build Server RPM packages
+--log-file <path>        Save build logs to a file located at <path>
+--help | -h              Display help
+EOF
+}
+
 needs-to-pull() {
   local UPSTREAM=${1:-'@{u}'}
   local LOCAL=$(git rev-parse @)
@@ -83,7 +94,7 @@ rewind() {
   local DIR="$1"
   local BRANCH="$2"
 
-  cd "$DIR"
+  cd "$DIR" > /dev/null
   CURRENT=$(git branch --show-current)
   git fetch
 
@@ -96,10 +107,10 @@ rewind() {
     git pull origin
     echo "Submodule has pulled from upstream"
     git logs -n 2
-    cd - >/dev/null
+    cd - > /dev/null
     git add "$DIR"
   else
-    cd - >/dev/null
+    cd - > /dev/null
     echo "Submodule is up-to-date with upstream"
   fi
 }
@@ -223,25 +234,8 @@ purge_files() {
 }
 
 init() {
-  # Define global variables
-  pmm_commit=$(git submodule status | grep 'sources/pmm/src' | awk -F ' ' '{print $1}')
-  echo $pmm_commit > apiCommitSha
-  pmm_branch=$(get_branch_name pmm)
-  echo $pmm_branch > apiBranch
-  pmm_url=$(git config -f .gitmodules submodule.pmm.url)
-  echo $pmm_url > apiURL
-  pmm_qa_branch=$(get_branch_name pmm-qa)
-  echo $pmm_qa_branch > pmmQABranch
-  pmm_qa_commit=$(git submodule status | grep 'pmm-qa' | awk -F ' ' '{print $1}')
-  echo $pmm_qa_commit > pmmQACommitSha
-  pmm_ui_tests_branch=$(get_branch_name pmm-ui-tests)
-  echo $pmm_ui_tests_branch > pmmUITestBranch
-  pmm_ui_tests_commit=$(git submodule status | grep 'pmm-ui-tests' | awk -F ' ' '{print $1}')
-  echo $pmm_ui_tests_commit > pmmUITestsCommitSha
-  fb_commit_sha=$(git rev-parse HEAD)
-  echo $fb_commit_sha > fbCommitSha
-
   PATH_TO_SCRIPTS="sources/pmm/src/github.com/percona/pmm/build/scripts"
+  "${PATH_TO_SCRIPTS}/build-submodules"
   export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:3
 
   # Create cache directories. Read more in the section about `rpmbuild`.
@@ -290,10 +284,10 @@ main() {
     build_with_logs build-client-binary
 
     # Building client source rpm takes 13s (caching does not apply)
-    build_with_logs build-client-srpm
+    # build_with_logs build-client-srpm
 
     # Building client rpm takes 1m40s
-    build_with_logs build-client-rpm
+    # build_with_logs build-client-rpm
   fi
 
   # Building client docker image takes 17s
@@ -327,7 +321,7 @@ main() {
   fi
 
   if [ "$NO_SERVER_DOCKER" -eq 0 ]; then
-    export DOCKER_TAG=local/pmm-server:${GIT_COMMIT}
+    export DOCKER_TAG=percona/pmm-server:${GIT_COMMIT}
     export DOCKERFILE=Dockerfile.el9
     build_with_logs build-server-docker
   fi

From 1aad3f74dc3617a2ebecb8734627e2e6d5283f62 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Sun, 20 Oct 2024 01:38:07 +0300
Subject: [PATCH 32/35] PMM-12899 fix architecture for the client

---
 build/scripts/build-client-binary | 4 ++++
 build/scripts/build-client-rpm    | 3 +--
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/build/scripts/build-client-binary b/build/scripts/build-client-binary
index 53cb44a8b0..8fdacdd3d8 100755
--- a/build/scripts/build-client-binary
+++ b/build/scripts/build-client-binary
@@ -27,6 +27,10 @@ gobuild_component() {
     local archive=$(ls ${source_dir} | grep -e "${parent_component}-\w*\.tar\.gz" | head -n1)
     local build_command
 
+    if [ "$(uname -o)" = "Darwin" ]; then
+      local GOARCH=$(uname -m)
+    fi
+
     if [ -d "${root_dir}/go-build" ]; then
         volume_mounts+=" -v ${root_dir}/go-build:/home/builder/.cache/go-build"
     fi
diff --git a/build/scripts/build-client-rpm b/build/scripts/build-client-rpm
index 6ec65cc016..517993a868 100755
--- a/build/scripts/build-client-rpm
+++ b/build/scripts/build-client-rpm
@@ -25,14 +25,13 @@ main() {
 
         export pmm_version=$pmm_version
         export pmm_release=$pmm_release
-        export sudo_path=\$$(command -v sudo)
         mkdir -p /tmp/pmm
         cd /home/builder/results
         /home/builder/bin/build-client-packages \
             --builddir=/tmp/pmm \
             --build_rpm=1
 
-        \$sudo_path chown -R $(id -u):$(id -g) /home/builder/results/rpm
+        sudo_path chown -R $(id -u):$(id -g) /home/builder/results/rpm
     "
 }
 

From 23aaaf6f3c63970753ed645c22ff0aef899bf10e Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Sun, 20 Oct 2024 02:44:57 +0300
Subject: [PATCH 33/35] PMM-12899 fix the build time for MacOS

---
 build/local/build.sh              | 234 ++++++++++++++++++------------
 build/scripts/build-server-docker |   5 +-
 2 files changed, 148 insertions(+), 91 deletions(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index 76b0327185..0811217ca9 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -1,80 +1,86 @@
 #!/bin/bash -e
-
-# Define global variables
-NO_UPDATE=0
-UPDATE_ONLY=0
-NO_CLIENT=0
-NO_CLIENT_DOCKER=0
-NO_SERVER_RPM=0
-NO_SERVER_DOCKER=0
-START_TIME=$(date +%s)
-LOG_FILE="/tmp/build.log"
-BASE_NAME=$(basename $0)
-USAGE="Usage: $BASE_NAME [--no-update | --update-only] [--no-client] [--no-client-docker] [--no-server-rpm] [--log-file <path>] [--help | -h]"
-
-while test "$#" -gt 0; do
-  case "$1" in
-    --update-only)
-      UPDATE_ONLY=1; NO_UPDATE=0
-      ;;
-    --no-update)
-      NO_UPDATE=1
-      ;;
-    --no-client)
-      NO_CLIENT=1; NO_CLIENT_DOCKER=1
-      ;;
-    --no-client-docker)
-      NO_CLIENT_DOCKER=1
-      ;;
-    --no-server-rpm)
-      if [ "$NO_SERVER_DOCKER" -eq 1 ]; then
-        echo "Cannot disable both server RPM and server Docker"
-        exit 1
-      fi
-      NO_SERVER_RPM=1
-      ;;
-    --no-server-docker)
-      if [ "$NO_SERVER_RPM" -eq 1 ]; then
-        echo "Cannot disable both server RPM and server Docker"
-        exit 1
-      fi
-      NO_SERVER_DOCKER=1
-      ;;
-    --log-file)
-      shift
-      if [ -z "$1" ]; then
-        echo "Missing argument for --log-file"
-        exit 1
-      fi
-      LOG_FILE="$1"
-      ;;
-    --help | -h)
-      shift
-      usage
-      exit 0
-      ;;
-    *)
-      echo "Unknown argument: $1"
-      usage
-      exit 1
-      ;;
-  esac
-  shift
-done
+set -o errexit
+set -o nounset
 
 usage() {
   cat <<-EOF
-Usage: $BASE_NAME [--no-update | --update-only] [--no-client] [--no-client-docker] [--no-server-rpm] [--log-file <path>] [--help | -h]
+Usage: $BASE_NAME [--no-update | --update-only] [--no-client] [--no-client-docker] [--no-server-rpm] [--no-server-docker] [--log-file <path>] [--help | -h]
 --no-update              Do not fetch the latest changes from the repo
 --update-only            Only fetch the latest changes from the repo
---no-client              Do not (re)build PMM client
---no-client-docker       Do not (re)build PMM Client docker image
---no-server-rpm          Do not (re)build Server RPM packages
+--no-client              Do not build PMM Client
+--client-docker          Build PMM Client docker image
+--no-server-rpm          Do not build Server RPM packages
+--no-server-docker       Do not build PMM Server docker image
 --log-file <path>        Save build logs to a file located at <path>
 --help | -h              Display help
 EOF
 }
 
+parse-params() {
+  # Define global variables
+  NO_UPDATE=0
+  UPDATE_ONLY=0
+  NO_CLIENT=0
+  NO_CLIENT_DOCKER=1
+  NO_SERVER_RPM=0
+  NO_SERVER_DOCKER=0
+  START_TIME=$(date +%s)
+  LOG_FILE="$(dirname $0)/build.log"
+  BASE_NAME=$(basename $0)
+  SUBMODULES=pmm-submodules
+  PATH_TO_SCRIPTS="sources/pmm/src/github.com/percona/pmm/build/scripts"
+
+  while test "$#" -gt 0; do
+    case "$1" in
+      --update-only)
+        UPDATE_ONLY=1; NO_UPDATE=0
+        ;;
+      --no-update)
+        NO_UPDATE=1
+        ;;
+      --no-client)
+        NO_CLIENT=1; NO_CLIENT_DOCKER=1
+        ;;
+      --client-docker)
+        NO_CLIENT_DOCKER=0
+        ;;
+      --no-server-rpm)
+        if [ "$NO_SERVER_DOCKER" -eq 1 ]; then
+          echo "Error: cannot disable both server RPM and server Docker"
+          exit 1
+        fi
+        NO_SERVER_RPM=1
+        ;;
+      --no-server-docker)
+        if [ "$NO_SERVER_RPM" -eq 1 ]; then
+          echo "Error: cannot disable both server RPM and server Docker"
+          exit 1
+        fi
+        NO_SERVER_DOCKER=1
+        ;;
+      --log-file)
+        shift
+        if [ -z "$1" ]; then
+          echo "Missing argument for --log-file"
+          exit 1
+        fi
+        LOG_FILE="$1"
+        ;;
+      --help | -h)
+        shift
+        usage
+        exit 0
+        ;;
+      *)
+        echo "Unknown argument: $1"
+        usage
+        exit 1
+        ;;
+    esac
+    shift
+  done
+}
+
 needs-to-pull() {
   local UPSTREAM=${1:-'@{u}'}
   local LOCAL=$(git rev-parse @)
@@ -95,7 +101,7 @@ rewind() {
   local BRANCH="$2"
 
   cd "$DIR" > /dev/null
-  CURRENT=$(git branch --show-current)
+  local CURRENT=$(git branch --show-current)
   git fetch
 
   if [ "$CURRENT" != "$BRANCH" ]; then
@@ -130,14 +136,17 @@ check-files() {
 update() {
   local DEPS=
   local CURDIR="$PWD"
-  local DIR=pmm-submodules
 
   # Thouroughly verify the presence of known files, otherwise bail out
-  if check-files "."; then # pwd is pmm-submodules
-    DIR="."
-  elif [ -d "$DIR" ]; then # pwd is outside pmm-submodules
-    if ! check-files "$DIR"; then
-      echo "Fatal: could not locate known files in ${PWD}/${DIR}"
+  if [ ! -d "$SUBMODULES" ] ; then # pwd must outside of pmm-submodules
+    echo "Warn: the current working directory must be outside of pmm-submodules"
+    echo "cd .."
+    cd .. > /dev/null
+  fi
+
+  if [ -d "$SUBMODULES" ]; then # pwd is outside pmm-submodules
+    if ! check-files "$SUBMODULES"; then
+      echo "Fatal: could not locate known files in ${PWD}/${SUBMODULES}"
       exit 1
     fi
   else
@@ -145,7 +154,7 @@ update() {
     exit 1
   fi
 
-  cd "$DIR"
+  cd "$SUBMODULES"
 
   # Join the dependencies from ci-default.yml and ci.yml
   DEPS=$(yq -o=json eval-all '. as $item ireduce ({}; . *d $item )' ci-default.yml ci.yml | jq '.deps')
@@ -186,13 +195,17 @@ get_branch_name() {
 }
 
 build_with_logs() {
+  local CURDIR="$PWD"
   local script="$PATH_TO_SCRIPTS/$1"
   local start_time
   local end_time
   local script_name="$1"
 
+  cd "$SUBMODULES" > /dev/null
+
   if [ ! -f "$script" ]; then
     echo "Fatal: script $script does not exist"
+    cd "$CURDIR" > /dev/null
     exit 1
   fi
 
@@ -209,43 +222,82 @@ build_with_logs() {
   echo ---
   echo "Execution time (in sec) for $script_name: $((end_time - start_time))" | tee -a $LOG_FILE
   echo ---
+
+  cd "$CURDIR" > /dev/null
 }
 
 purge_files() {
+  local CURDIR=$PWD
   local tmp_files
+
+  cd "$SUBMODULES" > /dev/null
   # Remove stale files and directories
   if [ -d tmp ]; then
     echo "Removing stale files and directories..."
+
     if [ -d "tmp/pmm-server" ]; then
       tmp_files=$(find tmp/pmm-server | grep -v "RPMS" | grep -Ev "^tmp/pmm-server$" || :)
-      tmp_files=($tmp_files)
-      for f in "${tmp_files[@]}"; do
-        rm -rf "$f"
-      done
+      if [ -n "$tmp_files" ]; then
+        tmp_files=( $tmp_files )
+        for f in "${tmp_files[@]}"; do
+          rm -rf "$f"
+        done
+      fi
     fi
+
     if [ -d "tmp/source/pmm" ]; then
       rm -rf tmp/source/pmm
     fi
   fi
+
   if [ -f "$LOG_FILE" ]; then
     echo "Removing the log file..."
     rm -f $LOG_FILE
   fi
+
+  cd "$CURDIR"
 }
 
 init() {
-  PATH_TO_SCRIPTS="sources/pmm/src/github.com/percona/pmm/build/scripts"
-  "${PATH_TO_SCRIPTS}/build-submodules"
+  local CURDIR="$PWD"
+
   export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:3
 
+  if [ -d "$SUBMODULES" ]; then
+    cd "$SUBMODULES" > /dev/null
+  fi
+
+  pmm_commit=$(git submodule status | grep 'sources/pmm/src' | awk -F ' ' '{print $1}')
+  pmm_branch=$(git config -f .gitmodules submodule.pmm.branch)
+  pmm_url=$(git config -f .gitmodules submodule.pmm.url)
+  pmm_qa_branch=$(git config -f .gitmodules submodule.pmm-qa.branch)
+  pmm_qa_commit=$(git submodule status | grep 'pmm-qa' | awk -F ' ' '{print $1}')
+  pmm_ui_tests_branch=$(git config -f .gitmodules submodule.pmm-ui-tests.branch)
+  pmm_ui_tests_commit=$(git submodule status | grep 'pmm-ui-tests' | awk -F ' ' '{print $1}')
+  fb_commit_sha=$(git rev-parse HEAD)
+
+  echo $fb_commit_sha > fbCommitSha
+  echo $pmm_commit > apiCommitSha
+  echo $pmm_branch > apiBranch
+  echo $pmm_url > apiURL
+  echo $pmm_qa_branch > pmmQABranch
+  echo $pmm_qa_commit > pmmQACommitSha
+  echo $pmm_ui_tests_branch > pmmUITestBranch
+  echo $pmm_ui_tests_commit > pmmUITestsCommitSha
+
   # Create cache directories. Read more in the section about `rpmbuild`.
-  test -d "${root_dir}/go-path" || mkdir -p "go-path"
-  test -d "${root_dir}/go-build" || mkdir -p "go-build"
-  test -d "${root_dir}/yarn-cache" || mkdir -p "yarn-cache"
-  test -d "${root_dir}/yum-cache" || mkdir -p "yum-cache"
+  test -d "go-path" || mkdir -p "go-path"
+  test -d "go-build" || mkdir -p "go-build"
+  test -d "yarn-cache" || mkdir -p "yarn-cache"
+  test -d "yum-cache" || mkdir -p "yum-cache"
+
+  cd "$CURDIR" > /dev/null
 }
 
 cleanup() {
+  local CURDIR="$PWD"
+  cd "$SUBMODULES" > /dev/null
+
   # Clean up temporary files
   rm -f apiBranch \
     apiCommitSha \
@@ -254,13 +306,15 @@ cleanup() {
     pmmQABranch \
     pmmQACommitSha \
     pmmUITestBranch \
-    pmmUITestsCommitSha
+    pmmUITestsCommitSha || :
+
+  cd "$CURDIR" > /dev/null
 }
 
 main() {
   if [ "$NO_UPDATE" -eq 0 ]; then
     MD5SUM=$(md5sum $(dirname $0)/build.sh)
-    
+
     # Update submodules and PR branches
     update
 
@@ -274,6 +328,7 @@ main() {
   fi
 
   init
+
   purge_files
 
   if [ "$NO_CLIENT" -eq 0 ]; then
@@ -301,7 +356,6 @@ main() {
   # total time: 6m26s - build from scratch, no initial cache
   # total time: 2m49s - subsequent build, using cache from prior builds
 
-
   # Building PMM CLient in a CI environment, i.e. Jenkins running on AWS
   # total time: 8m45s - build from scratch, no initial cache
   # total time: ??? - subsequent build, using cache from prior builds
@@ -335,9 +389,11 @@ main() {
   cleanup
 }
 
-# Local reference test environment
+# Reference test environment
 # CPU: 4 cores
-# RAM: 16GB
+# RAM: 16 GB
 # OS: Ubuntu 22.04.1 LTS
 
+parse-params "$@"
+
 main
diff --git a/build/scripts/build-server-docker b/build/scripts/build-server-docker
index 005fbee466..dd096f5c31 100755
--- a/build/scripts/build-server-docker
+++ b/build/scripts/build-server-docker
@@ -34,10 +34,11 @@ main() {
       DOCKER_TAG=perconalab/pmm-server-fb:${full_pmm_version}
   fi
 
-  IMAGE_VERSION=$(echo "$DOCKER_TAG" | cut -d ':' -f2)
+  local IMAGE_VERSION=$(echo "$DOCKER_TAG" | cut -d ':' -f2)
+  local BUILD_DATE=$(date -u +'%F %T%z' | sed 's@^.\{22\}@&:@')
 
   docker buildx build \
-      --build-arg BUILD_DATE="$(date --rfc-3339=seconds)" \
+      --build-arg BUILD_DATE="$BUILD_DATE" \
       --build-arg VERSION="$IMAGE_VERSION" \
       --progress plain \
       -f ${docker_root}/${docker_file} \

From c2357c5d4674af6ae05cda4f5cbc831482cab688 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Sun, 20 Oct 2024 03:06:33 +0300
Subject: [PATCH 34/35] PMM-12899 fix git commit variable

---
 build/local/build.sh | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index 0811217ca9..723945780d 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -267,6 +267,8 @@ init() {
     cd "$SUBMODULES" > /dev/null
   fi
 
+  GIT_COMMIT=$(git rev-parse HEAD | head -c 8)
+
   pmm_commit=$(git submodule status | grep 'sources/pmm/src' | awk -F ' ' '{print $1}')
   pmm_branch=$(git config -f .gitmodules submodule.pmm.branch)
   pmm_url=$(git config -f .gitmodules submodule.pmm.url)
@@ -313,6 +315,7 @@ cleanup() {
 
 main() {
   if [ "$NO_UPDATE" -eq 0 ]; then
+    local UPDATED_SCRIPT="$SUBMODULES/$PATH_TO_SCRIPTS/build/local/build.sh"
     MD5SUM=$(md5sum $(dirname $0)/build.sh)
 
     # Update submodules and PR branches
@@ -320,9 +323,10 @@ main() {
 
     test "$UPDATE_ONLY" -eq 1 && return
 
-    if [ "$MD5SUM" != "$(md5sum $(dirname $0)/build.sh)" ]; then
-      echo "The updated version of this script has been fetched from the repository, exiting..."
-      echo "Please run it again, i.e. '/bin/bash $(dirname $0)/build.sh --no-update'"
+    if [ -f "$UPDATED_SCRIPT" ] && [ "$MD5SUM" != "$(md5sum $UPDATED_SCRIPT)" ]; then
+      echo "The local copy of this script differs from the one fetched from the repo." 
+      echo "Apparently, that version is newer. We will halt to give you the change to run a fresh version."
+      echo "You can copy it over and run it again, i.e. '/bin/bash $(dirname $0)/build.sh --no-update'"
       return
     fi
   fi
@@ -346,7 +350,6 @@ main() {
   fi
 
   # Building client docker image takes 17s
-  GIT_COMMIT=$(git rev-parse HEAD | head -c 8)
   export DOCKER_CLIENT_TAG=local/pmm-client:${GIT_COMMIT}
   if [ "$NO_CLIENT_DOCKER" -eq 0 ]; then
     build_with_logs build-client-docker

From 79250792ebf331db5edfc8a36550d98300658736 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Sun, 20 Oct 2024 15:28:58 +0300
Subject: [PATCH 35/35] PMM-12899 don't use S3 cache for local builds

---
 build/local/build.sh | 1 +
 build/scripts/vars   | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index 723945780d..e52e60b3d5 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -364,6 +364,7 @@ main() {
   # total time: ??? - subsequent build, using cache from prior builds
 
   export RPM_EPOCH=1
+  export FORCE_REBUILD=1 # Don't use S3 cache
   if [ "$NO_SERVER_RPM" -eq 0 ]; then
     build_with_logs build-server-rpm percona-dashboards grafana-dashboards
     build_with_logs build-server-rpm pmm-managed pmm
diff --git a/build/scripts/vars b/build/scripts/vars
index 338e6512e2..7a7d15fb3b 100644
--- a/build/scripts/vars
+++ b/build/scripts/vars
@@ -38,7 +38,7 @@ unset new_pmm_version
 unset new_pmm_release
 echo -e "\n\n\n>>> full_pmm_version=${full_pmm_version} pmm_version=${pmm_version} pmm_release=${pmm_release}\n\n\n"
 
-rpmbuild_docker_image=${RPMBUILD_DOCKER_IMAGE:-public.ecr.aws/e7j3v3n0/rpmbuild:2}
+rpmbuild_docker_image=${RPMBUILD_DOCKER_IMAGE:-public.ecr.aws/e7j3v3n0/rpmbuild:3}
 rpms_dir=${root_dir}/tmp/pmm-server/RPMS
 rpmbuild_dir=${root_dir}/sources/pmm/src/github.com/percona/pmm/build/packages/rpm/server
 rpmbuild_dist=${RPMBUILD_DIST:-"el9"}