From e2b498566043eb6081ea6f4703eed12307f066f6 Mon Sep 17 00:00:00 2001 From: "m.desoutter" Date: Thu, 31 Oct 2024 15:21:15 +0100 Subject: [PATCH 1/3] build: introduce Docker build system and more Vertica SDK support --- Dockerfile | 7 +++++++ README.md | 47 ++++++++++++++++++++++++++++++++++++++++++ SOURCES/CMakeLists.txt | 16 ++++++++++++++ start-build-env.sh | 9 ++++++++ 4 files changed, 79 insertions(+) create mode 100644 Dockerfile create mode 100755 start-build-env.sh diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..a5dec5c --- /dev/null +++ b/Dockerfile @@ -0,0 +1,7 @@ +FROM gcc:8.5.0-buster +RUN echo deb http://archive.debian.org/debian buster-backports main contrib non-free | tee -a /etc/apt/sources.list +RUN apt-get update && apt-get -y install cmake/buster-backports +RUN mkdir -p /opt/vertica/sdk /sources /build +RUN useradd builder && chown builder: /opt/vertica /sources/ build +WORKDIR /build +USER builder diff --git a/README.md b/README.md index bbf1f9d..1856e57 100644 --- a/README.md +++ b/README.md @@ -423,6 +423,53 @@ If the container gets somehow corrupted, destroy the volume with persistent stor docker volume rm docker-vertica-itests ``` + + +## Updated build info + +### Install proper GCC/G++ for Vertica + +Vertica needs (as of 24.4) G++ version 8. You can't get that decently well on Ubuntu Jammy+. + +So, we're going to build using Docker! This section will not help you install old GCC on your recent machine, it's not reliable. + + +### Get the Vertica SDK from a Vertica instance or a Docker image. + +If you have a Vertica deployment, just get the SDK from `/opt/vertica/sdk` + +If you don't have one handy, use: + +`docker run --rm opentext/vertica-k8s:24.4.0-0 tar -C /opt/vertica -c -v sdk > /tmp/vertica-sdk.tar` + +### Install the Vertica SDK somewhere useful + +On your dev machine (don't do that on a Vertica server, you'll mess things up) + +```bash +# load the SDK to a dev directory in your home +mkdir -p ~/dev/vertica-sdks +cd ~/dev/vertica-sdks +tar xf /tmp/vertica-sdk.tar +mv sdk 24.4.0 +# link the expected SDK location to your local SDK copy +sudo mkdir -p /opt/vertica && sudo chown -R $(id -u) /opt/vertica +rm /opt/vertica/sdk; ln -sf ~/dev/vertica-sdks/24.4.0 /opt/vertica/sdk +``` + +### Run the build environment + +You need Docker and the SDK properly ready in /opt/vertica/sdk. Run `./start-build-env.sh` and then drop to the shell: `docker exec -ti vbuilder-hyperloglog bash` + +You may then try to build using (inside the container) + +```bash +cd /build +cmake /sources +make +``` + + ## Licensing This work is distributed under the Apache License, Version 2.0. diff --git a/SOURCES/CMakeLists.txt b/SOURCES/CMakeLists.txt index f169c54..8659025 100644 --- a/SOURCES/CMakeLists.txt +++ b/SOURCES/CMakeLists.txt @@ -17,6 +17,22 @@ option(BUILD_DATA_GEN "Build data generator for functional tests in Vertica" OFF option(BUILD_TESTS "Build all tests." OFF) option(BUILD_BENCHMARK "Build benchmark to run HLL estimations." OFF) + +## When building for Vertica 11, you need to set specific defines +file(READ "/opt/vertica/sdk/include/BuildInfo.h" buildinfosdk) +string(REGEX MATCH "#define[ ]+VERTICA_BUILD_ID_SDK_Version_Major[ ]+([0-9]+)" _ ${buildinfosdk}) +set(ver_major ${CMAKE_MATCH_1}) +message("Version", ${ver_major}) +if (${ver_major} LESS_EQUAL 11) + add_definitions(-D_GLIBCXX_USE_CXX11_ABI=0) + message("Using CXX11 ABI because Vertica <= 11:", ${ver_major}) +else() + message("Not using CXX11 ABI because newer SDK than Vertica 11", ${ver_major}) +endif() + +## For Vertica 24, you don't need it. + + ##################### ## COMPILE FLAGS ## ##################### diff --git a/start-build-env.sh b/start-build-env.sh new file mode 100755 index 0000000..5e146e7 --- /dev/null +++ b/start-build-env.sh @@ -0,0 +1,9 @@ +#!/bin/bash +docker build -t vertica-builder:local . + +rm -rf build +mkdir -p build +docker kill vbuilder-hyperloglog || true +docker rm vbuilder-hyperloglog || true +docker run --rm --name vbuilder-hyperloglog -v=$PWD/SOURCES:/sources -v=$PWD/build:/build -v=/opt/vertica/sdk:/opt/vertica/sdk -d vertica-builder:local sleep 3600 +echo "Run docker exec -ti vbuilder-hyperloglog bash # now" From 9dfc02cf46b37b2a5b73ef6c6ac7cc9a9ebad8b9 Mon Sep 17 00:00:00 2001 From: "m.desoutter" Date: Wed, 13 Nov 2024 09:47:39 +0100 Subject: [PATCH 2/3] build: disable -march=native --- SOURCES/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/SOURCES/CMakeLists.txt b/SOURCES/CMakeLists.txt index 8659025..5257802 100644 --- a/SOURCES/CMakeLists.txt +++ b/SOURCES/CMakeLists.txt @@ -68,8 +68,8 @@ if (BUILD_VERTICA_LIB) add_library(hll SHARED ${HLL_SRC} src/hll-criteo/HllCombine.cpp src/hll-criteo/HllDistinctCount.cpp src/hll-criteo/HllCreateSynopsis.cpp) add_library(loglogbeta SHARED ${HLL_SRC} src/hll-criteo/LogLogBetaDistinctCount.cpp) - set_target_properties(hll PROPERTIES COMPILE_FLAGS "${CMAKE_CXX_FLAGS} -march=native") - set_target_properties(loglogbeta PROPERTIES COMPILE_FLAGS "${CMAKE_CXX_FLAGS} -march=native") + set_target_properties(hll PROPERTIES COMPILE_FLAGS "${CMAKE_CXX_FLAGS}") + set_target_properties(loglogbeta PROPERTIES COMPILE_FLAGS "${CMAKE_CXX_FLAGS}") # Installation process just copies the binary HLL to Vertica lib folder if (INSTALL_PREFIX_PATH) From 47c85821038324dae09e6f6b09c0dcd5b2224100 Mon Sep 17 00:00:00 2001 From: "m.desoutter" Date: Wed, 13 Nov 2024 14:35:56 +0100 Subject: [PATCH 3/3] build: add docs and details --- BUILD.md | 95 ++++++++++++++++++++++++++++++++++++++++++++++ centos8.Dockerfile | 8 ++++ start-build-env.sh | 24 +++++++++--- 3 files changed, 122 insertions(+), 5 deletions(-) create mode 100644 BUILD.md create mode 100644 centos8.Dockerfile diff --git a/BUILD.md b/BUILD.md new file mode 100644 index 0000000..f3f4943 --- /dev/null +++ b/BUILD.md @@ -0,0 +1,95 @@ +# Quick building/compilation guide + +## Install Vertica SDK (or SDKs) + +Obtain the SDK or SDKs you need from Vertica instances, from Vertica Docker images... + +The active SDK should be accessible through `/opt/vertica/sdk` so that the path +`/opt/vertica/sdk/include/BuildInfo.h` exists and belongs to the wanted SDK. + +This may be achieved through the following, assuming the contents of `/opt/vertica/sdk` +is in the tarball `/tmp/vertica-sdk-24.0.x.tgz`: + +```bash +# 1: create the expected path for a Vertica SDK and make yourself its owner +sudo mkdir -p /opt/vertica/; sudo chown $(whoami): /opt/vertica +# 2: create the directory for SDKs in your home +mkdir -p $HOME/dev/vertica-sdk/24.0.x +# 3: extract the SDK to the directory you just created +tar -C $HOME/dev/vertica-sdk/24.0.x -xvf /tmp/vertica-sdk-24.0.x.tgz +# 4: make this SDK the active SDK +rm /opt/vertica/sdk; ln -s $HOME/dev/vertica-sdk/24.0.x /opt/vertica/sdk +# if you want to add other SDKs, repeat steps 2 and 3. +# if you want to enable another SDK, repeat step 4. +``` + +## Enable the Vertica SDK you want to compile against + +See previous step. + +## Install Docker + +Refer to the documentation for your environment. + +## Build and run the builder Docker image + +Run `./start-build-env.sh` and follow the instructions. + +This is an example run: + +```bash +# ./start-build-env.sh +[+] Building 1.3s (12/12) FINISHED docker:default + => [internal] load build definition from centos8.Dockerfile 0.0s + => => transferring dockerfile: 435B 0.0s + => [internal] load metadata for docker.io/library/rockylinux:8.5.20220308 1.3s + => [auth] library/rockylinux:pull token for registry-1.docker.io 0.0s + => [internal] load .dockerignore 0.0s + => => transferring context: 2B 0.0s + => [1/7] FROM docker.io/library/rockylinux:8.5.20220308@sha256:c7d13ea4d57355aaad6b6ebcdcca50f5be65fc821f54161430f5c25 0.0s + => CACHED [2/7] RUN dnf -y install cmake gcc-toolset-9 cmake gcc-toolset-9-gcc-c++ 0.0s + => CACHED [3/7] RUN mkdir -p /opt/vertica/sdk /sources /build 0.0s + => CACHED [4/7] RUN useradd builder && chown builder: /opt/vertica /sources/ build 0.0s + => CACHED [5/7] WORKDIR /build 0.0s + => CACHED [6/7] RUN echo 'source scl_source enable gcc-toolset-9' >> ~/.bashrc 0.0s + => CACHED [7/7] RUN echo 'echo "# you may build using: cmake /sources; make;" 1>&2; echo;' >> ~/.bashrc 0.0s + => exporting to image 0.0s + => => exporting layers 0.0s + => => writing image sha256:d87191b04ef3897d1ec3962e7228acc36e8355e07634d60a937872c81da2b96c 0.0s + => => naming to docker.io/library/vertica-builder-hll-druid:local 0.0s +vbuilder-hll-druid +Error response from daemon: No such container: vbuilder-hll-druid +94d81b98487f2aea8480969ad4f9c2ac5a3b36200c97aa626bf99e44c2d07a02 +Run docker exec -ti vbuilder-hll-druid bash # now +Then, you may run the following to build +cmake /sources; make clean; make +``` + +## Perform any development you want + +You may touch the source code in the SOURCES directory. + +## Build and recover built assets + +Within the container, run the following: + +```bash +# you may want to cleanup the /build directory +rm -rf /build/* +# ensure you're in /build (you should be, by default) +cd /build +# prepare the environment with CMake +# you also want to run that if you modify the CMakeLists.txt or other components +cmake /sources +# build the code +make +``` + +You will find the UDx as a .so file in the build directory of your local copy. + +## Build for another SDK + +- Perform the SDK change as indicated in this document. +- Then, **re-run start-build-env.sh** because the container needs to restart to take in + account the new SDK. +- Just build again. diff --git a/centos8.Dockerfile b/centos8.Dockerfile new file mode 100644 index 0000000..7e27fc0 --- /dev/null +++ b/centos8.Dockerfile @@ -0,0 +1,8 @@ +FROM rockylinux:8.5.20220308 +RUN dnf -y install cmake gcc-toolset-9 cmake gcc-toolset-9-gcc-c++ +RUN mkdir -p /opt/vertica/sdk /sources /build +RUN useradd builder && chown builder: /opt/vertica /sources/ build +WORKDIR /build +USER builder +RUN echo 'source scl_source enable gcc-toolset-9' >> ~/.bashrc +RUN echo 'echo "# you may build using: cmake /sources; make;" 1>&2; echo;' >> ~/.bashrc diff --git a/start-build-env.sh b/start-build-env.sh index 5e146e7..d3806dc 100755 --- a/start-build-env.sh +++ b/start-build-env.sh @@ -1,9 +1,23 @@ #!/bin/bash -docker build -t vertica-builder:local . +_b_env=${1:-centos8.Dockerfile} +errout() { + _r=$1 + shift 1; + echo $@; + exit $_r; +} +BUILDER_IMAGE="vertica-builder-hyperloglog:local" +BUILDER_INSTANCE_NAME="vbuilder-hyperloglog" +docker build -f ${_b_env} -t ${BUILDER_IMAGE} . +test -f /opt/vertica/sdk/include/BuildInfo.h || errout 30 "Vertica SDK is missing. Put it in /opt/vertica/sdk so that /opt/vertica/sdk/include/BuildInfo.h exists" rm -rf build mkdir -p build -docker kill vbuilder-hyperloglog || true -docker rm vbuilder-hyperloglog || true -docker run --rm --name vbuilder-hyperloglog -v=$PWD/SOURCES:/sources -v=$PWD/build:/build -v=/opt/vertica/sdk:/opt/vertica/sdk -d vertica-builder:local sleep 3600 -echo "Run docker exec -ti vbuilder-hyperloglog bash # now" +docker kill ${BUILDER_INSTANCE_NAME} || true +sleep 1 +docker rm ${BUILDER_INSTANCE_NAME} || true +sleep 1 +docker run --rm --name ${BUILDER_INSTANCE_NAME} -v=$PWD/SOURCES:/sources -v=$PWD/build:/build -v=/opt/vertica/sdk:/opt/vertica/sdk -d $BUILDER_IMAGE sleep 3600 +echo "Run docker exec -ti ${BUILDER_INSTANCE_NAME} bash # now" +echo "Then, you may run the following to build" +echo "cmake /sources; make clean; make"