build: introduce Docker build system and more Vertica SDK support

BUILD.md

@@ -0,0 +1,95 @@
+# Quick building/compilation guide
+
+## Install Vertica SDK (or SDKs)
+
+Obtain the SDK or SDKs you need from Vertica instances, from Vertica Docker images...
+
+The active SDK should be accessible through `/opt/vertica/sdk` so that the path
+`/opt/vertica/sdk/include/BuildInfo.h` exists and belongs to the wanted SDK.
+
+This may be achieved through the following, assuming the contents of `/opt/vertica/sdk`
+is in the tarball `/tmp/vertica-sdk-24.0.x.tgz`:
+
+```bash
+# 1: create the expected path for a Vertica SDK and make yourself its owner
+sudo mkdir -p /opt/vertica/; sudo chown $(whoami): /opt/vertica
+# 2: create the directory for SDKs in your home
+mkdir -p $HOME/dev/vertica-sdk/24.0.x
+# 3: extract the SDK to the directory you just created
+tar -C $HOME/dev/vertica-sdk/24.0.x -xvf /tmp/vertica-sdk-24.0.x.tgz
+# 4: make this SDK the active SDK
+rm /opt/vertica/sdk; ln -s $HOME/dev/vertica-sdk/24.0.x /opt/vertica/sdk
+# if you want to add other SDKs, repeat steps 2 and 3.
+# if you want to enable another SDK, repeat step 4.
+```
+
+## Enable the Vertica SDK you want to compile against
+
+See previous step.
+
+## Install Docker
+
+Refer to the documentation for your environment.
+
+## Build and run the builder Docker image
+
+Run `./start-build-env.sh` and follow the instructions.
+
+This is an example run:
+
+```bash
+# ./start-build-env.sh
+[+] Building 1.3s (12/12) FINISHED                                                                             docker:default
+ => [internal] load build definition from centos8.Dockerfile                                                             0.0s
+ => => transferring dockerfile: 435B                                                                                     0.0s
+ => [internal] load metadata for docker.io/library/rockylinux:8.5.20220308                                               1.3s
+ => [auth] library/rockylinux:pull token for registry-1.docker.io                                                        0.0s
+ => [internal] load .dockerignore                                                                                        0.0s
+ => => transferring context: 2B                                                                                          0.0s
+ => [1/7] FROM docker.io/library/rockylinux:8.5.20220308@sha256:c7d13ea4d57355aaad6b6ebcdcca50f5be65fc821f54161430f5c25  0.0s
+ => CACHED [2/7] RUN dnf -y install cmake gcc-toolset-9 cmake gcc-toolset-9-gcc-c++                                      0.0s
+ => CACHED [3/7] RUN mkdir -p /opt/vertica/sdk /sources /build                                                           0.0s
+ => CACHED [4/7] RUN useradd builder && chown builder: /opt/vertica /sources/ build                                      0.0s
+ => CACHED [5/7] WORKDIR /build                                                                                          0.0s
+ => CACHED [6/7] RUN echo 'source scl_source enable gcc-toolset-9' >> ~/.bashrc                                          0.0s
+ => CACHED [7/7] RUN echo 'echo "# you may build using: cmake /sources; make;" 1>&2; echo;' >> ~/.bashrc                 0.0s
+ => exporting to image                                                                                                   0.0s
+ => => exporting layers                                                                                                  0.0s
+ => => writing image sha256:d87191b04ef3897d1ec3962e7228acc36e8355e07634d60a937872c81da2b96c                             0.0s
+ => => naming to docker.io/library/vertica-builder-hll-druid:local                                                       0.0s
+vbuilder-hll-druid
+Error response from daemon: No such container: vbuilder-hll-druid
+94d81b98487f2aea8480969ad4f9c2ac5a3b36200c97aa626bf99e44c2d07a02
+Run docker exec -ti vbuilder-hll-druid bash # now
+Then, you may run the following to build
+cmake /sources; make clean; make
+```
+
+## Perform any development you want
+
+You may touch the source code in the SOURCES directory.
+
+## Build and recover built assets
+
+Within the container, run the following:
+
+```bash
+# you may want to cleanup the /build directory
+rm -rf /build/*
+# ensure you're in /build (you should be, by default)
+cd /build
+# prepare the environment with CMake
+# you also want to run that if you modify the CMakeLists.txt  or other components
+cmake /sources
+# build the code
+make
+```
+
+You will find the UDx as a .so file in the build directory of your local copy.
+
+## Build for another SDK
+
+- Perform the SDK change as indicated in this document.
+- Then, **re-run start-build-env.sh** because the container needs to restart to take in
+  account the new SDK.
+- Just build again.

Dockerfile

@@ -0,0 +1,7 @@
+FROM gcc:8.5.0-buster
+RUN echo deb http://archive.debian.org/debian buster-backports main contrib non-free | tee -a /etc/apt/sources.list
+RUN apt-get update && apt-get -y install cmake/buster-backports
+RUN mkdir -p /opt/vertica/sdk /sources /build
+RUN useradd builder && chown builder: /opt/vertica /sources/ build
+WORKDIR /build
+USER builder

README.md

@@ -423,6 +423,53 @@ If the container gets somehow corrupted, destroy the volume with persistent stor
 docker volume rm docker-vertica-itests
 ```
 
+
+
+## Updated build info
+
+### Install proper GCC/G++ for Vertica
+
+Vertica needs (as of 24.4) G++ version 8. You can't get that decently well on Ubuntu Jammy+.
+
+So, we're going to build using Docker! This section will not help you install old GCC on your recent machine, it's not reliable.
+
+
+### Get the Vertica SDK from a Vertica instance or a Docker image.
+
+If you have a Vertica deployment, just get the SDK from `/opt/vertica/sdk`
+
+If you don't have one handy, use:
+
+`docker run --rm opentext/vertica-k8s:24.4.0-0 tar -C /opt/vertica -c -v sdk > /tmp/vertica-sdk.tar`
+
+### Install the Vertica SDK somewhere useful
+
+On your dev machine (don't do that on a Vertica server, you'll mess things up)
+
+```bash
+# load the SDK to a dev directory in your home
+mkdir -p ~/dev/vertica-sdks
+cd ~/dev/vertica-sdks
+tar xf /tmp/vertica-sdk.tar
+mv sdk 24.4.0
+# link the expected SDK location to your local SDK copy
+sudo mkdir -p /opt/vertica && sudo chown -R $(id -u) /opt/vertica
+rm /opt/vertica/sdk; ln -sf ~/dev/vertica-sdks/24.4.0 /opt/vertica/sdk
+```
+
+### Run the build environment
+
+You need Docker and the SDK properly ready in /opt/vertica/sdk. Run `./start-build-env.sh` and then drop to the shell: `docker exec -ti vbuilder-hyperloglog bash`
+
+You may then try to build using (inside the container)
+
+```bash
+cd /build
+cmake /sources
+make
+```
+
+
 ## Licensing
 This work is distributed under the Apache License, Version 2.0.
 

SOURCES/CMakeLists.txt

@@ -17,6 +17,22 @@ option(BUILD_DATA_GEN "Build data generator for functional tests in Vertica" OFF
 option(BUILD_TESTS "Build all tests." OFF)
 option(BUILD_BENCHMARK "Build benchmark to run HLL estimations." OFF)
 
+
+## When building for Vertica 11, you need to set specific defines
+file(READ "/opt/vertica/sdk/include/BuildInfo.h" buildinfosdk)
+string(REGEX MATCH "#define[ ]+VERTICA_BUILD_ID_SDK_Version_Major[ ]+([0-9]+)" _ ${buildinfosdk})
+set(ver_major ${CMAKE_MATCH_1})
+message("Version", ${ver_major})
+if (${ver_major} LESS_EQUAL 11)
+  add_definitions(-D_GLIBCXX_USE_CXX11_ABI=0)
+  message("Using CXX11 ABI because Vertica <= 11:", ${ver_major})
+else()
+  message("Not using CXX11 ABI because newer SDK than Vertica 11", ${ver_major})
+endif()
+
+## For Vertica 24, you don't need it.
+
+
 #####################
 ##  COMPILE FLAGS  ##
 #####################
@@ -52,8 +68,8 @@ if (BUILD_VERTICA_LIB)
 
   add_library(hll SHARED ${HLL_SRC} src/hll-criteo/HllCombine.cpp src/hll-criteo/HllDistinctCount.cpp src/hll-criteo/HllCreateSynopsis.cpp)
   add_library(loglogbeta SHARED ${HLL_SRC} src/hll-criteo/LogLogBetaDistinctCount.cpp)
-  set_target_properties(hll PROPERTIES COMPILE_FLAGS "${CMAKE_CXX_FLAGS} -march=native")
-  set_target_properties(loglogbeta PROPERTIES COMPILE_FLAGS "${CMAKE_CXX_FLAGS} -march=native")
+  set_target_properties(hll PROPERTIES COMPILE_FLAGS "${CMAKE_CXX_FLAGS}")
+  set_target_properties(loglogbeta PROPERTIES COMPILE_FLAGS "${CMAKE_CXX_FLAGS}")
 
   # Installation process just copies the binary HLL to Vertica lib folder
   if (INSTALL_PREFIX_PATH)

centos8.Dockerfile

@@ -0,0 +1,8 @@
+FROM rockylinux:8.5.20220308
+RUN dnf -y install cmake gcc-toolset-9 cmake gcc-toolset-9-gcc-c++
+RUN mkdir -p /opt/vertica/sdk /sources /build
+RUN useradd builder && chown builder: /opt/vertica /sources/ build
+WORKDIR /build
+USER builder
+RUN echo 'source scl_source enable gcc-toolset-9' >> ~/.bashrc
+RUN echo 'echo "# you may build using: cmake /sources; make;" 1>&2; echo;' >> ~/.bashrc

start-build-env.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+_b_env=${1:-centos8.Dockerfile}
+errout() {
+  _r=$1
+  shift 1;
+  echo $@;
+  exit $_r;
+}
+BUILDER_IMAGE="vertica-builder-hyperloglog:local"
+BUILDER_INSTANCE_NAME="vbuilder-hyperloglog"
+docker build -f ${_b_env} -t ${BUILDER_IMAGE} .
+test -f /opt/vertica/sdk/include/BuildInfo.h || errout 30 "Vertica SDK is missing. Put it in /opt/vertica/sdk so that /opt/vertica/sdk/include/BuildInfo.h exists"
+
+rm -rf build
+mkdir -p build
+docker kill ${BUILDER_INSTANCE_NAME} || true
+sleep 1
+docker rm ${BUILDER_INSTANCE_NAME} || true
+sleep 1
+docker run --rm --name ${BUILDER_INSTANCE_NAME} -v=$PWD/SOURCES:/sources -v=$PWD/build:/build -v=/opt/vertica/sdk:/opt/vertica/sdk  -d $BUILDER_IMAGE sleep 3600
+echo "Run docker exec -ti ${BUILDER_INSTANCE_NAME} bash # now"
+echo "Then, you may run the following to build"
+echo "cmake /sources; make clean; make"