Skip to content

Commit

Permalink
[DAPHNE-daphne-eu#530] Docker Improvements
Browse files Browse the repository at this point in the history
* reduced to two images:
  - daphneeu/daphne - ready-to-run daphne precompiled with minimal deps
  - daphneeu/daphne-dev - everything needed for developing DAPHNE itself
* ssh access for daphne-dev: this comes handy when using IDE extensions like VSCode Remote SSH or CLion
* CUDA 12.1
* CUDNN 8.9
* building deps from future-deps branch to already include upcoming libraries
* a few new deps like pandas (for DaphneLIB)

Closes daphne-eu#530
  • Loading branch information
corepointer committed May 16, 2023
1 parent 5a0640c commit fff27a9
Show file tree
Hide file tree
Showing 8 changed files with 323 additions and 123 deletions.
9 changes: 7 additions & 2 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,19 @@ thirdparty/*

# Python
__pycache__/
/venv
/venv*

# Jetbrains IDE
.idea/
cmake-*/

tmpdaphne.daphne

# release scripts output
/artifacts

bin/
logs/
profiler/
precompiled-dependencies/
/cmake*/
/data
124 changes: 78 additions & 46 deletions containers/build-containers.sh
Original file line number Diff line number Diff line change
Expand Up @@ -14,15 +14,20 @@
# See the License for the specific language governing permissions and
# limitations under the License.

# Stop immediately if any command fails.
set -e

echo "This script is primarily meant for image maintainers. Users are advised to pull from https://hub.docker.com/u/daphneeu"
echo
echo "Comment/uncomment the docker command for the image(s) to be built"
echo "If images are wrongly rebuilt from cache, add --no-cache to the docker command"
echo "For verbose output add --progress=plain to the docker build command"
echo
sleep 4

GIT_REPO=daphne
GIT_BRANCH="docker-cuda"
GH_USER="corepointer"
GIT_BRANCH="main"
GH_USER="daphne-eu"
DAPHNE_REPO_URL="https://github.com/$GH_USER/$GIT_REPO.git"

export DOCKER_BUILDKIT=1
Expand All @@ -45,65 +50,92 @@ function build_daphne() {
}

#------------------------------------------------------------------------------
# Images for continuous integration (GitHub Actions)
# Images for building dependencies and continuous integration (GitHub Actions)
#------------------------------------------------------------------------------
#BASE_IMAGE=ubuntu:20.04
#FINAL_BASE_IMAGE=ubuntu:20.04
#DAPHNE_TARGET=github-action
#DAPHNE_TAG=$TIMESTAMP_DATE
#IMAGE_REPO=daphneeu/$DAPHNE_TARGET
#build_daphne -dev

#------------------------------------------------------------------------------
# Images for DAPHNE development
#------------------------------------------------------------------------------
## testing:
#BASE_IMAGE=ubuntu:20.04
#DAPHNE_TARGET=daphne-dev
#DAPHNE_TAG=$TIMESTAMP_DATE
#IMAGE_REPO=daphneeu/$DAPHNE_TARGET
#build_daphne -dev
#DAPHNE_TARGET=build-cmake
#IMAGE_REPO=local/build-cmake
##bulid deps stage
#build_daphne -deps
#exit

GIT_BRANCH="future-deps"
GH_USER="corepointer"
DAPHNE_REPO_URL="https://github.com/$GH_USER/$GIT_REPO.git"

DAPHNE_TARGET=daphne-deps
BASE_IMAGE=ubuntu:20.04
DAPHNE_TAG=$TIMESTAMP_DATE
IMAGE_REPO=daphneeu/$DAPHNE_TARGET
#bulid deps stage
build_daphne -deps

## build ci stage (based on deps stage)
DAPHNE_TARGET=github-action
IMAGE_REPO=daphneeu/$DAPHNE_TARGET
build_daphne -deps

## switch to main branch to build images from there (by default)
GIT_BRANCH="main"
GH_USER="daphne-eu"
DAPHNE_REPO_URL="https://github.com/$GH_USER/$GIT_REPO.git"

#------------------------------------------------------------------------------
# Images for running DAPHNE
# Images for DAPHNE development (BASE)
#------------------------------------------------------------------------------
#BASE_IMAGE=daphneeu/daphne-deps-compile
#FINAL_BASE_IMAGE=ubuntu:20.04
#DAPHNE_TARGET=daphne
#DAPHNE_TAG=ubuntu20.04_$TIMESTAMP_DATE
#IMAGE_REPO=daphneeu/$DAPHNE_TARGET
#DAPHNE_BUILD_FLAGS="--mpi"
#build_daphne
DAPHNE_TARGET=daphne-dev
BASE_IMAGE=ubuntu:20.04
DAPHNE_TAG=${TIMESTAMP_DATE}_BASE_ubuntu20.04
IMAGE_REPO=daphneeu/$DAPHNE_TARGET
build_daphne -dev
docker tag $IMAGE_REPO:$DAPHNE_TAG daphneeu/daphne-dev:latest_BASE

#------------------------------------------------------------------------------
# Images for DAPHNE development (CUDA)
#------------------------------------------------------------------------------
DAPHNE_TARGET=daphne-dev
CUDA_TAG=12.1.1-cudnn8-devel-ubuntu20.04
BASE_IMAGE=nvidia/cuda:$CUDA_TAG
DAPHNE_TAG=${TIMESTAMP_DATE}_CUDA_${CUDA_TAG}
IMAGE_REPO=daphneeu/$DAPHNE_TARGET
build_daphne -dev
docker tag $IMAGE_REPO:$DAPHNE_TAG daphneeu/daphne-dev:latest_CUDA

#-----------------------------------------------------------------------------
# Images for DAPHNE development (OneAPI)
#------------------------------------------------------------------------------
#DAPHNE_TARGET=daphne-dev
#CUDA_TAG=12.0.1-cudnn8-devel-ubuntu20.04
#DAPHNE_TAG=${CUDA_TAG}_$TIMESTAMP_DATE
#IMAGE_REPO=daphneeu/$DAPHNE_TARGET-cuda
#FINAL_BASE_IMAGE=nvidia/cuda:$CUDA_TAG
#BASE_IMAGE=ubuntu:20.04
#ONEAPI_TAG=2023.1.0-devel-ubuntu20.04
#BASE_IMAGE=intel/oneapi:$ONEAPI_TAG
#DAPHNE_TAG=${TIMESTAMP_DATE}_${ONEAPI_TAG}
#IMAGE_REPO=daphneeu/$DAPHNE_TARGET
#build_daphne -dev

#-----------------------------------------------------------------------------
# Images for running DAPHNE (CUDA)
#------------------------------------------------------------------------------
#DAPHNE_TARGET=daphne
#CUDA_TAG=12.0.1-cudnn8-runtime-ubuntu20.04
#DAPHNE_TAG=${CUDA_TAG}_$TIMESTAMP_DATE
#IMAGE_REPO=daphneeu/$DAPHNE_TARGET-cuda
#BASE_IMAGE=daphneeu/daphne-dev-cuda
#FINAL_BASE_IMAGE=nvidia/cuda:$CUDA_TAG
#DAPHNE_BUILD_FLAGS="--mpi --cuda"
#build_daphne
# Images for running DAPHNE
#------------------------------------------------------------------------------
DAPHNE_TARGET=daphne
BASE_IMAGE=daphneeu/daphne-deps
FINAL_BASE_IMAGE=ubuntu:20.04
DAPHNE_TAG=${TIMESTAMP_DATE}_BASE_ubuntu20.04
IMAGE_REPO=daphneeu/$DAPHNE_TARGET
DAPHNE_BUILD_FLAGS="--mpi"
build_daphne
docker tag $IMAGE_REPO:$DAPHNE_TAG daphneeu/daphne:latest_BASE

#-----------------------------------------------------------------------------
# Images for DAPHNE development (OneAPI)
# Images for running DAPHNE (CUDA)
#------------------------------------------------------------------------------
#DAPHNE_TARGET=daphne-dev-interactive
#ONEAPI_TAG=2023.0.0-devel-ubuntu22.04
#DAPHNE_TAG=${ONEAPI_TAG}_$TIMESTAMP_DATE
#IMAGE_REPO=daphneeu/$DAPHNE_TARGET-oneapi
#BASE_IMAGE=ubuntu:20.04
#FINAL_BASE_IMAGE=intel/oneapi:$ONEAPI_TAG
#build_daphne -dev
DAPHNE_TARGET=daphne
CUDA_TAG=12.1.1-cudnn8-runtime-ubuntu20.04
DAPHNE_TAG=${TIMESTAMP_DATE}_CUDA_${CUDA_TAG}
IMAGE_REPO=daphneeu/$DAPHNE_TARGET
BASE_IMAGE=daphneeu/daphne-dev
FINAL_BASE_IMAGE=nvidia/cuda:$CUDA_TAG
DAPHNE_BUILD_FLAGS="--mpi --cuda"
build_daphne
docker tag $IMAGE_REPO:$DAPHNE_TAG daphneeu/daphne:latest_CUDA
set +e
89 changes: 89 additions & 0 deletions containers/daphne-deps.Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
# syntax=docker/dockerfile:1

# Copyright 2023 The DAPHNE Consortium
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


# This Dockerfile provides a basic DAPHNE compilation environment with all
# third party dependencies precompiled (use ''./build.sh --no-deps --installPrefix /usr/local'' to compile DAPHNE)


# defaults:
ARG BASE_IMAGE=ubuntu:20.04
ARG CMAKE_VERSION=3.26.3
ARG DEBIAN_FRONTEND="noninteractive"
ARG DEBCONF_NOWARNINGS="yes"
ARG DAPHNE_DIR=/daphne
ARG DAPHNE_REPO=https://github.com/daphne-eu/daphne.git
ARG DAPHNE_BRANCH=main
ARG TIMESTAMP=0
ARG CREATION_DATE=0
ARG GIT_HASH=0

FROM ${BASE_IMAGE} as base
ARG DEBIAN_FRONTEND
ARG DEBCONF_NOWARNINGS
RUN apt-get -qq -y update && apt-get -y upgrade \
&& apt-get -y --no-install-recommends install \
ca-certificates file git openssh-client unzip wget tar \
libomp-dev libpfm4-dev libssl-dev libxml2-dev uuid-dev zlib1g-dev \
build-essential clang gfortran lld llvm ninja-build openjdk-11-jdk-headless pkg-config python3 \
&& apt-get clean && rm -rf /var/lib/apt/lists/*

FROM base as build-cmake
ARG NUM_CORES=4
ARG CMAKE_VERSION
ARG BUILD_DIR=/build-cmake
WORKDIR $BUILD_DIR
RUN wget -qO- https://github.com/Kitware/CMake/releases/download/v$CMAKE_VERSION/cmake-$CMAKE_VERSION.tar.gz | tar xzf \
- --strip-components=1
RUN ./bootstrap --parallel=$NUM_CORES --generator=Ninja --no-qt-gui --prefix=/usr/local
RUN ninja
RUN strip --strip-unneeded bin/*
RUN ninja install
WORKDIR /
RUN rm -rf ${BUILD_DIR}

FROM build-cmake as build
ARG DAPHNE_DIR=/daphne
ARG DAPHNE_REPO=https://github.com/daphne-eu/daphne.git
ARG DAPHNE_BRANCH=main
RUN git clone --depth=1 --single-branch --branch=$DAPHNE_BRANCH $DAPHNE_REPO $DAPHNE_DIR
WORKDIR $DAPHNE_DIR
RUN ./build.sh --no-fancy --no-submodule-update --installPrefix /usr/local
RUN find /usr/local -exec file {} \; | grep -e "not stripped" | cut -d ":" -f 1 | xargs strip --strip-unneeded
RUN rm -rf $DAPHNE_DIR
RUN ldconfig
WORKDIR /

FROM base as daphne-deps
ARG DAPHNE_REPO
ARG DAPHNE_BRANCH
ARG TIMESTAMP
ARG CREATION_DATE
ARG GIT_HASH
LABEL "org.opencontainers.image.source"="${DAPHNE_REPO}"
LABEL "org.opencontainers.image.base.name"="${BASE_IMAGE}"
LABEL "org.opencontainers.image.version"="branch_${DAPHNE_BRANCH}_from_${TIMESTAMP}"
LABEL "org.opencontainers.image.created"="${CREATION_DATE}"
LABEL "org.opencontainers.image.revision"="${GIT_HASH}"
COPY --from=build /usr/local/bin/ /usr/local/bin/
COPY --from=build /usr/local/include/ /usr/local/include/
COPY --from=build /usr/local/lib/ /usr/local/lib/
COPY --from=build /usr/local/share/ /usr/local/share/
RUN ldconfig

FROM daphneeu/daphne-deps as github-action
RUN apt-get -qq -y update && apt-get -y upgrade && apt-get -y --no-install-recommends install \
moreutils python3-numpy python3-pandas && apt-get clean && rm -rf /var/lib/apt/lists/*
Loading

0 comments on commit fff27a9

Please sign in to comment.