From 78491dc749fbb304ea80010fbe011b373a2b5638 Mon Sep 17 00:00:00 2001 From: Jared Lewis Date: Wed, 29 May 2024 22:48:51 +0200 Subject: [PATCH] chore: Use conda for libraries --- .github/workflows/build.yaml | 4 +- Dockerfile | 33 +++++-- environment.yml | 11 +++ scripts/build_wrf.sh | 16 ++-- scripts/install_deps.sh | 154 --------------------------------- scripts/install_system_deps.sh | 3 - scripts/setup_logging.sh | 15 ---- 7 files changed, 50 insertions(+), 186 deletions(-) create mode 100644 environment.yml delete mode 100755 scripts/install_deps.sh delete mode 100644 scripts/install_system_deps.sh delete mode 100644 scripts/setup_logging.sh diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index b79fcca..f2c86d9 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -54,7 +54,7 @@ jobs: type=schedule type=ref,event=branch type=ref,event=pr - type=raw,value=${{ env.WRF_VERSION }} + type=raw,value=${{ env.WRF_VERSION }},enable=${{ github.ref == format('refs/heads/{0}', 'main') }} - name: Build and push image uses: docker/build-push-action@v5 id: build @@ -66,3 +66,5 @@ jobs: labels: ${{ steps.meta.outputs.labels }} tags: ${{ steps.meta.outputs.tags }} push: true + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/Dockerfile b/Dockerfile index 454ac6f..38acd0b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:20.04 as base +FROM continuumio/miniconda3 as build MAINTAINER Jared Lewis @@ -6,21 +6,38 @@ ARG DEBIAN_FRONTEND=noninteractive ENV TZ=Etc/UTC RUN apt-get update && \ - apt-get install -y sudo curl build-essential gfortran m4 csh git jq wget aria2 imagemagick libmpich-dev file libnetcdff-dev && \ + apt-get install -y curl m4 csh jq file && \ rm -rf /var/lib/apt/lists/* -FROM base as build +COPY environment.yml /opt/environment.yml +RUN conda env create -f /opt/environment.yml -ARG TARGETPLATFORM -ARG WRF_VERSION=4.5.1 -ARG WPS_VERSION=4.5 +# Install conda-pack: +RUN conda install -c conda-forge conda-pack +# Use conda-pack to create a standalone enviornment +# in /venv: +RUN conda-pack -n wrf -o /tmp/env.tar && \ + mkdir /opt/venv && cd /opt/venv && tar xf /tmp/env.tar && \ + rm /tmp/env.tar + +# We've put venv in same path it'll be in final image, +# so now fix up paths: +RUN /opt/venv/bin/conda-unpack COPY scripts /opt/wrf/build/scripts/ +RUN PLATFORM=${TARGETPLATFORM} WRF_VERSION=${WRF_VERSION} WPS_VERSION=${WPS_VERSION} bash /opt/wrf/build/scripts/build_wrf.sh + + +FROM debian:bookworm AS runtime + +ARG TARGETPLATFORM +ARG WRF_VERSION=4.5.1 +ARG WPS_VERSION=4.5 WORKDIR /opt/wrf +COPY --from=build /opt/venv /opt/venv +COPY --from=build /opt/wrf /opt/wrf -RUN PLATFORM=${TARGETPLATFORM} bash /opt/wrf/build/scripts/install_deps.sh -RUN PLATFORM=${TARGETPLATFORM} WRF_VERSION=${WRF_VERSION} WPS_VERSION=${WPS_VERSION} bash /opt/wrf/build/scripts/build_wrf.sh ENTRYPOINT ["/bin/bash"] \ No newline at end of file diff --git a/environment.yml b/environment.yml new file mode 100644 index 0000000..8932f80 --- /dev/null +++ b/environment.yml @@ -0,0 +1,11 @@ +name: wrf +channels: + - conda-forge + - defaults +dependencies: + - hdf5=1.14.3 + - jasper=4.2.4 + - mpich=4.2.1 + - netcdf-fortran=4.6.1 + - wget + - gfortran=10.4 diff --git a/scripts/build_wrf.sh b/scripts/build_wrf.sh index 3712311..a9a62c2 100755 --- a/scripts/build_wrf.sh +++ b/scripts/build_wrf.sh @@ -1,13 +1,12 @@ #!/usr/bin/env bash # Builds WRF and WPS using the prebuilt dependencies -set -x -set -e +set -xe # Setup WRF_VERSION="${WRF_VERSION:-4.5.1}" WPS_VERSION="${WPS_VERSION:-4.5}" -DIR=/opt/wrf/libs +DIR=/opt/venv # Link to the compiled dependencies export PATH=$DIR/bin:$PATH @@ -21,10 +20,11 @@ export F77=gfortran export FFLAGS="-m64 -fallow-argument-mismatch" export NETCDF=$(nc-config --prefix) export NETCDF4=1 -#export HDF5=$DIR +export HDF5=$DIR export JASPERLIB=$DIR/lib export JASPERINC=$DIR/include export J="-j 8" +export ARCH=$(uname -m) pushd $DIR/.. @@ -33,7 +33,13 @@ if [ ! -f WRF-${WRF_VERSION}/run/real.exe ]; then wget -nv https://github.com/wrf-model/WRF/releases/download/v${WRF_VERSION}/v${WRF_VERSION}.tar.gz -O WRF-v${WRF_VERSION}.tar.gz tar -xzvf WRF-v${WRF_VERSION}.tar.gz pushd WRFV${WRF_VERSION} || exit - echo "34\n1\n" | ./configure + + if (( $ARCH == "aarch64" )); then + echo "12\n1\n" | ./configure + else + echo "34\n1\n" | ./configure + fi + ./compile em_real popd ln -s WRFV${WRF_VERSION} WRF diff --git a/scripts/install_deps.sh b/scripts/install_deps.sh deleted file mode 100755 index 226c9b1..0000000 --- a/scripts/install_deps.sh +++ /dev/null @@ -1,154 +0,0 @@ -#!/usr/bin/env bash - -set -x -set -e - -# Set up the required ENV variables -# The current configuration uses GNU compilers -export DIR=/opt/wrf/libs -export CC=gcc -export CXX=g++ -export FC=gfortran -export FCFLAGS="-m64" -export F77=gfortran -export FFLAGS="-m64" -export NC_VERSION=4.9.2 -export NC_FORTRAN_VERSION=4.6.1 -MPICH_VERSION=4.0 -ZLIB_VERSION=1.3.1 -LIBPNG_VERSION=1.6.34 -LIBCURL_VERSION=7.65.3 -JASPER_VERSION=1.900.1 -HDF5_VERSION=1.10.11 -NUM_CORES=8 - - -# Install pip and wrfconf -#wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -#bash Miniconda2-latest-Linux-x86_64.sh -b -p $HOME/miniconda -#rm Miniconda2-latest-Linux-x86_64.sh -#export PATH="$HOME/miniconda/bin:$PATH" -#pip install wrfconf -#pip install git+http://github.com/lewisjared/augurycli.git#egg=augurycli -#conda install -c conda-forge -y wrf-python matplotlib netCDF4 basemap basemap-data-hires -#conda clean --all -y - -# Ensure the plotting works without a display -#mkdir -p ~/.config/matplotlib -#echo "backend : Agg" > ~/.config/matplotlib/matplotlibrc - -# Install yq for parsing yaml files -if [ ! -f /usr/bin/yq ]; then - wget -nv https://github.com/mikefarah/yq/releases/download/2.4.0/yq_linux_amd64 - chmod +x yq_linux_amd64 - sudo mv yq_linux_amd64 /usr/bin/yq -fi - -#sudo chown root:ubuntu /opt && sudo chmod g+w /opt -sudo mkdir -p $DIR/src -#sudo chown ubuntu:ubuntu -R /opt/wrf -pushd $DIR/src - -export LDFLAGS=-L$DIR/lib -export CPPFLAGS=-I$DIR/include -export PATH=$DIR/bin:$PATH -export NETCDF=$DIR - - -#Install libcurl -if [ ! -f $DIR/lib/libcurl.a ]; then - wget -nv https://curl.haxx.se/download/curl-$LIBCURL_VERSION.tar.gz - tar xzvf curl-$LIBCURL_VERSION.tar.gz - pushd curl-$LIBCURL_VERSION - ./configure --prefix=$DIR - make -j $NUM_CORES - make install - popd -fi - -#Install zlib -if [ ! -f $DIR/lib/libz.a ]; then - wget -nv -N http://www.zlib.net/zlib-$ZLIB_VERSION.tar.gz - tar xzvf zlib-$ZLIB_VERSION.tar.gz - pushd zlib-$ZLIB_VERSION - ./configure --prefix=$DIR - make -j $NUM_CORES - make install - popd -fi - -# Install libpng -if [ ! -f $DIR/lib/libpng.a ]; then - wget -nv -N ftp://ftp-osl.osuosl.org/pub/libpng/src/libpng16/libpng-$LIBPNG_VERSION.tar.gz - tar xzvf libpng-$LIBPNG_VERSION.tar.gz - pushd libpng-$LIBPNG_VERSION - ./configure --prefix=$DIR - make -j $NUM_CORES - make install - popd -fi - -# Install Jasper -if [ ! -f $DIR/lib/libjasper.a ]; then - wget -nv -N http://www2.mmm.ucar.edu/wrf/OnLineTutorial/compile_tutorial/tar_files/jasper-$JASPER_VERSION.tar.gz - tar xzvf jasper-$JASPER_VERSION.tar.gz #or just .tar if no .gz present - pushd jasper-$JASPER_VERSION - - if [ "$PLATFORM" = "linux/arm64" ]; then - EXTRA_ARGS="--build=aarch64-unknown-linux-gnu" - else - EXTRA_ARGS="" - fi - ./configure --prefix=$DIR $EXTRA_ARGS - make -j $NUM_CORES - make install - popd -fi - -## Install MPICH -# Ignoring compiling MPICH for now as we are only targetting a single node -#if [ ! -f $DIR/lib/libmpi.a ]; then -# wget -nv -N http://www.mpich.org/static/downloads/$MPICH_VERSION/mpich-$MPICH_VERSION.tar.gz -# tar xzvf mpich-$MPICH_VERSION.tar.gz -# pushd mpich-$MPICH_VERSION -# ./configure --prefix=$DIR --build=aarch64-unknown-linux-gnu -# make -j $NUM_CORES -# make install -# popd -#fi - -##Install libhdf5 -#if [ ! -f $DIR/lib/libhdf5.a ]; then -# wget -nv -N https://github.com/HDFGroup/hdf5/releases/download/hdf5-1_10_11/hdf5-1_10_11.tar.gz -# tar xzvf hdf5-1_10_11.tar.gz -# pushd hdfsrc -# ./configure --prefix=$DIR --enable-fortran -# make -j $NUM_CORES -# make install -# popd -#fi -# -## Download and install netCDF -#if [ ! -f $DIR/lib/libnetcdf.a ]; then -# wget -nv -N https://github.com/Unidata/netcdf-c/archive/v$NC_VERSION.tar.gz -O netcdf-c-$NC_VERSION.tar.gz -# tar xzvf netcdf-c-$NC_VERSION.tar.gz -# pushd netcdf-c-$NC_VERSION -# ./configure --prefix=$DIR --disable-dap --enable-netcdf4 --disable-libxml2 -# make -j $NUM_CORES -# make install -# popd -#fi - -## Install netcdf-fortran -#if [ ! -f $DIR/lib/libnetcdff.a ]; then -# wget -nv -N https://github.com/Unidata/netcdf-fortran/archive/v$NC_FORTRAN_VERSION.tar.gz -O netcdf-fortran-$NC_FORTRAN_VERSION.tar.gz -# tar xzvf netcdf-fortran-$NC_FORTRAN_VERSION.tar.gz -# pushd netcdf-fortran-$NC_FORTRAN_VERSION -# ./configure --prefix=$DIR -# make -j $NUM_CORES -# make install -# popd -#fi - - -rm ./*.tar.gz diff --git a/scripts/install_system_deps.sh b/scripts/install_system_deps.sh deleted file mode 100644 index 6c9a37f..0000000 --- a/scripts/install_system_deps.sh +++ /dev/null @@ -1,3 +0,0 @@ -apt-get update -apt-get install -y sudo curl build-essential libstdc++-12-dev gfortran-12 m4 csh git jq wget aria2 imagemagick libmpich-dev -rm -rf /var/lib/apt/lists/* \ No newline at end of file diff --git a/scripts/setup_logging.sh b/scripts/setup_logging.sh deleted file mode 100644 index 8cf0f27..0000000 --- a/scripts/setup_logging.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash - -ES_VERSION=6.2.3 - -# Install filebeat -curl -L -O https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-${ES_VERSION}-amd64.deb -sudo dpkg -i filebeat-${ES_VERSION}-amd64.deb -rm filebeat-${ES_VERSION}-amd64.deb - -# Install metricbeat -curl -L -O https://artifacts.elastic.co/downloads/beats/metricbeat/metricbeat-${ES_VERSION}-amd64.deb -sudo dpkg -i metricbeat-${ES_VERSION}-amd64.deb -rm metricbeat-${ES_VERSION}-amd64.deb - -echo "Logging successfully setup" \ No newline at end of file