re-factor commands in Dockerfile to bash scripts (#54)

Author: haobibo

.travis.yml

@@ -0,0 +1,38 @@
+# Distributed under the terms of the Modified BSD License.
+
+# default value: Latest LTS version of Ubuntu (https://hub.docker.com/_/ubuntu)
+
+FROM ubuntu:latest
+
+LABEL maintainer="haobibo@gmail.com"
+
+USER root
+
+COPY work /opt/utils/
+
+ENV SHELL=/bin/bash \
+    DEBIAN_FRONTEND=noninteractive \
+    LC_ALL="" \
+    LC_CTYPE=C.UTF-8 \
+    LC_TYPE=en_US.UTF-8 \
+    LANG=en_US.UTF-8 \
+    LANGUAGE=en_US.UTF-8 \
+    HOME_DIR=/root
+
+SHELL ["/bin/bash", "-c"]
+
+# --> Install OS libraries and setup some configurations
+RUN cd /tmp \
+ && apt-get  -y update --fix-missing > /dev/null && apt-get -y -qq upgrade \
+ && apt-get -qq install -y --no-install-recommends \
+     apt-utils apt-transport-https ca-certificates gnupg2 dirmngr locales sudo lsb-release curl \
+ && localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 \
+ && echo "en_US.UTF-8 UTF-8"             >  /etc/locale.gen && locale-gen \
+ && echo "ALL ALL=(ALL) NOPASSWD:ALL"    >> /etc/sudoers \
+ && mv /root/.bashrc /etc/bash_profile \
+ && echo '[ $BASH ] && [ -f /etc/bash_profile ] && . /etc/bash_profile'	>> /etc/bash.bashrc \
+ && echo '[ $BASH ] && [ -f /root/.bashrc ]     && . /root/.bashrc'		>> /etc/bash.bashrc \
+ # Clean up and display components version information...
+ && source /opt/utils/script-utils.sh && install__clean
+
+WORKDIR $HOME_DIR

README.md

@@ -0,0 +1,63 @@
+
+# function to debug, resolve package names from a text file and display.
+install_echo()    { cat $1 | cut -d "%" -f 1 | sed '/^$/d' | xargs -r -n1 printf '%s\n' ; }
+
+# function to install apt-get packages from a text file which lists package names (add comments with % char)
+install_apt()     { apt-get -y update --fix-missing > /dev/null && apt-get install -yq --no-install-recommends `cat $1 | cut -d '%' -f 1` ; }
+
+# function to install conda packages from a text file which lists package names (add comments with % char)
+install_conda()   { cat $1 | cut -d "%" -f 1 | sed '/^$/d' | xargs -r -n1 conda install -yq ; }
+
+# function to install python packages with pip from a text file which lists package names (add comments with % char)
+install_pip()     { cat $1 | cut -d "%" -f 1 | sed '/^$/d' | xargs -r -n1 pip install -U --pre --use-feature=2020-resolver ; }
+
+# function to install R packages from a text file which lists package names (add comments with % char, use quiet=T to be less verbose)
+install_R()       { R -e "options(Ncpus=4);lapply(scan('$1','c',comment.char='%'),function(x){cat(x,system.time(install.packages(x,clean=T,quiet=T)),'\n')})"; }
+
+# function to install go packages with go from a text file which lists package names (add comments with % char)
+install_go()      { cat $1 | cut -d "%" -f 1 | sed '/^$/d' | xargs -r -n1 go get -u ; }
+
+# function to install julia packages from a text file which lists package names (add comments with % char)
+install_julia()   { julia -e "import Pkg; l=filter(x->length(x)>0, [split(i,r\"%| |\t\")[1] for i in readlines(\"$1\")]); Pkg.add(l)" ; }
+
+# function to install octave packages with go from a text file which lists package names (add comments with % char)
+install_octave()  { cat $1 | cut -d "%" -f 1 | sed '/^$/d' | xargs -r -n1 -I {} sh -c 'octave --eval "pkg install -forge {}"' ; }
+
+# function to download a ZIP file and unzip it to /opt/
+install_zip()     { curl -o /tmp/TMP.zip -sL $1 && unzip -q -d /opt/ /tmp/TMP.zip && rm /tmp/TMP.zip ; }
+
+# function to download a .tar.gz file and unzip it to /opt/, add a second argument to extract only those file
+install_tar_gz()  { curl -o /tmp/TMP.tgz -sL $1 && tar -C /opt/ -xzf /tmp/TMP.tgz $2 && rm /tmp/TMP.tgz ; }
+
+# function to download a .tar.xz file and unzip it to /opt/, add a second argument to extract only those file
+install_tar_xz()  { curl -o /tmp/TMP.txz -sL $1 && tar -C /opt/ -xJf /tmp/TMP.txz $2 && rm /tmp/TMP.txz ; }
+
+# function to install java packages from a text file which lists JAR file maven full names (add comments with % char)
+install_mvn() { cat $1 | cut -d "%" -f 1 | xargs -r -n1 -I {} mvn dependency:copy -DlocalRepositoryDirectory="/tmp/m2repo" -Djavax.net.ssl.trustStorePassword=changeit -Dartifact="{}" -DoutputDirectory="$2" ; }
+
+# function to clean up
+install__clean(){
+  which apt-get && apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/*
+  which conda   && conda clean -ya
+  which npm     && npm cache clean --force
+  rm -rf /opt/conda/share/jupyter/lab/staging
+  ( rm -rf /root/.* /tmp/.* /tmp/* /var/log/* /var/cache/* || true )
+  chmod ugo+rwXt /tmp
+  find /opt/conda/lib | grep -E "(__pycache__|\.pyc|\.pyo$)" | xargs rm -rf
+  echo "@ System environment variables:" && for e in $(echo $(printenv) | tr " " "\n") ; do echo $e ; done
+  echo "@ Version of image: building finished at:" `date` `uname -a`
+  true
+}
+
+# function to list installed packages
+list_installed_packages() {
+  type pip    && echo "@ Version of Python and packages:" && python --version && pip list
+  type conda  && echo "@ Version of Conda and packages:"  && conda info && conda list | grep -v "<pip>"
+  type node   && echo "@ Version of NodeJS and packages:" && node --version && npm --version && npm list -g --depth 0
+  type java   && echo "@ Version of Java (java/javac):"   && java -version && javac -version
+  type R      && echo "@ Version of R and libraries:"     && R --version && R -e "R.Version()\$version.string;installed.packages()[,c(3,10)]"
+  type julia  && echo "@ Version of Julia and packages"   && julia --version && julia -e "using Pkg; for(k,v) in Pkg.dependencies(); println(v.name,\"==\",v.version); end"
+  type go     && echo "@ Version of golang and packages:" && go version && go list ...
+  type octave && echo "@ Version of Octave and packages:" && octave --version && octave --eval "pkg list"
+  true
+}

docker_atom/Dockerfile

@@ -1,67 +1,31 @@
 # Distributed under the terms of the Modified BSD License.
 
-# default value: Latest LTS version of Ubuntu (https://hub.docker.com/_/ubuntu)
-ARG base=ubuntu:latest
-FROM ${base:-base}
+ARG BASE_NAMESPACE
+ARG BASE_IMG="atom"
+FROM ${BASE_NAMESPACE:+$BASE_NAMESPACE/}${BASE_IMG}
 
 LABEL maintainer="haobibo@gmail.com"
 
-USER root
-
-ENV SHELL=/bin/bash \
-    DEBIAN_FRONTEND=noninteractive \
-    LC_ALL=en_US.UTF-8 \
-    LC_TYPE=en_US.UTF-8 \
-    LC_CTYPE=C.UTF-8 \
-    LANG=en_US.UTF-8 \
-    LANGUAGE=en_US.UTF-8 \
-    PATH=/opt/conda/bin:$PATH \
-    HOME_DIR=/root
-
-SHELL ["/bin/bash", "-c"]
-
 COPY work /opt/utils/
 
-# --> Install OS libraries and setup some configurations
-RUN cd /tmp \
- && apt-get -y update --fix-missing > /dev/null && apt-get -y -qq upgrade \
- && apt-get -qq install -y --no-install-recommends \
-    apt-utils apt-transport-https ca-certificates lsb-release gnupg2 dirmngr wget jq locales sudo \
-    build-essential cmake unzip \
- && localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 \
- && echo "en_US.UTF-8 UTF-8"             >  /etc/locale.gen && locale-gen \
- && echo "ALL ALL=(ALL) NOPASSWD: ALL"   >> /etc/sudoers \
- && mv /root/.bashrc /etc/bash_profile \
- && echo '[ $BASH ] && [ -f /etc/bash_profile ] && . /etc/bash_profile'	>> /etc/bash.bashrc \
- && echo '[ $BASH ] && [ -f /root/.bashrc ]     && . /root/.bashrc'		  >> /etc/bash.bashrc \
- && cat /opt/utils/script-utils.sh       >> /etc/bash.bashrc \
- # Install libraries libs, utilities
- && source /opt/utils/script-utils.sh \
- && install_apt /opt/utils/install_list_base.apt \
- && chmod 777 /tmp
+RUN source      /opt/utils/script-utils.sh \
+ && install_apt /opt/utils/install_list_base.apt
 
-# --> Install Python3 (Miniconda3), replace conda packages with pip source.
-RUN cd /tmp/ \
- && wget -qO- "https://repo.continuum.io/miniconda/Miniconda3-py38_4.8.3-Linux-$(arch).sh" -O conda.sh && bash /tmp/conda.sh -f -b -p /opt/conda \
- && conda config --system --prepend channels conda-forge \
- && conda config --system --set auto_update_conda false  \
- && conda config --system --set show_channel_urls true   \
- && conda config --set channel_priority strict \
- && conda update --all --quiet --yes \
- # These conda pkgs shouldn't be removed (otherwise will cause RemoveError) since they are directly reqiuired by conda: pip setuptools pycosat pyopenssl requests ruamel_yaml python-libarchive-c
- && CONDA_PY_PKGS=`conda list | grep "py3" | cut -d " " -f 1 | sed "/#/d;/conda/d;/pip/d;/setuptools/d;/pycosat/d;/pyopenssl/d;/requests/d;/ruamel_yaml/d;/python-libarchive-c/d;"` \
- && conda remove --force -yq $CONDA_PY_PKGS \
- && pip install -UIq pip setuptools $CONDA_PY_PKGS \
- # Replace system Python3 with Conda's Python, and take care of `lsb_releaes`
- && rm /usr/bin/python3 && ln -s /opt/conda/bin/python /usr/bin/python3 \
- && mv /usr/share/pyshared/lsb_release.py /usr/bin/ \
- # Print Conda and Python packages information in the docker build log
- && echo "@ Version of Conda & Python:" && conda info && conda list | grep -v "<pip>"
-
-# --> Clean up and display components version information...
-RUN  source /opt/utils/script-utils.sh \
-  && install__clean && cd \
-  && echo "@ Version of image: building finished at:" `date` `uname -a` \
-  && echo "@ System environment variables:" `printenv`
-
-WORKDIR $HOME_DIR
+# Build and install tini, which will be entry point later...
+RUN cd /tmp \
+ && TINI_VERSION=$(curl -sL https://github.com/krallin/tini/releases.atom | grep 'releases/tag' | head -1 | grep -Po '\d[\d.]+' ) \
+ && curl -o tini.zip -sL "https://github.com/krallin/tini/archive/v${TINI_VERSION}.zip" && unzip -q /tmp/tini.zip \
+ && cmake /tmp/tini-* && make install && mv /tmp/tini /usr/bin/tini && chmod +x /usr/bin/tini
+
+ENV PATH=/opt/conda/bin:$PATH
+
+## --> Install Python3 (Miniconda3)
+RUN source /opt/utils/script-setup.sh && setup_conda && install__clean
+ # Replace system Python3 with Conda's Python - note that /bin and /sbin are symlinks of /usr/bin in docker image ubuntu
+ # && cp --verbose -rn /usr/lib/python3.8/* /opt/conda/lib/python3.8/ \
+ # && mv /usr/share/pyshared/lsb_release.py /usr/bin/ \
+ # && rm --verbose -rf $(/usr/bin/python3.8 -c 'import sys; print(" ".join(sys.path))') /usr/lib/python3* /usr/share/pyshared/ \
+ # && ln -s /opt/conda/lib/python3.8 /usr/lib/ \
+ # && TO_REPLACE="/usr/bin/python3;/usr/bin/python3.8" \
+ # && for F in $(echo ${TO_REPLACE} | tr ";" "\n") ; do ( rm -f ${F} && ln -s /opt/conda/bin/python ${F} ) ; done \
+ 

docker_atom/work/script-utils.sh

@@ -1,8 +1,11 @@
 % This file contains apt packages to be installed with apt-get install line by line.
 % Use percent char as line comment separator.
 
-openssh-client          % To generate keys
-htop vim git            % Process Management, Editor, Version Control
-curl rsync p7zip-full   % Utilites
-iputils-ping            % ping command
-python2.7               % required for some legacy packages like node-gyp
+htop vim git jq         % Process Management, Editor, Version Control
+build-essential cmake   % build essentials
+wget iputils-ping       % download, network (ping)
+unzip p7zip-full        % zip/unzip
+rsync                   % file transfer
+openssh-client          % Provide ssh client and generate keys
+openssh-server          % Provide ssh server (sshd)
+python2.7               % required by some legacy packages like node-gyp

docker_base/Dockerfile

@@ -0,0 +1,158 @@
+source /opt/utils/script-utils.sh
+
+
+setup_conda() {
+       wget -qO- "https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-$(arch).sh" -O /tmp/conda.sh \
+    && bash /tmp/conda.sh -f -b -p /opt/conda \
+    && conda config --system --prepend channels conda-forge \
+    && conda config --system --set auto_update_conda false  \
+    && conda config --system --set show_channel_urls true   \
+    && conda config --set channel_priority strict \
+    && conda update --all --quiet --yes
+
+    # These conda pkgs shouldn't be removed (otherwise will cause RemoveError) since they are directly reqiuired by conda: pip setuptools pycosat pyopenssl requests ruamel_yaml
+    CONDA_PY_PKGS=`conda list | grep "py3" | cut -d " " -f 1 | sed "/#/d;/conda/d;/pip/d;/setuptools/d;/pycosat/d;/pyopenssl/d;/requests/d;/ruamel_yaml/d;"` \
+    && conda remove --force -yq $CONDA_PY_PKGS \
+    && pip install -UIq pip setuptools $CONDA_PY_PKGS
+
+    # Print Conda and Python packages information in the docker build log
+    echo "@ Version of Conda & Python:" && conda info && conda list | grep -v "<pip>"
+}
+
+
+setup_java_base() {
+   #   VERSION_OPENJDK=16 && VERSION_OPENJDK_EA=8 \
+   #   && URL_OPENJDK="https://download.java.net/java/early_access/jdk${VERSION_OPENJDK}/${VERSION_OPENJDK_EA}/GPL/openjdk-${VERSION_OPENJDK}-ea+${VERSION_OPENJDK_EA}_linux-x64_bin.tar.gz" \
+      URL_OPENJDK="https://download.java.net/java/GA/jdk14.0.2/205943a0976c4ed48cb16f1043c5c647/12/GPL/openjdk-14.0.2_linux-x64_bin.tar.gz" \
+   && install_tar_gz ${URL_OPENJDK} && mv /opt/jdk-* /opt/jdk \
+   && ln -s /opt/jdk/bin/* /usr/bin/ \
+   && echo "@ Version of Java (java/javac):" && java -version && javac -version
+}
+
+setup_java_maven() {
+      MAVEN_VERSION="3.6.3" \
+   && install_zip "http://archive.apache.org/dist/maven/maven-3/${MAVEN_VERSION}/binaries/apache-maven-${MAVEN_VERSION}-bin.zip" \
+   && mv /opt/apache-maven-${MAVEN_VERSION} /opt/maven \
+   && ln -s /opt/maven/bin/mvn* /usr/bin/ \
+   && echo "@ Version of Maven:" && mvn --version
+}
+
+
+setup_node() {
+    # NODEJS_VERSION_MAJOR="v14" && grep "v${NODEJS_VERSION_MAJOR}."
+       ARCH="x64" \
+    && NODEJS_VERSION=$(curl -sL https://github.com/nodejs/node/releases.atom | grep 'releases/tag' | head -1 | grep -Po '\d[.\d]+') \
+    && NODEJS_VERSION_MAJOR=$(echo ${NODEJS_VERSION} | cut -d '.' -f1 ) \
+    && install_tar_gz "https://nodejs.org/download/release/latest-v${NODEJS_VERSION_MAJOR}.x/node-v${NODEJS_VERSION}-linux-${ARCH}.tar.gz" \
+    && mv /opt/node* /opt/node \
+    && echo  "PATH=/opt/node/bin:$PATH" >> /etc/bash.bashrc \
+    && export PATH=/opt/node/bin:$PATH \
+    && npm install -g npm yarn \
+    && ln -s /opt/node/bin/* /usr/bin/ \
+    && echo "@ Version of Node, npm, and yarn:" `node -v` `npm -v` `yarn -v`
+}
+
+
+setup_R_base() {
+       apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 \
+    && echo "deb https://cloud.r-project.org/bin/linux/ubuntu focal-cran40/" > /etc/apt/sources.list.d/cran.list \
+    && install_apt  /opt/utils/install_list_R_base.apt \
+    && echo "options(repos=structure(c(CRAN=\"https://cloud.r-project.org\")))" >> /etc/R/Rprofile.site \
+    && R -e "install.packages(c('devtools'),clean=T,quiet=T);" \
+    && ( type java && type R && R CMD javareconf || true ) \
+    && echo "@ Version of R:" && R --version 
+}
+
+
+setup_R_rstudio() {
+       RSTUDIO_VERSION=`curl -sL https://dailies.rstudio.com/rstudioserver/oss/ubuntu/x86_64/ | grep -Po "(?<=rstudio-server-)[0-9]\.[0-9]\.[0-9]+" | sort | tail -n 1` \
+    && wget -qO- "https://s3.amazonaws.com/rstudio-ide-build/server/bionic/amd64/rstudio-server-${RSTUDIO_VERSION}-amd64.deb" -O /tmp/rstudio.deb \
+    && dpkg -x /tmp/rstudio.deb /tmp && mv /tmp/usr/lib/rstudio-server/ /opt/ \
+    && ln -s /opt/rstudio-server         /usr/lib/ \
+    && ln -s /opt/rstudio-server/bin/rs* /usr/bin/
+    
+    # Allow RStudio server run as root user
+    # Configuration to make RStudio server disable authentication and do not run as daemon
+       mkdir -p /etc/rstudio \
+    && echo "server-daemonize=0"     >> /etc/rstudio/rserver.conf \
+    && echo "server-user=root"       >> /etc/rstudio/rserver.conf \
+    && echo "auth-none=1"            >> /etc/rstudio/rserver.conf \
+    && echo "auth-minimum-user-id=0" >> /etc/rstudio/rserver.conf \
+    && echo "auth-validate-users=0"  >> /etc/rstudio/rserver.conf \
+    && printf "#!/bin/bash\nexport USER=root\nrserver --www-port=8888" > /usr/local/bin/start-rstudio.sh \
+    && chmod u+x /usr/local/bin/start-rstudio.sh
+
+    # Remove RStudio's pandoc and pandoc-proc to reduce size if they are already installed in the jpy-latex step.
+       ( which pandoc          && rm /opt/rstudio-server/bin/pandoc/pandoc          || true ) \
+    && ( which pandoc-citeproc && rm /opt/rstudio-server/bin/pandoc/pandoc-citeproc || true ) \
+    && echo "@ Version of rstudio-server:" && rstudio-server version
+}
+
+
+setup_R_rshiny() {
+       RSHINY_VERSION=$(curl -sL https://s3.amazonaws.com/rstudio-shiny-server-os-build/ubuntu-14.04/x86_64/VERSION) \
+    && wget -qO- "https://download3.rstudio.org/ubuntu-14.04/x86_64/shiny-server-${RSHINY_VERSION}-amd64.deb" -O /tmp/rshiny.deb \
+    && dpkg -i /tmp/rshiny.deb \
+    && sed  -i "s/run_as shiny;/run_as root;/g"  /etc/shiny-server/shiny-server.conf \
+    && sed  -i "s/3838/8888/g"                   /etc/shiny-server/shiny-server.conf \
+    && printf "#!/bin/bash\nexport USER=root\nshiny-server" > /usr/local/bin/start-shiny-server.sh \
+    && chmod u+x /usr/local/bin/start-shiny-server.sh
+    
+    # Remove shiny's pandoc and pandoc-proc to reduce size if they are already installed in the jpy-latex step.
+       ( which pandoc          && rm /opt/shiny-server/ext/pandoc/pandoc          || true ) \
+    && ( which pandoc-citeproc && rm /opt/shiny-server/ext/pandoc/pandoc-citeproc || true ) \
+    && rm    /opt/shiny-server/ext/node/bin/shiny-server \
+    && ln -s /opt/shiny-server/ext/node/bin/node /opt/shiny-server/ext/node/bin/shiny-server
+    
+    # hack shiny-server to allow run in root user: https://github.com/rstudio/shiny-server/pull/391
+       sed  -i "s/throw new Error/logger.warn/g"  /opt/shiny-server/lib/worker/app-worker.js \
+    && echo "@ Version of shiny-server:" && shiny-server --version
+}
+
+
+setup_R_datascience() {
+   # firstly install rgl stub to work around, which has too many deps, but required by some libs
+   R -e "devtools::install_git(\"git://github.com/sorhawell/rgl.git\",quiet=T,clean=T)"
+
+      install_apt  /opt/utils/install_list_R_datascience.apt \
+   && install_R    /opt/utils/install_list_R_datascience.R
+}
+
+
+setup_GO() {
+       GO_VERSION=$(curl -sL https://github.com/golang/go/releases.atom | grep 'releases/tag' | head -1 | grep -Po '\d[\d.]+') \
+    && GO_URL="https://dl.google.com/go/go$GO_VERSION.linux-$(dpkg --print-architecture).tar.gz" \
+    && install_tar_gz $GO_URL go \
+    && ln -s /opt/go/bin/go /usr/bin/ \
+    && echo "@ Version of golang and packages:" && go version 
+}
+
+
+setup_julia() {
+       JULIA_URL="https://julialangnightlies-s3.julialang.org/bin/linux/x64/julia-latest-linux64.tar.gz" \
+    && install_tar_gz $JULIA_URL \
+    && mv /opt/julia-* /opt/julia \
+    && ln -fs /opt/julia/bin/julia /usr/bin/julia \
+    && mkdir -p /opt/julia/pkg \
+    && echo "import Libdl; push!(Libdl.DL_LOAD_PATH, \"/opt/conda/lib\")" >> /opt/julia/etc/julia/startup.jl \
+    && echo "DEPOT_PATH[1]=\"/opt/julia/pkg\""                            >> /opt/julia/etc/julia/startup.jl \
+    && echo "@ Version of Julia" && julia --version
+}
+
+
+setup_octave() {
+    # TEMPFIX: javac version
+    # && OCTAVE_VERSION="5.2.0" \
+    # && install_tar_xz "https://ftp.gnu.org/gnu/octave/octave-${OCTAVE_VERSION}.tar.xz" \
+    # && cd /opt/octave-* \
+    # && sed  -i "s/1.6/11/g" ./Makefile.in \
+    # && sed  -i "s/1.6/11/g" ./scripts/java/module.mk \
+    # && ./configure --prefix=/opt/octave --disable-docs --without-opengl \
+    # && make -j8 && make install -j8 \
+    # && cd /opt/utils && rm -rf /opt/octave-*
+
+       install_apt       /opt/utils/install_list_octave.apt \
+    && install_octave    /opt/utils/install_list_octave.pkg \
+    && echo "@ Version of Octave and installed packages:" \
+    && /opt/octave/bin/octave --version
+}