From 2ca158210fe8e9c876e068a3067e5a60ea181e70 Mon Sep 17 00:00:00 2001
From: Omer Yair <omeryair@gmail.com>
Date: Sun, 14 Jul 2019 22:08:15 +0300
Subject: [PATCH 01/16] Improve control over the container's user

---
 README.md               |   3 +-
 mldock.sh               |  78 ++++++++++++++++-----------
 resources/entrypoint.sh | 117 ++++++++++++++++++++--------------------
 3 files changed, 109 insertions(+), 89 deletions(-)

diff --git a/README.md b/README.md
index ec43eb7..cdd8e16 100644
--- a/README.md
+++ b/README.md
@@ -151,5 +151,4 @@ For convenient the docker image comes with a few preconfigured scripts (which ar
 - *default-jupyterlab*: Opens Jupyter Lab at the root directory using port 9901 (without a password nor token)
 - *run_server*: Starts Jupyter Notebook and Lab using a tmux session.
 
-
-The Enjoy ;)
+Enjoy ;)
diff --git a/mldock.sh b/mldock.sh
index 1094a52..995a190 100755
--- a/mldock.sh
+++ b/mldock.sh
@@ -7,15 +7,15 @@ app_name=mldock
 
 repository="omeryair/"
 image_name="mldock"
-version_name="v0.3"
+version_name="v0.4"
 
-container_name="mldock"
+container_name="mldock_$USER"
 
 main_cli() {
     ## Parse args
     ## ==========
     usage() {
-        echo "A CLI tool for working with the mldock docker"
+        echo "A CLI tool for working with the $app_name docker"
         echo ""
         echo "usage: $app_name  <command> [<options>]"
         echo "   or: $app_name -h         to print this help message."
@@ -182,12 +182,16 @@ run_cli() {
         ## No display
         connect_to_x_server=false
     fi
-    userstring="$(whoami):$(id -u):$(id -gn):$(id -g)"
+    if [[ "$(whoami)" == "root" ]]; then
+        userstring="dockuser:4283:dockuser:4283"
+    else
+        userstring="$(whoami):$(id -u):$(id -gn):$(id -g)"
+    fi
     map_host=true
     detach_container=false
     home_folder=""
     command_to_run=""
-    extra_args=""
+    extra_args=()
     if nvidia-smi >& /dev/null ; then 
         use_nvidia_runtime=true
     else
@@ -204,6 +208,7 @@ run_cli() {
         echo "    -f home_folder          A folder to map as the dockuser's home folder."
         echo "    -s                      Run the command as root."
         echo "    -u                      Run the command as dockuser user."
+        echo "    -i username             Run the command as *username*."
         echo "    -x                      Don't connect X-server"
         echo "    -r                      Don't map the root folder on the host machine to /host inside the container."
         echo "    -d                      Detach the container."
@@ -215,7 +220,7 @@ run_cli() {
         exit 0
     fi
 
-    while getopts "v:c:f:suxrde:" opt; do
+    while getopts "v:c:f:sui:xrde:" opt; do
         case $opt in
             v)
                 version_name=$OPTARG
@@ -232,6 +237,10 @@ run_cli() {
             u)
                 userstring="dockuser:4283:dockuser:4283"
                 ;;
+            i)
+                username=$OPTARG
+                userstring="$username:$(id -u $username):$(id -gn $username):$(id -g $username)"
+                ;;
             x)
                 connect_to_x_server=false
                 ;;
@@ -242,7 +251,8 @@ run_cli() {
                 detach_container=true
                 ;;
             e)
-                extra_args=$OPTARG
+                IFS=$'\n' extra_args=( $(xargs -n1 printf "%s\n" <<< "$OPTARG") )
+                unset IFS
                 ;;
             :)
                 echo "Error: -$OPTARG requires an argument" 1>&2
@@ -258,6 +268,8 @@ run_cli() {
     done
     shift $((OPTIND -1))
 
+    userstring="${userstring// /-}"
+
     if [ "$#" -gt 0 ]; then
         command_to_run=( "$@" )
     fi
@@ -267,7 +279,7 @@ run_cli() {
 }
 
 exec_cli() {
-    extra_args=""
+    extra_args=()
     command_to_run="bash"
     usage () {
         echo "Execute a command inside an existing container"
@@ -290,7 +302,8 @@ exec_cli() {
                 container_name=$OPTARG
                 ;;
             e)
-                extra_args=$OPTARG
+                IFS=$'\n' extra_args=( $(xargs -n1 printf "%s\n" <<< "$OPTARG") )
+                unset IFS
                 ;;
             :)
                 echo "Error: -$OPTARG requires an argument" 1>&2
@@ -367,7 +380,7 @@ check_docker_for_sudo() {
             exit
         fi
     else
-        docker_sudo_prefix="sudo "
+        docker_sudo_prefix=""
     fi
 }
 
@@ -399,49 +412,54 @@ run_command() {
 
     if [ "$connect_to_x_server" = true ]; then
         xhost +local:root > /dev/null
-        extra_args="$extra_args -e DISPLAY=${DISPLAY} -e MPLBACKEND=Qt5Agg -e QT_X11_NO_MITSHM=1 -v /tmp/.X11-unix:/tmp/.X11-unix"
+        extra_args+=("-e" "DISPLAY=${DISPLAY}" "-e" "MPLBACKEND=Qt5Agg" "-e" "QT_X11_NO_MITSHM=1" "-v" "/tmp/.X11-unix:/tmp/.X11-unix")
     fi
 
     if [ "$map_host" = true ]; then
-        extra_args="$extra_args -v /:/host/"
+        extra_args+=("-v" "/:/host/")
     fi
 
     if [[ ! -z $userstring ]]; then
         userstringsplit=(${userstring//:/ })
         new_username=${userstringsplit[0]}
 
-        extra_args="$extra_args -e \"USERSTRING=$userstring\" --label new_username=$new_username"
+        extra_args+=("-e" "USERSTRING=$userstring" "--label" "new_username=$new_username")
 
         if [[ ! -z $home_folder ]]; then
-            extra_args="$extra_args -v $(readlink -f $home_folder):/home/$new_username/"
+            if [[ "$new_username" == "root" ]]; then
+                extra_args+=("-v" "$(readlink -f $home_folder):/root/")
+            else
+                extra_args+=("-v" "$(readlink -f $home_folder):/home/$new_username/")
+            fi
         fi
     fi
 
     if [ "$detach_container" = true ]; then
-        extra_args="$extra_args -d"
+        extra_args+=("-d")
     else
-        extra_args="$extra_args -it"
+        extra_args+=("-it")
     fi
 
     if [ "$use_nvidia_runtime" = true ]; then
-        extra_args="$extra_args --runtime=nvidia"
+        extra_args+=("--runtime=nvidia")
     fi
 
-    if [[ ! -z "$command_to_run" ]]; then
-        eval "${docker_sudo_prefix}docker run" \
-            "--rm" \
-            "--network host" \
-            "--name $container_name" \
-            "$extra_args" \
-            "$repository$image_name:$version_name \"\${command_to_run[@]}\""
+    if [ ! -z "$docker_sudo_prefix" ]; then
+        cmd=("sudo")
     else
-        eval "${docker_sudo_prefix}docker run" \
-            "--rm" \
-            "--network host" \
-            "--name $container_name" \
-            "$extra_args" \
-            "$repository$image_name:$version_name"
+        cmd=()
+    fi
+    cmd+=("docker" "run" \
+         "--rm" \
+         "--network" "host" \
+         "--name" "$container_name")
+    cmd+=( "${extra_args[@]}" )
+    if [[ ! -z "$command_to_run" ]]; then
+        cmd+=("$repository$image_name:$version_name" "${command_to_run[@]}")
     fi
+    echo "Running: ${cmd[@]}"
+    echo ""
+    "${cmd[@]}"
 }
 
 exec_command() {
diff --git a/resources/entrypoint.sh b/resources/entrypoint.sh
index 7b775fd..47f1db8 100755
--- a/resources/entrypoint.sh
+++ b/resources/entrypoint.sh
@@ -3,79 +3,82 @@ set -e
 
 trap 'shutdown' TERM INT  ## This line makes shutting down the container faster
 
-if [ ! -z $USERSTRING ]; then
-    USERSTRINGSPLIT=(${USERSTRING//:/ })
+if [ ! -z "$USERSTRING" ]; then
+    IFS=':' read -ra USERSTRINGSPLIT <<< "$USERSTRING"
 
     new_username="${USERSTRINGSPLIT[0]}"
     new_uid=${USERSTRINGSPLIT[1]}
     new_groupname="${USERSTRINGSPLIT[2]}"
     new_gid=${USERSTRINGSPLIT[3]}
-    if ! id $new_uid > /dev/null 2>&1; then
-        ## User does not exist. Create user
-        if [[ -z $(getent group $new_gid) ]]; then
-            ## Group does not exist. Create group
-            groupadd -g $new_gid "$new_groupname"
-        else
-            ## Group exist. Check that the group's name matches
-            if [[ ! $(getent group $new_gid | awk -F ":" '{ print $1 }') == "$new_groupname" ]]; then
-                echo "!! Error: A group with the gid of \"$new_gid\" but with a different group name already exist: \"$(getent group $new_gid | awk -F ":" '{ print $1 }')\""
-                exit 1
+
+    if [[ ! $new_username == "root" ]]; then
+        if ! id $new_uid > /dev/null 2>&1; then
+            ## User does not exist. Create user
+            if [[ -z $(getent group $new_gid) ]]; then
+                ## Group does not exist. Create group
+                groupadd -g $new_gid "$new_groupname"
+            else
+                ## Group exist. Check that the group's name matches
+                if [[ ! $(getent group $new_gid | awk -F ":" '{ print $1 }') == "$new_groupname" ]]; then
+                    echo "!! Error: A group with the gid of \"$new_gid\" but with a different group name already exist: \"$(getent group $new_gid | awk -F ":" '{ print $1 }')\""
+                    exit 1
+                fi
             fi
-        fi
 
-        new_shell="/bin/bash"
-        if [[ ! -d "/home/$new_username"  ]]; then
-            echo "-> Creating user's home folder."
-            home_folder_arg="--create-home"
+            new_shell="/bin/bash"
+            if [[ ! -d "/home/$new_username"  ]]; then
+                echo "-> Creating user's home folder."
+                home_folder_arg="--create-home"
+            else
+                home_folder_arg="--home /home/$new_username"
+                if [[ -f "/home/$new_username/.zshrc" ]]; then
+                    ## Assume the user prefers Zsh as his shell
+                    new_shell="/bin/zsh"
+                fi
+            fi
+            useradd --system $home_folder_arg --shell $new_shell -G sudo -G video -g $new_gid -u $new_uid $new_username
+            echo "$new_username ALL=(ALL:ALL) NOPASSWD: ALL" > /etc/sudoers.d/$new_username
+            # mkdir /tmp/runtime-$new_username
+            # chown $new_username:$new_groupname /tmp/runtime-$new_username
         else
-            home_folder_arg="--home /home/$new_username"
-            if [[ -f "/home/$new_username/.zshrc" ]]; then
-                ## Assume the user prefers Zsh as his shell
-                new_shell="/bin/zsh"
+            ## User exist. Check that the user's name matches
+            if [[ ! $(id -un $new_uid) == "$new_username" ]]; then
+                echo "!! Error: A user with the uid of \"$new_uid\" but with a different user name already exist: \"$(id -un $new_uid)\""
+                exit 1
             fi
-        fi
-        useradd --system $home_folder_arg --shell $new_shell -G sudo -g $new_gid -u $new_uid $new_username
-        echo "$new_username ALL=(ALL:ALL) NOPASSWD: ALL" > /etc/sudoers.d/$new_username
-        # mkdir /tmp/runtime-$new_username
-        # chown $new_username:$new_groupname /tmp/runtime-$new_username
-    else
-        ## User exist. Check that the user's name matches
-        if [[ ! $(id -un $new_uid) == "$new_username" ]]; then
-            echo "!! Error: A user with the uid of \"$new_uid\" but with a different user name already exist: \"$(id -un $new_uid)\""
-            exit 1
-        fi
 
-        ## User exist. Check that the gid matches
-        if [[ ! $(id -g $new_uid) == $new_gid ]]; then
-            echo "!! Error: A user with the uid of \"$new_uid\" but with a different gid already exist: \"$(id -g $new_uid)\""
-            exit 1
-        fi
+            ## User exist. Check that the gid matches
+            if [[ ! $(id -g $new_uid) == $new_gid ]]; then
+                echo "!! Error: A user with the uid of \"$new_uid\" but with a different gid already exist: \"$(id -g $new_uid)\""
+                exit 1
+            fi
 
-        ## User exist. Check that the group's name matches
-        if [[ ! $(id -gn $new_uid) == "$new_groupname" ]]; then
-            echo "!! Error: A user with the uid of \"$new_uid\" but with a different group name already exist: \"$(id -gn $new_uid)\""
-            exit 1
+            ## User exist. Check that the group's name matches
+            if [[ ! $(id -gn $new_uid) == "$new_groupname" ]]; then
+                echo "!! Error: A user with the uid of \"$new_uid\" but with a different group name already exist: \"$(id -gn $new_uid)\""
+                exit 1
+            fi
         fi
-    fi
     
-    if [[ -z "$(ls -A /home/$new_username)" ]]; then
-        echo "-> Reseting user's home folder."
-        cp -rT /etc/skel/. /home/$new_username/
-        chown $new_username:"$new_groupname" -R /home/$new_username
-    fi
+        if [[ -z "$(ls -A /home/$new_username)" ]]; then
+            echo "-> Reseting user's home folder."
+            cp -rT /etc/skel/. /home/$new_username/
+            chown $new_username:"$new_groupname" -R /home/$new_username
+        fi
 
-    if [[ "$(pwd)" == "/root" ]]; then
-        cd /home/$new_username
-    fi
+        if [[ "$(pwd)" == "/root" ]]; then
+            cd /home/$new_username
+        fi
 
-    ln -sfT /home/$new_username/dockvenv /app/dockvenv
-    if [ "$#"  -gt 0 ]; then
-        runuser -u $new_username -- "$@"
-    else
-        if [[ -f /home/$new_username/default_cmd.sh ]]; then
-            runuser -u $new_username /home/$new_username/default_cmd.sh
+        ln -sfT /home/$new_username/dockvenv /app/dockvenv
+        if [ "$#"  -gt 0 ]; then
+            runuser -u $new_username -- "$@"
         else
-            su $new_username
+            if [[ -f /home/$new_username/default_cmd.sh ]]; then
+                runuser -u $new_username /home/$new_username/default_cmd.sh
+            else
+                su $new_username
+            fi
         fi
     fi
 

From 64cc1307dae594cdfb7cb4e4eccefc32bbf078c7 Mon Sep 17 00:00:00 2001
From: Omer Yair <omeryair@gmail.com>
Date: Mon, 15 Jul 2019 09:07:12 +0300
Subject: [PATCH 02/16] Cosmetics

---
 docs/installing_dpendencies.md | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/docs/installing_dpendencies.md b/docs/installing_dpendencies.md
index cf5179c..94a096f 100644
--- a/docs/installing_dpendencies.md
+++ b/docs/installing_dpendencies.md
@@ -1,17 +1,20 @@
 # Installing dependencies
 
 ## NVIDIA dirvers
+
 This docker image uses CUDA 9.0 which requires NVIDIA drivers of version 384.81 or later.
 (To fine the list of NVIDIA drivers version and CUDA version supported be your graphic card
 go to: https://www.geforce.com/drivers)
 
 For some reason installing NVIDIA drivers on Linux tend to cause problems, but in many cases the following
 In some cases the following line would simply to the tick:
+
 ``` bash
 sudo apt -y install nvidia-driver-{driver_version}
 ```
 
 In the case where apt is not able to fine the desired driver try the following method:
+
 1. Go to https://developer.download.nvidia.com/compute/cuda/repos/ and find the the link to the repository
 installation file which fits the desired CUDA version and your system. It should be of the format of:
 *https://developer.download.nvidia.com/compute/cuda/repos/{os_version}/{arch}/cuda-repo-{os_version}_{cuda_version}_{arch}.deb*
@@ -19,6 +22,7 @@ In addition find in the same folder the link to the repository keys, it should b
 *https://developer.download.nvidia.com/compute/cuda/repos/{os_version}/{arch}/{some_number}.pub*
 
 2. Run the following commands to install the NVIDIA drivers + CUDA
+
 ``` bash
 cd /tmp
 curl -L {link_to_the_repo_file_from_stage_1} -o cuda_installation.deb
@@ -28,7 +32,9 @@ sudo apt-get update
 sudo apt-get install -y cuda
 rm cuda_installation.deb
 ```
+
 Example, installing CUDA 10.0 on Ubnutnu 18.04
+
 ``` bash
 cd /tmp
 curl -L wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/cuda-repo-ubuntu1804_10.0.130-1_amd64.deb -o cuda_installation.deb
@@ -39,19 +45,24 @@ apt-get install cuda -y
 ```
 
 For more information you can look at the guides from NVIDIA website:
+
 - CUDA installation guide: https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html
 - The quick start guide: https://docs.nvidia.com/cuda/cuda-quick-start-guide/index.html
 
 ## Docker
+
 Install he latest docker version by following the instructions here:
 https://docs.docker.com/install/linux/docker-ce/ubuntu/#set-up-the-repository
 
 To enable a user to run docker without the need to use *sudo* run the following command:
-```
+
+``` bash
 sudo usermod -aG docker {username}
 ```
+
 Replace {username} with the name of the desired user. To apply this change you will need the user to logout
 and back in again.
 
 ## NVIDIA docker
+
 Install nvidia-docker by following the instructions here: https://github.com/NVIDIA/nvidia-docker

From 946e7ef6d5eec23e0b23162dd9e25af4cbb0111b Mon Sep 17 00:00:00 2001
From: Omer Yair <omeryair@gmail.com>
Date: Tue, 23 Jul 2019 10:26:33 +0000
Subject: [PATCH 03/16] Added run_remote command

---
 mldock.sh | 140 ++++++++++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 137 insertions(+), 3 deletions(-)

diff --git a/mldock.sh b/mldock.sh
index 995a190..6479fe3 100755
--- a/mldock.sh
+++ b/mldock.sh
@@ -1,5 +1,7 @@
 #!/bin/bash
 set -e
+# set -x  # Debug: be verbose
+# PS4='$LINENO: '  # Add lines number to debug output
 
 ## Main CLI function
 ## =================
@@ -24,6 +26,7 @@ main_cli() {
         echo "    setup                   Create a link or a copy of the $app_name.sh script in the /usr/bin folder (requiers sudo)."
         echo "    build                   Build the image."
         echo "    run                     Run a command inside a new container."
+        echo "    run_remote              Run a command inside a new container on a remote machine."
         echo "    exec                    Execute a command inside an existing container."
         echo "    stop                    Stop a running container."
         echo "Use $app_name <command> -h for specific help on each command."
@@ -68,6 +71,9 @@ main_cli() {
         run)
             run_cli "$@"
             ;;
+        run_remote)
+            run_remote_cli "$@"
+            ;;
         exec)
             exec_cli "$@"
             ;;
@@ -275,9 +281,123 @@ run_cli() {
     fi
 
     check_docker_for_sudo
+    gen_command
     run_command
 }
 
+run_remote_cli() {
+    if xhost >& /dev/null ; then
+        ## Display exist
+        connect_to_x_server=true
+    else
+        ## No display
+        connect_to_x_server=false
+    fi
+    if [[ "$(whoami)" == "root" ]]; then
+        userstring="dockuser:4283:dockuser:4283"
+    else
+        userstring="$(whoami):$(id -u):$(id -gn):$(id -g)"
+    fi
+    map_host=true
+    detach_container=false
+    home_folder=""
+    command_to_run=""
+    extra_args=()
+    if nvidia-smi >& /dev/null ; then 
+        use_nvidia_runtime=true
+    else
+        use_nvidia_runtime=false
+    fi
+    usage () {
+        echo "Run a command inside a new container on a remote machine"
+        echo ""
+        echo "usage: $app_name $subcommand $remote_ip [<options>] [command]"
+        echo "   or: $app_name $subcommand -h    to print this help message."
+        echo "Options:"
+        echo "    -v version_name         The version name to use for the build image. Default: \"$version_name\""
+        echo "    -c container_name       The name to for the created container. Default: \"$container_name\""
+        echo "    -f home_folder          A folder to map as the dockuser's home folder."
+        echo "    -s                      Run the command as root."
+        echo "    -u                      Run the command as dockuser user."
+        echo "    -i username             Run the command as *username*."
+        echo "    -x                      Don't connect X-server"
+        echo "    -r                      Don't map the root folder on the host machine to /host inside the container."
+        echo "    -d                      Detach the container."
+        echo "    -e extra_args           Extra arguments to pass to the docker run command."
+    }
+
+    if [ "$#" -eq 1 ] && [ "$1" ==  "-h" ]; then
+        usage
+        exit 0
+    fi
+
+    if [ "$#" -lt 1 ]; then
+        echo "Error: Was expecting a remote ip" 1>&2
+        usage
+        exit 1
+    fi
+
+    remote_ip=$1; shift
+
+    while getopts "v:c:f:sui:xrde:" opt; do
+        case $opt in
+            v)
+                version_name=$OPTARG
+                ;;
+            c)
+                container_name=$OPTARG
+                ;;
+            f)
+                home_folder=$OPTARG
+                ;;
+            s)
+                userstring=""
+                ;;
+            u)
+                userstring="dockuser:4283:dockuser:4283"
+                ;;
+            i)
+                username=$OPTARG
+                userstring="$username:$(id -u $username):$(id -gn $username):$(id -g $username)"
+                ;;
+            x)
+                connect_to_x_server=false
+                ;;
+            r)
+                map_host=false
+                ;;
+            d)
+                detach_container=true
+                ;;
+            e)
+                IFS=$'\n' extra_args=( $(xargs -n1 printf "%s\n" <<< "$OPTARG") )
+                unset IFS
+                ;;
+            :)
+                echo "Error: -$OPTARG requires an argument" 1>&2
+                usage
+                exit 1
+                ;;
+            \?)
+                echo "Error: unknown option -$OPTARG" 1>&2
+                usage
+                exit 1
+                ;;
+        esac
+    done
+    shift $((OPTIND -1))
+
+    userstring="${userstring// /-}"
+
+    if [ "$#" -gt 0 ]; then
+        command_to_run=( "$@" )
+    fi
+
+    check_docker_for_sudo
+    gen_command
+    run_remote_command
+}
+
 exec_cli() {
     extra_args=()
     command_to_run="bash"
@@ -404,7 +524,7 @@ build_image() {
     fi
 }
 
-run_command() {
+gen_command() {
     # # if [ "user_host_user" = true ]; then
     # if [ true = true ]; then
     #     extra_args="-v /etc/passwd:/etc/passwd -u $(id -u ${USER}):$(id -g ${USER})"
@@ -426,10 +546,15 @@ run_command() {
         extra_args+=("-e" "USERSTRING=$userstring" "--label" "new_username=$new_username")
 
         if [[ ! -z $home_folder ]]; then
+            home_folder_full=$(readlink -f $home_folder || echo "") >/dev/null 2>&1
+            if [[ ! -z $home_folder_full ]]; then
+                home_folder=home_folder_full
+            fi
+
             if [[ "$new_username" == "root" ]]; then
-                extra_args+=("-v" "$(readlink -f $home_folder):/root/")
+                extra_args+=("-v" "$home_folder:/root/")
             else
-                extra_args+=("-v" "$(readlink -f $home_folder):/home/$new_username/")
+                extra_args+=("-v" "$home_folder:/home/$new_username/")
             fi
         fi
     fi
@@ -457,11 +582,20 @@ run_command() {
     if [[ ! -z "$command_to_run" ]]; then
         cmd+=("$repository$image_name:$version_name" "${command_to_run[@]}")
     fi
+}
+
+run_command() {
     echo "Running: ${cmd[@]}"
     echo ""
     "${cmd[@]}"
 }
 
+run_remote_command() {
+    echo "Running on $remote_ip: ${cmd[@]}"
+    echo ""
+    ssh $remote_ip -t "${cmd[@]}"
+}
+
 exec_command() {
     # new_username=$(${docker_sudo_prefix}docker inspect $container_name | jq -r '.[0]["Config"]["Labels"]["new_username"]')
     # if [[ "$new_username" != "null" ]]; then

From 93b9ae3021e12960df90a3782d928b8a9ff74b07 Mon Sep 17 00:00:00 2001
From: Omer Yair <omeryair@gmail.com>
Date: Wed, 14 Aug 2019 10:33:12 +0300
Subject: [PATCH 04/16] Moved to ubuntu 18.04

---
 Dockerfile | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 3c27b20..0e42d60 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,5 +1,4 @@
-FROM nvidia/cuda:9.0-cudnn7-devel-ubuntu16.04
-# FROM nvidia/cuda:9.2-cudnn7-devel-ubuntu18.04
+FROM nvidia/cuda:10.0-cudnn7-devel-ubuntu18.04
 
 ## Install basic packages and useful utilities
 ## ===========================================

From f9f3b7ced345fb61b793a8c085e26930865fa1b8 Mon Sep 17 00:00:00 2001
From: Omer Yair <omeryair@gmail.com>
Date: Wed, 14 Aug 2019 10:43:27 +0300
Subject: [PATCH 05/16] Added pylint and ncurses

---
 Dockerfile | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/Dockerfile b/Dockerfile
index 0e42d60..174ecce 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -24,6 +24,7 @@ RUN apt-get update -y && \
         python \
         python-dev \
         python-pip \
+        pylint \
         sshfs \
         wget \
         curl \
@@ -68,6 +69,8 @@ RUN apt-get update -y && \
         texlive-latex-extra \
         texlive-xetex \
         graphviz \
+        libncurses5-dev \
+        libncursesw5-dev \
         && \
     apt-get install -y neovim && \
     pip install pynvim==0.3.2 && \

From f26d77ed0a8061b53d842ddd6793f4b09700952e Mon Sep 17 00:00:00 2001
From: Omer Yair <omeryair@gmail.com>
Date: Wed, 14 Aug 2019 10:45:25 +0300
Subject: [PATCH 06/16] Added dash torchsymmery visdom and line_profiles

---
 Dockerfile | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 174ecce..3e0c85c 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -120,7 +120,7 @@ RUN mkdir /app && \
 
 ## Setup python environment
 ## ========================
-RUN pip3 install pip==18.1 && \
+RUN pip3 install pip==19.2.2 && \
     hash -r pip && \
     pip3 install -U \
         virtualenv==16.2.0 \
@@ -131,6 +131,7 @@ RUN pip3 install pip==18.1 && \
         PyQt5==5.11.3 \
         seaborn==0.9.0 \
         plotly==3.5.0 \
+        dash==1.0.2 \
         bokeh==1.0.4 \
         ggplot==0.11.5 \
         altair==2.3.0 \
@@ -149,6 +150,7 @@ RUN pip3 install pip==18.1 && \
         scikit-learn==0.20.0 \
         imageio==2.4.1 \
         torchvision==0.2.1 \
+        torchsummary==1.5.1 \
         tensorflow-gpu==1.12.0 \
         tensorboardX==1.4 \
         jupyter==1.0.0 \
@@ -156,6 +158,8 @@ RUN pip3 install pip==18.1 && \
         jupyter_contrib_nbextensions==0.5.0 \
         jupyterlab==0.4.0 \
         ipywidgets==7.4.2 \
+        visdom==0.1.8.8 \
+        line_profiler==2.1.2 \
         && \
         rm -r /root/.cache/pip
 ENV MPLBACKEND=Agg

From 25a2393f39f521f0222688ab0b5efb83b7844865 Mon Sep 17 00:00:00 2001
From: Omer Yair <omeryair@gmail.com>
Date: Wed, 14 Aug 2019 11:03:11 +0300
Subject: [PATCH 07/16] renamed entrypoint -> switch_user_run

---
 Dockerfile                                      | 4 ++--
 resources/{entrypoint.sh => switch_user_run.sh} | 0
 2 files changed, 2 insertions(+), 2 deletions(-)
 rename resources/{entrypoint.sh => switch_user_run.sh} (100%)

diff --git a/Dockerfile b/Dockerfile
index 3e0c85c..76a9612 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -212,7 +212,7 @@ RUN mkdir /app/bin && \
     chmod a=u -R /app/bin && \
     sed -i "s/^\(PATH=\"\)\(.*\)$/\1\/app\/bin\/:\2/g" /etc/environment
 ENV PATH="/app/bin:$PATH"
-COPY /resources/entrypoint.sh /app/bin/run
+COPY /resources/switch_user_run.sh /app/bin/switch_user_run
 COPY /resources/default_notebook.sh /app/bin/default_notebook
 COPY /resources/default_jupyterlab.sh /app/bin/default_jupyterlab
 COPY /resources/run_server.sh /app/bin/run_server
@@ -234,4 +234,4 @@ ENV LANGUAGE en_US:en
 ENV LC_ALL en_US.UTF-8  
 
 WORKDIR /root
-ENTRYPOINT ["/usr/bin/dumb-init", "--", "run"]
+ENTRYPOINT ["/usr/bin/dumb-init", "--", "switch_user_run"]
diff --git a/resources/entrypoint.sh b/resources/switch_user_run.sh
similarity index 100%
rename from resources/entrypoint.sh
rename to resources/switch_user_run.sh

From 633d20ca88b86b94013fca6d96ebc3406a7aa206 Mon Sep 17 00:00:00 2001
From: Omer Yair <omeryair@gmail.com>
Date: Wed, 14 Aug 2019 11:04:48 +0300
Subject: [PATCH 08/16] Bug fix - No default command when running as user

---
 resources/switch_user_run.sh | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/resources/switch_user_run.sh b/resources/switch_user_run.sh
index 47f1db8..af09409 100755
--- a/resources/switch_user_run.sh
+++ b/resources/switch_user_run.sh
@@ -3,6 +3,12 @@ set -e
 
 trap 'shutdown' TERM INT  ## This line makes shutting down the container faster
 
+if [ "$#"  -gt 0 ]; then
+    cmd=("$@")
+else
+    cmd=("bash")
+fi
+
 if [ ! -z "$USERSTRING" ]; then
     IFS=':' read -ra USERSTRINGSPLIT <<< "$USERSTRING"
 
@@ -72,7 +78,7 @@ if [ ! -z "$USERSTRING" ]; then
 
         ln -sfT /home/$new_username/dockvenv /app/dockvenv
         if [ "$#"  -gt 0 ]; then
-            runuser -u $new_username -- "$@"
+            runuser -u $new_username -- "${cmd[@]}"
         else
             if [[ -f /home/$new_username/default_cmd.sh ]]; then
                 runuser -u $new_username /home/$new_username/default_cmd.sh
@@ -81,11 +87,6 @@ if [ ! -z "$USERSTRING" ]; then
             fi
         fi
     fi
-
 else
-    if [ "$#"  -gt 0 ]; then
-        "$@"
-    else
-        bash
-    fi
-fi
+    "$cmd"
+fi
\ No newline at end of file

From 7705c70a593962b53fe426b9781cb0c3e5c1e99c Mon Sep 17 00:00:00 2001
From: Omer Yair <omeryair@gmail.com>
Date: Wed, 14 Aug 2019 11:07:49 +0300
Subject: [PATCH 09/16] Store username inside the conatiner

---
 Dockerfile                   | 5 +++++
 mldock.sh                    | 6 ++----
 resources/switch_user_run.sh | 2 ++
 3 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 76a9612..2e53efa 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -206,6 +206,11 @@ RUN cd /tmp && \
     dpkg -i dumb-init.deb && \
     rm dumb-init.deb
 
+## Create container config file
+## ============================
+RUN mkdir /tmp/dock_config && \
+    chmod a+wrx /tmp/dock_config
+
 ## Copy scripts
 ## ============
 RUN mkdir /app/bin && \
diff --git a/mldock.sh b/mldock.sh
index 6479fe3..dfd7102 100755
--- a/mldock.sh
+++ b/mldock.sh
@@ -543,7 +543,7 @@ gen_command() {
         userstringsplit=(${userstring//:/ })
         new_username=${userstringsplit[0]}
 
-        extra_args+=("-e" "USERSTRING=$userstring" "--label" "new_username=$new_username")
+        extra_args+=("-e" "USERSTRING=$userstring")
 
         if [[ ! -z $home_folder ]]; then
             home_folder_full=$(readlink -f $home_folder || echo "") >/dev/null 2>&1
@@ -597,9 +597,7 @@ run_remote_command() {
 }
 
 exec_command() {
-    # new_username=$(${docker_sudo_prefix}docker inspect $container_name | jq -r '.[0]["Config"]["Labels"]["new_username"]')
-    # if [[ "$new_username" != "null" ]]; then
-    new_username=$(${docker_sudo_prefix}docker inspect $container_name | sed -n 's/^[[:space:]]*"new_username":[[:space:]]*"\(.*\)"$/\1/p')
+    new_username=$(${docker_sudo_prefix}docker exec $container_name cat /tmp/dock_config/username)
     if [[ ! -z "$new_username" ]]; then
         extra_args="$extra_args -u $new_username -w /home/$new_username"
     fi
diff --git a/resources/switch_user_run.sh b/resources/switch_user_run.sh
index af09409..bc05933 100755
--- a/resources/switch_user_run.sh
+++ b/resources/switch_user_run.sh
@@ -17,6 +17,8 @@ if [ ! -z "$USERSTRING" ]; then
     new_groupname="${USERSTRINGSPLIT[2]}"
     new_gid=${USERSTRINGSPLIT[3]}
 
+    echo "$new_username" > /tmp/dock_config/username
+
     if [[ ! $new_username == "root" ]]; then
         if ! id $new_uid > /dev/null 2>&1; then
             ## User does not exist. Create user

From 91f59eda34303d76803c008a5bcb1a245f295b51 Mon Sep 17 00:00:00 2001
From: Omer Yair <omeryair@gmail.com>
Date: Wed, 14 Aug 2019 11:11:31 +0300
Subject: [PATCH 10/16] Added: run using tmux

---
 Dockerfile                        |  1 +
 mldock.sh                         | 44 +++++++++++++++++++++++--------
 resources/run_in_detached_tmux.sh | 14 ++++++++++
 3 files changed, 48 insertions(+), 11 deletions(-)
 create mode 100755 resources/run_in_detached_tmux.sh

diff --git a/Dockerfile b/Dockerfile
index 2e53efa..b5ca794 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -221,6 +221,7 @@ COPY /resources/switch_user_run.sh /app/bin/switch_user_run
 COPY /resources/default_notebook.sh /app/bin/default_notebook
 COPY /resources/default_jupyterlab.sh /app/bin/default_jupyterlab
 COPY /resources/run_server.sh /app/bin/run_server
+COPY /resources/run_in_detached_tmux.sh /app/bin/run_in_detached_tmux
 
 RUN touch /etc/skel/.sudo_as_admin_successful
 
diff --git a/mldock.sh b/mldock.sh
index dfd7102..de2e738 100755
--- a/mldock.sh
+++ b/mldock.sh
@@ -197,6 +197,7 @@ run_cli() {
     detach_container=false
     home_folder=""
     command_to_run=""
+    use_tmux=false
     extra_args=()
     if nvidia-smi >& /dev/null ; then 
         use_nvidia_runtime=true
@@ -218,6 +219,7 @@ run_cli() {
         echo "    -x                      Don't connect X-server"
         echo "    -r                      Don't map the root folder on the host machine to /host inside the container."
         echo "    -d                      Detach the container."
+        echo "    -t                      Run in at TMUX session."
         echo "    -e extra_args           Extra arguments to pass to the docker run command."
     }
 
@@ -226,7 +228,7 @@ run_cli() {
         exit 0
     fi
 
-    while getopts "v:c:f:sui:xrde:" opt; do
+    while getopts "v:c:f:sui:xrdte:" opt; do
         case $opt in
             v)
                 version_name=$OPTARG
@@ -256,6 +258,9 @@ run_cli() {
             d)
                 detach_container=true
                 ;;
+            t)
+                use_tmux=true
+                ;;
             e)
                 IFS=$'\n' extra_args=( $(xargs -n1 printf "%s\n" <<< "$OPTARG") )
                 unset IFS
@@ -302,6 +307,7 @@ run_remote_cli() {
     detach_container=false
     home_folder=""
     command_to_run=""
+    use_tmux=false
     extra_args=()
     if nvidia-smi >& /dev/null ; then 
         use_nvidia_runtime=true
@@ -323,6 +329,7 @@ run_remote_cli() {
         echo "    -x                      Don't connect X-server"
         echo "    -r                      Don't map the root folder on the host machine to /host inside the container."
         echo "    -d                      Detach the container."
+        echo "    -t                      Run in at TMUX session."
         echo "    -e extra_args           Extra arguments to pass to the docker run command."
     }
 
@@ -339,7 +346,7 @@ run_remote_cli() {
 
     remote_ip=$1; shift
 
-    while getopts "v:c:f:sui:xrde:" opt; do
+    while getopts "v:c:f:sui:xrdte:" opt; do
         case $opt in
             v)
                 version_name=$OPTARG
@@ -369,6 +376,9 @@ run_remote_cli() {
             d)
                 detach_container=true
                 ;;
+            t)
+                use_tmux=true
+                ;;
             e)
                 IFS=$'\n' extra_args=( $(xargs -n1 printf "%s\n" <<< "$OPTARG") )
                 unset IFS
@@ -559,6 +569,10 @@ gen_command() {
         fi
     fi
 
+    if [ "$use_tmux" = true ]; then
+        detach_tmux="$detach_container"
+        detach_container=true
+    fi
     if [ "$detach_container" = true ]; then
         extra_args+=("-d")
     else
@@ -568,32 +582,40 @@ gen_command() {
     if [ "$use_nvidia_runtime" = true ]; then
         extra_args+=("--runtime=nvidia")
     fi
-
-    if [ ! -z "$docker_sudo_prefix" ]; then
-        cmd=("sudo")
-    else
-        cmd=()
-    fi
     cmd+=("docker" "run" \
          "--rm" \
          "--network" "host" \
          "--name" "$container_name")
     cmd+=( "${extra_args[@]}" )
+    cmd+=("$repository$image_name:$version_name")
+    if [ "$use_tmux" = true ]; then
+        cmd+=("run_in_detached_tmux")
+    fi
     if [[ ! -z "$command_to_run" ]]; then
-        cmd+=("$repository$image_name:$version_name" "${command_to_run[@]}")
+        cmd+=("${command_to_run[@]}")
     fi
 }
 
 run_command() {
-    echo "Running: ${cmd[@]}"
+    echo "Running: ${docker_sudo_prefix}${cmd[@]}"
     echo ""
-    "${cmd[@]}"
+    "${docker_sudo_prefix}${cmd[@]}"
+
+    if [ "$use_tmux" = true ] && [[ "$detach_tmux" = false ]]; then
+        new_username=$(${docker_sudo_prefix}docker exec $container_name cat /tmp/dock_config/username)
+        ${docker_sudo_prefix}docker exec -it -u $new_username $container_name tmux a
+    fi
 }
 
 run_remote_command() {
     echo "Running on $remote_ip: ${cmd[@]}"
     echo ""
     ssh $remote_ip -t "${cmd[@]}"
+
+    if [ "$use_tmux" = true ] && [[ "$detach_tmux" = false ]]; then
+        new_username=$(ssh $remote_ip -t docker exec $container_name cat /tmp/dock_config/username)
+        ssh $remote_ip -t docker exec -it -u $new_username $container_name tmux a
+    fi
 }
 
 exec_command() {
diff --git a/resources/run_in_detached_tmux.sh b/resources/run_in_detached_tmux.sh
new file mode 100755
index 0000000..c36f158
--- /dev/null
+++ b/resources/run_in_detached_tmux.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+set -e
+
+if [ "$#"  -gt 0 ]; then
+    cmd=("$@")
+else
+    cmd=("bash")
+fi
+
+trap 'exit 0' TERM INT  ## This line makes shutting down the container faster
+
+tmux new-session -n run_window -s run_session -d "${cmd[@]}"
+tmux set-hook -t run_session session-closed "wait-for -S finished"
+tmux wait-for finised

From 3ae38b5eea4ba3ab914a69a029a62b7da7063bd4 Mon Sep 17 00:00:00 2001
From: Omer Yair <omeryair@gmail.com>
Date: Mon, 19 Aug 2019 11:41:07 +0300
Subject: [PATCH 11/16] Bugfix:  in setting home folder

---
 mldock.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mldock.sh b/mldock.sh
index de2e738..2d2a79e 100755
--- a/mldock.sh
+++ b/mldock.sh
@@ -558,7 +558,7 @@ gen_command() {
         if [[ ! -z $home_folder ]]; then
             home_folder_full=$(readlink -f $home_folder || echo "") >/dev/null 2>&1
             if [[ ! -z $home_folder_full ]]; then
-                home_folder=home_folder_full
+                home_folder=$home_folder_full
             fi
 
             if [[ "$new_username" == "root" ]]; then

From 5da287432988c2e988f08ccbc551a8ef26ec3dc7 Mon Sep 17 00:00:00 2001
From: Omer Yair <omeryair@gmail.com>
Date: Mon, 19 Aug 2019 11:52:25 +0300
Subject: [PATCH 12/16] Minor Bigfix: wrong error massage

---
 mldock.sh | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/mldock.sh b/mldock.sh
index 2d2a79e..f7eb27c 100755
--- a/mldock.sh
+++ b/mldock.sh
@@ -43,7 +43,7 @@ main_cli() {
                 exit 0
                 ;;
             \? )
-                echo "Error: Invalid Option: -$OPTARG" 1>&2
+                echo "Error: Invalid Option: -$opt" 1>&2
                 usage
                 exit 1
                 ;;
@@ -109,12 +109,12 @@ setup_cli() {
                 copy_script_file=true
                 ;;
             :)
-                echo "Error: -$OPTARG requires an argument" 1>&2
+                echo "Error: -$opt requires an argument" 1>&2
                 usage
                 exit 1
                 ;;
             \?)
-                echo "Error: unknown option -$OPTARG" 1>&2
+                echo "Error: unknown option -$opt" 1>&2
                 usage
                 exit 1
                 ;;
@@ -157,12 +157,12 @@ build_cli() {
                 tag_as_latest=false
                 ;;
             :)
-                echo "Error: -$OPTARG requires an argument" 1>&2
+                echo "Error: -$opt requires an argument" 1>&2
                 usage
                 exit 1
                 ;;
             \?)
-                echo "Error: unknown option -$OPTARG" 1>&2
+                echo "Error: unknown option -$opt" 1>&2
                 usage
                 exit 1
                 ;;
@@ -266,12 +266,12 @@ run_cli() {
                 unset IFS
                 ;;
             :)
-                echo "Error: -$OPTARG requires an argument" 1>&2
+                echo "Error: -$opt requires an argument" 1>&2
                 usage
                 exit 1
                 ;;
             \?)
-                echo "Error: unknown option -$OPTARG" 1>&2
+                echo "Error: unknown option -$opt" 1>&2
                 usage
                 exit 1
                 ;;
@@ -384,12 +384,12 @@ run_remote_cli() {
                 unset IFS
                 ;;
             :)
-                echo "Error: -$OPTARG requires an argument" 1>&2
+                echo "Error: -$opt requires an argument" 1>&2
                 usage
                 exit 1
                 ;;
             \?)
-                echo "Error: unknown option -$OPTARG" 1>&2
+                echo "Error: unknown option -$opt" 1>&2
                 usage
                 exit 1
                 ;;
@@ -441,7 +441,7 @@ exec_cli() {
                 exit 1
                 ;;
             \?)
-                echo "Error: unknown option -$OPTARG" 1>&2
+                echo "Error: unknown option -$opt" 1>&2
                 usage
                 exit 1
                 ;;
@@ -478,12 +478,12 @@ stop_cli() {
                 container_name=$OPTARG
                 ;;
             :)
-                echo "Error: -$OPTARG requires an argument" 1>&2
+                echo "Error: -$opt requires an argument" 1>&2
                 usage
                 exit 1
                 ;;
             \?)
-                echo "Error: unknown option -$OPTARG" 1>&2
+                echo "Error: unknown option -$opt" 1>&2
                 usage
                 exit 1
                 ;;

From 187a6030fd8da9f6bac516bba185268c7269efd2 Mon Sep 17 00:00:00 2001
From: Omer Yair <omeryair@gmail.com>
Date: Mon, 19 Aug 2019 12:01:18 +0300
Subject: [PATCH 13/16] Try to install nvtop (if using nvidia runtime)

---
 Dockerfile | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/Dockerfile b/Dockerfile
index b5ca794..b0dd277 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -78,6 +78,17 @@ RUN apt-get update -y && \
 
     ## ToDo: increase memory limit to 10GB in: /etc/ImageMagick-6/policy.xml
 
+## Install nvtop
+## =============
+RUN git clone https://github.com/Syllo/nvtop.git /tmp/nvtop && \
+    mkdir /tmp/nvtop/build && \
+    cd /tmp/nvtop/build && \
+    cmake .. || : && \
+    make || : && \
+    make install || : && \
+    cd / && \
+    rm -r /tmp/nvtop
+
 ## Set locale
 ## ==========
 RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \

From a4a644aea6bcb6b99873b079164ec5f5e27e4bc5 Mon Sep 17 00:00:00 2001
From: Omer Yair <omeryair@gmail.com>
Date: Sun, 2 Feb 2020 13:54:27 +0200
Subject: [PATCH 14/16] Added exec_remote

---
 mldock.sh | 96 +++++++++++++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 89 insertions(+), 7 deletions(-)

diff --git a/mldock.sh b/mldock.sh
index f7eb27c..078c61d 100755
--- a/mldock.sh
+++ b/mldock.sh
@@ -28,6 +28,7 @@ main_cli() {
         echo "    run                     Run a command inside a new container."
         echo "    run_remote              Run a command inside a new container on a remote machine."
         echo "    exec                    Execute a command inside an existing container."
+        echo "    exec_remote             Execute a command inside an existing container on a remote machine."
         echo "    stop                    Stop a running container."
         echo "Use $app_name <command> -h for specific help on each command."
     }
@@ -77,6 +78,9 @@ main_cli() {
         exec)
             exec_cli "$@"
             ;;
+        exec_remote)
+            exec_remote_cli "$@"
+            ;;
         stop)
             stop_cli "$@"
             ;;
@@ -201,6 +205,11 @@ run_cli() {
     extra_args=()
     if nvidia-smi >& /dev/null ; then 
         use_nvidia_runtime=true
+        if docker run --rm --runtime=nvidia busybox echo test >& /dev/null ; then 
+            use_gpus_all=false
+        else
+            use_gpus_all=true
+        fi
     else
         use_nvidia_runtime=false
     fi
@@ -309,15 +318,10 @@ run_remote_cli() {
     command_to_run=""
     use_tmux=false
     extra_args=()
-    if nvidia-smi >& /dev/null ; then 
-        use_nvidia_runtime=true
-    else
-        use_nvidia_runtime=false
-    fi
     usage () {
         echo "Run a command inside a new container on a remote machine"
         echo ""
-        echo "usage: $app_name $subcommand $remote_ip [<options>] [command]"
+        echo "usage: $app_name $subcommand remote_ip [<options>] [command]"
         echo "   or: $app_name $subcommand -h    to print this help message."
         echo "Options:"
         echo "    -v version_name         The version name to use for the build image. Default: \"$version_name\""
@@ -403,6 +407,16 @@ run_remote_cli() {
         command_to_run=( "$@" )
     fi
 
+    if ssh $remote_ip nvidia-smi >& /dev/null ; then 
+        use_nvidia_runtime=true
+        if ssh $remote_ip docker run --rm --runtime=nvidia busybox echo test >& /dev/null ; then 
+            use_gpus_all=false
+        else
+            use_gpus_all=true
+        fi
+    else
+        use_nvidia_runtime=false
+    fi
     check_docker_for_sudo
     gen_command
     run_remote_command
@@ -457,6 +471,55 @@ exec_cli() {
     exec_command
 }
 
+exec_remote_cli() {
+    extra_args=()
+    usage () {
+        echo "Execute a command inside an existing container"
+        echo ""
+        echo "usage: $app_name $subcommand remote_ip [<options>] [command_to_run]"
+        echo "   or: $app_name $subcommand -h    to print this help message."
+        echo "Options:"
+        echo "    -c container_name       The name to for the created container. default: \"$container_name\""
+        echo "    -e extra_args           Extra arguments to pass to the docker exec command."
+    }
+
+    if [ "$#" -eq 1 ] && [ "$1" ==  "-h" ]; then
+        usage
+        exit 0
+    fi
+
+    remote_ip=$1; shift
+
+    while getopts "c:u:e:" opt; do
+        case $opt in
+            c)
+                container_name=$OPTARG
+                ;;
+            e)
+                IFS=$'\n' extra_args=( $(xargs -n1 printf "%s\n" <<< "$OPTARG") )
+                unset IFS
+                ;;
+            :)
+                echo "Error: -$OPTARG requires an argument" 1>&2
+                usage
+                exit 1
+                ;;
+            \?)
+                echo "Error: unknown option -$opt" 1>&2
+                usage
+                exit 1
+                ;;
+        esac
+    done
+    shift $((OPTIND -1))
+
+    if [ "$#" -gt 0 ]; then
+        command_to_run=( "$@" )
+    fi
+
+    exec_remote_command
+}
+
 stop_cli() {
     usage () {
         echo "Stop a running container."
@@ -580,7 +643,11 @@ gen_command() {
     fi
 
     if [ "$use_nvidia_runtime" = true ]; then
-        extra_args+=("--runtime=nvidia")
+        if [ "$use_gpus_all" = true ]; then
+            extra_args+=("--gpus=all")
+        else
+            extra_args+=("--runtime=nvidia")
+        fi
     fi
     cmd+=("docker" "run" \
          "--rm" \
@@ -628,6 +695,21 @@ exec_command() {
 }
 
 
+exec_remote_command() {
+    new_username=$(ssh -o LogLevel=QUIET $remote_ip -t "docker exec $container_name cat /tmp/dock_config/username" | tr -d '\r')
+
+    cmd+=("docker" "exec" \
+         "-it")
+    if [[ ! -z "$new_username" ]]; then
+        extra_args+=("-u" "$new_username" "-w" "/home/$new_username")
+    fi
+    cmd+=( "${extra_args[@]}" )
+    cmd+=("$container_name")
+    cmd+=("${command_to_run[@]}")
+    run_remote_command
+}
+
+
 stop_container() {
     echo "-> Stopping the container"
     ${docker_sudo_prefix}docker stop $container_name

From 7716b2a431c6fe76d52cc5ef8c52599bec2a5cce Mon Sep 17 00:00:00 2001
From: Omer Yair <omeryair@gmail.com>
Date: Sun, 2 Feb 2020 13:54:51 +0200
Subject: [PATCH 15/16] Updated tensorflow version

---
 Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index b0dd277..2f71c8b 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -98,7 +98,7 @@ RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \
 ## ==========
 RUN mkdir /var/run/sshd && \
     sed 's/^#\?PasswordAuthentication .*$/PasswordAuthentication yes/g' -i /etc/ssh/sshd_config && \
-    sed 's/^Port .*$/Port 9022/g' -i /etc/ssh/sshd_config && \
+    sed 's/^#\?Port .*$/Port 9022/g' -i /etc/ssh/sshd_config && \
     sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd
 
 ## VSCode
@@ -162,7 +162,7 @@ RUN pip3 install pip==19.2.2 && \
         imageio==2.4.1 \
         torchvision==0.2.1 \
         torchsummary==1.5.1 \
-        tensorflow-gpu==1.12.0 \
+        tensorflow-gpu==1.14.0 \
         tensorboardX==1.4 \
         jupyter==1.0.0 \
         jupyterthemes==0.19.6 \

From b6516c4c599f4b0a660abd998cfc990de4b1642a Mon Sep 17 00:00:00 2001
From: Omer Yair <omeryair@gmail.com>
Date: Sun, 15 Mar 2020 16:26:55 +0200
Subject: [PATCH 16/16] Updated python versions + Orca

---
 Dockerfile | 86 ++++++++++++++++++++++++++++++++----------------------
 mldock.sh  |  2 +-
 2 files changed, 52 insertions(+), 36 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 2f71c8b..5bfc33d 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -3,7 +3,7 @@ FROM nvidia/cuda:10.0-cudnn7-devel-ubuntu18.04
 ## Install basic packages and useful utilities
 ## ===========================================
 ENV DEBIAN_FRONTEND noninteractive
-RUN apt-get update -y && \
+RUN apt-get update -y  && \
     apt-get upgrade -y && \
     apt-get install -y software-properties-common && \
     add-apt-repository ppa:neovim-ppa/stable && \
@@ -131,46 +131,49 @@ RUN mkdir /app && \
 
 ## Setup python environment
 ## ========================
-RUN pip3 install pip==19.2.2 && \
+RUN pip3 install pip==20.0.2 && \
     hash -r pip && \
     pip3 install -U \
-        virtualenv==16.2.0 \
-        ipython==7.0.1 \
-        numpy==1.15.2 \
-        scipy==1.1.0 \
-        matplotlib==3.0.0 \
-        PyQt5==5.11.3 \
-        seaborn==0.9.0 \
-        plotly==3.5.0 \
-        dash==1.0.2 \
-        bokeh==1.0.4 \
+        virtualenv==20.0.10 \
+        ipython==7.13.0 \
+        numpy==1.18.1 \
+        scipy==1.4.0 \
+        cvxpy==1.0.28 \
+        matplotlib==3.2.0 \
+        PyQt5==5.14.1 \
+        seaborn==0.10.0 \
+        plotly==4.5.3 \
+        dash==1.9.1 \
+        bokeh==2.0.0 \
         ggplot==0.11.5 \
-        altair==2.3.0 \
-        pandas==0.23.4 \
-        pyyaml==3.13 \
-        protobuf==3.6.1 \
-        ipdb==0.11 \
-        flake8==3.5.0 \
-        cython==0.28.5 \
-        sympy==1.3 \
+        altair==4.0.1 \
+        pandas==1.0.1 \
+        pyyaml==5.3 \
+        protobuf==3.11.3 \
+        ipdb==0.13.2 \
+        flake8==3.7.9 \
+        cython==0.29.15 \
+        sympy==1.5.1 \
         nose==1.3.7 \
         sphinx==1.8.1 \
-        tqdm==4.27.0 \
-        opencv-contrib-python==3.4.3.18 \
-        scikit-image==0.14.1 \
-        scikit-learn==0.20.0 \
-        imageio==2.4.1 \
-        torchvision==0.2.1 \
+        tqdm==4.43.0 \
+        opencv-contrib-python==4.2.0.32 \
+        scikit-image==0.16.2 \
+        scikit-learn==0.22.2 \
+        imageio==2.8.0 \
+        torchvision==0.4.0 \
+        torchviz==0.0.1 \
+        Pillow==6.1 \
         torchsummary==1.5.1 \
-        tensorflow-gpu==1.14.0 \
-        tensorboardX==1.4 \
+        tensorflow-gpu==2.0.0 \
+        tensorboardX==2.0 \
         jupyter==1.0.0 \
-        jupyterthemes==0.19.6 \
-        jupyter_contrib_nbextensions==0.5.0 \
-        jupyterlab==0.4.0 \
-        ipywidgets==7.4.2 \
-        visdom==0.1.8.8 \
-        line_profiler==2.1.2 \
+        jupyterthemes==0.20.0 \
+        jupyter_contrib_nbextensions==0.5.1 \
+        jupyterlab==2.0.1 \
+        ipywidgets==7.5.1 \
+        visdom==0.1.8.9 \
+        line_profiler==3.0.2 \
         && \
         rm -r /root/.cache/pip
 ENV MPLBACKEND=Agg
@@ -191,11 +194,23 @@ RUN pip install six==1.11 && \
     jupyter-nbextension install rise --py --sys-prefix --system && \
     cp -r /root/.jupyter /etc/skel/
 
+## Install Orca (for exporting Plotly figures to images)
+## =====================================================
+RUN apt install -y xvfb libgconf2-4 && \
+    wget https://github.com/plotly/orca/releases/download/v1.1.1/orca-1.1.1-x86_64.AppImage -P /tmp && \
+    chmod 777 /tmp/orca-1.1.1-x86_64.AppImage && \
+    cd /tmp && \
+    ./orca-1.1.1-x86_64.AppImage --appimage-extract && \
+    mv /tmp/squashfs-root /opt/squashfs-root && \
+    chmod -R 777 /opt/squashfs-root && \
+    printf '#!/bin/bash \nxvfb-run --auto-servernum --server-args "-screen 0 640x480x24" /opt/squashfs-root/app/orca "$@"' > /usr/bin/orca && \
+    chmod 777 /usr/bin/orca && \
+    rm -r /tmp/orca-1.1.1-x86_64.AppImage
+
 ## Create virtual environment
 ## ==========================
 RUN cd /app/ && \
     virtualenv --system-site-packages dockvenv && \
-    virtualenv --relocatable dockvenv && \
     grep -rlnw --null /usr/local/bin/ -e '#!/usr/bin/python3' | xargs -0r cp -t /app/dockvenv/bin/ && \
     sed -i "s/#"'!'"\/usr\/bin\/python3/#"'!'"\/usr\/bin\/env python/g" /app/dockvenv/bin/* && \
     mv /app/dockvenv /root/ && \
@@ -203,6 +218,7 @@ RUN cd /app/ && \
     cp -rp /root/dockvenv /etc/skel/ && \
     sed -i "s/^\(PATH=\"\)\(.*\)$/\1\/app\/dockvenv\/bin\/:\2/g" /etc/environment
 ENV PATH=/app/dockvenv/bin:$PATH
+    # virtualenv dockvenv && \
 
 ## Node.js
 ## =======
diff --git a/mldock.sh b/mldock.sh
index 078c61d..8eb8332 100755
--- a/mldock.sh
+++ b/mldock.sh
@@ -9,7 +9,7 @@ app_name=mldock
 
 repository="omeryair/"
 image_name="mldock"
-version_name="v0.4"
+version_name="v0.5"
 
 container_name="mldock_$USER"