enhancing readme; setting token using ENVs and updating some libs

Author: flaviostutz

Dockerfile

@@ -1,4 +1,4 @@
-FROM tensorflow/tensorflow:2.0.0a0-py3-jupyter
+FROM tensorflow/tensorflow:2.1.0-py3-jupyter
 # FROM flaviostutz/datascience-tools:2.0.0
 
 RUN apt-get update && \
@@ -44,7 +44,7 @@ RUN pip install pydicom && \
     pip install scikit-image && \
     pip install opencv-python && \
     pip install ImageHash && \
-    apt-get install libav-tools -y && \
+    apt-get install ffmpeg -y && \
     apt-get install imagemagick -y && \
     pip install git+https://github.com/danoneata/selectivesearch.git
 
@@ -70,19 +70,22 @@ RUN pip install Geohash && \
     pip install rasterio && \
     pip install rasterstats && \
     pip install folium && \
-    pip install pyepsg && \
-    apt-get install -y software-properties-common && \
+    pip install pyepsg
+RUN apt-get install -y software-properties-common && \
     add-apt-repository -y ppa:ubuntugis/ubuntugis-unstable && \
     apt update && \
-    apt-get install libproj-dev libgeos-dev -y && \
-    apt-get install -y gdal-bin python-gdal python3-gdal && \
+    apt-get install -y gdal-bin && \
+    apt-get install -y python-gdal python3-gdal && \
+    apt-get install -y libproj-dev libgeos-dev && \
     pip install mgrspy && \
-    pip install git+https://github.com/flaviostutz/sentinelloader && \
-    apt-get install -y libspatialindex-dev && \
+    pip install git+https://github.com/flaviostutz/sentinelloader
+RUN apt-get install -y libspatialindex-dev && \
     pip install rtree && \
     pip uninstall -y pyepsg && \
     pip install git+https://github.com/flaviostutz/pyepsg && \
-    pip install cartopy
+    pip uninstall -y enum34 && \
+    pip install cython && \
+    pip install git+https://github.com/snowman2/cartopy.git@5e624fe
 
 #SPARK DRIVER
 # RUN apt-get install openjdk-8-jdk -y
@@ -107,4 +110,7 @@ RUN apt-get install supervisor -y
 ADD supervisord.conf /etc/supervisor/conf.d/supervisord.conf
 ADD /start-jupyter.sh /
 
+ENV JUPYTER_TOKEN ''
+ENV SPARK_MASTER ''
+
 CMD ["/usr/bin/supervisord"]

Dockerfile-gpu

@@ -1,4 +1,4 @@
-FROM tensorflow/tensorflow:2.0.0a0-gpu-py3-jupyter
+FROM tensorflow/tensorflow:2.1.0-gpu-py3-jupyter
 
 RUN apt-get update && \
     apt-get install git -y
@@ -68,19 +68,22 @@ RUN pip install Geohash && \
     pip install rasterio && \
     pip install rasterstats && \
     pip install folium && \
-    pip install pyepsg && \
-    apt-get install -y software-properties-common && \
+    pip install pyepsg
+RUN apt-get install -y software-properties-common && \
     add-apt-repository -y ppa:ubuntugis/ubuntugis-unstable && \
     apt update && \
-    apt-get install libproj-dev libgeos-dev -y && \
-    apt-get install -y gdal-bin python-gdal python3-gdal && \
+    apt-get install -y gdal-bin && \
+    apt-get install -y python-gdal python3-gdal && \
+    apt-get install -y libproj-dev libgeos-dev && \
     pip install mgrspy && \
-    pip install git+https://github.com/flaviostutz/sentinelloader && \
-    apt-get install -y libspatialindex-dev && \
+    pip install git+https://github.com/flaviostutz/sentinelloader
+RUN apt-get install -y libspatialindex-dev && \
     pip install rtree && \
     pip uninstall -y pyepsg && \
     pip install git+https://github.com/flaviostutz/pyepsg && \
-    pip install cartopy
+    pip uninstall -y enum34 && \
+    pip install cython && \
+    pip install git+https://github.com/snowman2/cartopy.git@5e624fe
 
 #SPARK DRIVER
 #RUN apt-get install openjdk-8-jdk -y
@@ -104,5 +107,9 @@ RUN mkdir -p /notebooks/data/output
 RUN apt-get install supervisor -y
 ADD supervisord.conf /etc/supervisor/conf.d/supervisord.conf
 ADD /start-jupyter.sh /
+ADD /hashpass.py /
+
+ENV JUPYTER_TOKEN ''
+ENV SPARK_MASTER ''
 
 CMD ["/usr/bin/supervisord"]

README.md

@@ -13,17 +13,15 @@ This container was created to support various experimentations on Datascience, m
 - Scoop, h5py, pandas, scikit, TFLearn, plotly
 - pyexcel-ods, pydicom, textblob, wavio, trueskill, cytoolz, ImageHash...
 
-**Run container:**
+## Run container:
 
    - CPU only:
 
       * create docker-compose.yml
 
       ```
       version: "3"
-
       services:
-
         datascience-tools:
           image: flaviostutz/datascience-tools
           ports:
@@ -32,8 +30,7 @@ This container was created to support various experimentations on Datascience, m
           volumes:
             - /notebooks:/notebooks
           environment:
-            - PASSWORD=flaviostutz
-            - SPARK_MASTER=spark-master
+            - JUPYTER_TOKEN=flaviostutz
       ```
       * `docker-compose up`
 
@@ -53,8 +50,12 @@ This container was created to support various experimentations on Datascience, m
          ./boot.sh >> /var/log/boot-script`
       - Change "/root/datascience-tools" to where you cloned this repo
 
+**Access:**
 
-**Autorun script:**
+- http://[ip]:8888 for Jupyter
+- http://[ip]:6006 for TensorBoard
+
+## Autorun script
 
    - When this container starts, it runs:
       - Jupyter Notebook server on port 8888
@@ -70,15 +71,24 @@ This container was created to support various experimentations on Datascience, m
             - `#!/bin/bash
                python test.py`
 
-**Access:**
+## Build instructions
+
+- `docker build . -f Dockerfile`
+- `docker build . -f Dockerfile-gpu`
+
+## Tips for development of your own Notebooks
+
+- A good practice is to store your notebook scripts in a git repository
+  - Ex.: http://github.com/flaviostutz/puzzler
+
+- Run datascience-tools container and map the volume "/notebooks", inside the container, to the path you cloned your git repository in your computer
+
+- You can edit/save/run the scripts from the web interface (http://localhost:8888) or directly with other tools on your computer. You can commit and push your code to the destination repository directly (no copy from/to container is needed because the volume is mapped)
 
-   - http://[ip]:8888 for Jupyter
-   - http://[ip]:6006 for TensorBoard
+- For running in production, create a new container with "FROM flaviostutz/datascience-tools" and add your script files to "/notebooks" so when you run the container it will have your custom scripts embedded into it. No "volume" mapping is needed for this container.
 
-**Build container:**
+## ENVs variables
 
-   - `docker-compose build cpu`
-     OR
-   - `docker build . -f Dockerfile`
-   - `docker build . -f Dockerfile-gpu`
+- JUPYTER_TOKEN - token needed for the users to open Jupyter. defaults to '', so that no token or password will asked to the user
 
+- SPARK_MASTER - Spark master address. Used if you want to send jobs to an external Spark cluster and still control the whole job from Jupyter Notebook itself.

docker-compose-gpu.yml

@@ -1,16 +1,16 @@
-version: "3"
+version: "3.5"
 
 services:
 
   datascience-tools-gpu:
     build:
       context: .
       dockerfile: Dockerfile-gpu
-    image: flaviostutz/datascience-tools:2.8.0-gpu
+    image: flaviostutz/datascience-tools:2.9.0-gpu
     ports:
       - 8888:8888
       - 6006:6006
-    volumes:
-      - /notebooks:/notebooks
+    # volumes:
+    #   - /notebooks:/notebooks
     environment:
-      - PASSWORD=flaviostutz
+      - JUPYTER_PASSWORD=flaviostutz

docker-compose.yml

@@ -1,17 +1,16 @@
-version: "3"
+version: "3.5"
 
 services:
 
   datascience-tools:
     build:
       context: .
       dockerfile: Dockerfile
-    image: flaviostutz/datascience-tools:2.8.0
+    image: flaviostutz/datascience-tools:2.9.0
     ports:
       - 8888:8888
       - 6006:6006
     # volumes:
     #   - /notebooks:/notebooks
     environment:
-      - PASSWORD=flaviostutz
-      - SPARK_MASTER=spark-master
+      - JUPYTER_TOKEN=flaviostutz

start-jupyter.sh

@@ -6,5 +6,5 @@ echo "Starting Jupyter..."
 # export PATH=$SPARK_HOME/bin:$PATH
 # export PYSPARK_SUBMIT_ARGS="--master $SPARK_MASTER"
 
-jupyter notebook "$@" --ip 0.0.0.0 --no-browser --allow-root --NotebookApp.token='' --NotebookApp.password='' --NotebookApp.notebook_dir='/notebooks'
+jupyter notebook "$@" --ip 0.0.0.0 --no-browser --allow-root --NotebookApp.allow_password_change=False --NotebookApp.token="$JUPYTER_TOKEN" --NotebookApp.password='' --NotebookApp.notebook_dir='/notebooks'