Commit all

eodcgmbh · Dec 3, 2024 · c76ae5a · c76ae5a
1 parent ffcc047
commit c76ae5a
Show file tree

Hide file tree

Showing 17 changed files with 1,508 additions and 2 deletions.
diff --git a/README.md b/README.md
@@ -1,2 +1,69 @@
-# julia-jeg-kernel
-Julia JEG Kernel
+# README
+
+## Components
+
+### container/
+
+These scripts run inside the Julia/Jupyter container or are used to create it.
+
+* `kernel-launchers/`
+
+  This directory is expected to be in the container at `/usr/local/bin/`.
+
+  * `julia/scripts/launch_ijuliakernel.jl`
+    * Started by `bootstrap_kernel.sh` with the relevant environment variables (kernel ID, public key, etc.) as parameters, which are set by the JEG at container startup. This, in turn, invokes the `server_listener.py` as a subprocess.
+  * `julia/scripts/server_listener.py`
+    * This takes the initial connection parameters from the JEG, attempts to find (but not bind to) available ports accordingly for [ZeroMQ](https://zeromq.org/), and finally responds to JEG with a so-called `connection_file`, which includes all parameters to establish a connection between JEG and the Julia kernel.
+
+* `bootstrap-kernel.sh`
+
+  * The script that is initially invoked when the kernel container starts. Depending on the chosen language, it calls the corresponding runtime. In the case of Julia, `launch_ijuliakernel.jl` with the necessary parameters from JEG is started.
+
+* `eventloop.jl`
+
+  * The event loop at the kernel's core that continously receives messages from JEG. This file is slightly extend from recent commits to [IJulia](https://github.com/JuliaLang/IJulia.jl) and needs to be fixed such that no adaptations are required (see comments and FIXME).
+
+* `init.jl`
+
+  * The last startup script for IJulia that is invoked by `launch_ijuliakernel.jl`. This takes the connection parameters created by `server_listener.py` in order to bind to ZeroMQ ports and set up corresponding message queues. Furthermore, all standard streams are redirected to IJulia.
+
+* `Dockerfile`
+  * The defining file for Julia kernels. It is based on `quay.io/jupyter/julia-notebook` but extended such that the modified files herein are included in the final image. The startup script is `bootstrap-kernel.sh`.
+
+
+### gateway/kernelspec-image
+
+These files are meant to be placed at the Jupyter Enterprise Gateway.
+
+* `julia_kubernetes/`
+  * `kernel.json`
+    * This is the kernel specification that defines the image to be used and how Kubernetes is supposed to start the pod that holds the running Julia kernel container.
+  * `logo-64x64.png`
+    * The Julia logo.
+  * `scripts/kernel-pod.yaml.j2`
+    * The Jinja template that defines the kernel's pod can be modified to mount certain paths or set memory and compute limits, for example.
+  * `scripts/launch_kubernetes.py`
+    * With this script, Kubernetes is instructed to initiate a pod according to the kernel specification. This is unaltered, but must be included nonetheless.
+
+
+* `Dockerfile`
+  * The Dockerfile for this kernelspec image that is mounted by the enterprise gateway pod.
+
+### scripts/
+
+* `build_kernel_image.sh`
+  Builds and pushes the Julia kernel image `julia-kernel:alpha`
+
+* `build_kernelspec_image.sh`
+  Builds and pushes the kernel specification image `julia-kernelspec:alpha` that includes Julia.
+
+* `cmd.sh`
+  Bash commands to start JEG and JupyterLab in a Minikube cluster.
+
+* `pre-pull.sh`
+  Some commands that prematurely pull relevant images on the kernel image pullers.
+
+### test/
+
+* `julia-testnb.ipynb`
+  * A Julia notebook that allows to test the basic functionalities, including multi-threaded code execution.
diff --git a/kernel-julia/container/Dockerfile b/kernel-julia/container/Dockerfile
@@ -0,0 +1,43 @@
+# Ubuntu 18.04.1 LTS Bionic
+ARG BASE_CONTAINER=quay.io/jupyter/julia-notebook:2024-05-27
+FROM ${BASE_CONTAINER}
+
+ENV PATH=${PATH}:${CONDA_DIR}/bin
+
+RUN pip install --upgrade ipykernel
+
+RUN conda install --quiet --yes \
+    cffi \
+    future \
+    pycryptodomex && \
+    conda clean --all && \
+    fix-permissions ${CONDA_DIR} && \
+    fix-permissions /home/${NB_USER}
+
+RUN conda update --all && \
+    conda clean --all
+
+USER jovyan
+ADD jupyter_enterprise_gateway_kernel_image_files*.tar.gz /usr/local/bin/
+
+USER root
+RUN mv /usr/local/bin/init.jl \
+    $(julia --eval "using IJulia; println(pathof(IJulia))" | rev | cut --delimiter='/' --fields=2- | rev)
+RUN mv /usr/local/bin/eventloop.jl \
+    $(julia --eval "using IJulia; println(pathof(IJulia))" | rev | cut --delimiter='/' --fields=2- | rev)
+
+RUN apt-get update && apt-get install --yes --quiet --no-install-recommends \
+    libkrb5-dev \
+    && rm --recursive --force /var/lib/apt/lists/*
+
+RUN chown jovyan:users /usr/local/bin/bootstrap-kernel.sh && \
+    chmod 0755 /usr/local/bin/bootstrap-kernel.sh && \
+    chown --recursive jovyan:users /usr/local/bin/kernel-launchers
+
+USER jovyan
+
+ENV KERNEL_LANGUAGE=julia
+
+HEALTHCHECK NONE
+
+CMD /usr/local/bin/bootstrap-kernel.sh
diff --git a/kernel-julia/container/bootstrap-kernel.sh b/kernel-julia/container/bootstrap-kernel.sh
@@ -0,0 +1,126 @@
+#!/bin/bash
+
+PORT_RANGE=${PORT_RANGE:-${EG_PORT_RANGE:-0..0}}
+RESPONSE_ADDRESS=${RESPONSE_ADDRESS:-${EG_RESPONSE_ADDRESS}}
+PUBLIC_KEY=${PUBLIC_KEY:-${EG_PUBLIC_KEY}}
+KERNEL_LAUNCHERS_DIR=${KERNEL_LAUNCHERS_DIR:-/usr/local/bin/kernel-launchers}
+KERNEL_SPARK_CONTEXT_INIT_MODE=${KERNEL_SPARK_CONTEXT_INIT_MODE:-none}
+KERNEL_CLASS_NAME=${KERNEL_CLASS_NAME}
+
+echo $0 env: `env`
+
+launch_python_kernel() {
+  # Launch the python kernel launcher - which embeds the IPython kernel and listens for interrupts
+  # and shutdown requests from Enterprise Gateway.
+
+  export JPY_PARENT_PID=$$  # Force reset of parent pid since we're detached
+
+  if [ -z "${KERNEL_CLASS_NAME}" ]
+  then
+    kernel_class_option=""
+  else
+    kernel_class_option="--kernel-class-name ${KERNEL_CLASS_NAME}"
+  fi
+
+	set -x
+	python ${KERNEL_LAUNCHERS_DIR}/python/scripts/launch_ipykernel.py --kernel-id ${KERNEL_ID} \
+	      --port-range ${PORT_RANGE} --response-address ${RESPONSE_ADDRESS} --public-key ${PUBLIC_KEY} \
+	      --spark-context-initialization-mode ${KERNEL_SPARK_CONTEXT_INIT_MODE} ${kernel_class_option}
+	{ set +x; } 2>/dev/null
+}
+
+launch_R_kernel() {
+    # Launch the R kernel launcher - which embeds the IRkernel kernel and listens for interrupts
+    # and shutdown requests from Enterprise Gateway.
+
+	set -x
+	Rscript ${KERNEL_LAUNCHERS_DIR}/R/scripts/launch_IRkernel.R --kernel-id ${KERNEL_ID} --port-range ${PORT_RANGE} \
+    --response-address ${RESPONSE_ADDRESS} --public-key ${PUBLIC_KEY} \
+    --spark-context-initialization-mode ${KERNEL_SPARK_CONTEXT_INIT_MODE}
+	{ set +x; } 2>/dev/null
+}
+
+launch_julia_kernel() {
+    # Launch the Julia kernel launcher - which embeds the IJulia kernel and listens for interrupts
+    # and shutdown requests from Enterprise Gateway.
+
+	set -x
+    export JULIA_NUM_THREADS=$(( ($(nproc) - 2) > 0 ? $(nproc) - 2 : 1 )) # Use all but 2 cores for Julia
+
+	julia --debug-info=1 --optimize=1 \
+        ${KERNEL_LAUNCHERS_DIR}/julia/scripts/launch_ijuliakernel.jl \
+        --kernel-id ${KERNEL_ID} --port-range ${PORT_RANGE} \
+        --response-address ${RESPONSE_ADDRESS} --public-key ${PUBLIC_KEY}
+
+	{ set +x; } 2>/dev/null
+}
+
+launch_scala_kernel() {
+    # Launch the scala kernel launcher - which embeds the Apache Toree kernel and listens for interrupts
+    # and shutdown requests from Enterprise Gateway.  This kernel is currenly always launched using
+    # spark-submit, so additional setup is required.
+
+    PROG_HOME=${KERNEL_LAUNCHERS_DIR}/scala
+    KERNEL_ASSEMBLY=`(cd "${PROG_HOME}/lib"; ls -1 toree-assembly-*.jar;)`
+    TOREE_ASSEMBLY="${PROG_HOME}/lib/${KERNEL_ASSEMBLY}"
+    if [ ! -f ${TOREE_ASSEMBLY} ]; then
+        echo "Toree assembly '${PROG_HOME}/lib/toree-assembly-*.jar' is missing.  Exiting..."
+        exit 1
+    fi
+
+    # Toree launcher jar path, plus required lib jars (toree-assembly)
+    JARS="${TOREE_ASSEMBLY}"
+    # Toree launcher app path
+    LAUNCHER_JAR=`(cd "${PROG_HOME}/lib"; ls -1 toree-launcher*.jar;)`
+    LAUNCHER_APP="${PROG_HOME}/lib/${LAUNCHER_JAR}"
+    if [ ! -f ${LAUNCHER_APP} ]; then
+        echo "Scala launcher jar '${PROG_HOME}/lib/toree-launcher*.jar' is missing.  Exiting..."
+        exit 1
+    fi
+
+    SPARK_OPTS="--name ${KERNEL_USERNAME}-${KERNEL_ID}"
+    TOREE_OPTS="--alternate-sigint USR2"
+
+    set -x
+    eval exec \
+         "${SPARK_HOME}/bin/spark-submit" \
+         "${SPARK_OPTS}" \
+         --jars "${JARS}" \
+         --class launcher.ToreeLauncher \
+         "${LAUNCHER_APP}" \
+         "${TOREE_OPTS}" \
+         "--kernel-id ${KERNEL_ID} --port-range ${PORT_RANGE} --response-address ${RESPONSE_ADDRESS} --public-key ${PUBLIC_KEY} --spark-context-initialization-mode ${KERNEL_SPARK_CONTEXT_INIT_MODE}"
+    { set +x; } 2>/dev/null
+}
+
+# Ensure that required envs are present, check language before the dynamic values
+if [ -z "${KERNEL_LANGUAGE+x}" ]
+then
+    echo "KERNEL_LANGUAGE is required.  Set this value in the image or when starting container."
+    exit 1
+fi
+if [ -z "${KERNEL_ID+x}" ] || [ -z "${RESPONSE_ADDRESS+x}" ] || [ -z "${PUBLIC_KEY+x}" ]
+then
+    echo "Environment variables, KERNEL_ID, RESPONSE_ADDRESS, and PUBLIC_KEY are required."
+    exit 1
+fi
+
+# Invoke appropriate launcher based on KERNEL_LANGUAGE (case-insensitive)
+
+if [[ "${KERNEL_LANGUAGE,,}" == "python" ]]
+then
+    launch_python_kernel
+elif [[ "${KERNEL_LANGUAGE,,}" == "julia" ]]
+then
+    launch_julia_kernel
+elif [[ "${KERNEL_LANGUAGE,,}" == "scala" ]]
+then
+    launch_scala_kernel
+elif [[ "${KERNEL_LANGUAGE,,}" == "r" ]]
+then
+    launch_R_kernel
+else
+	echo "Unrecognized value for KERNEL_LANGUAGE: '${KERNEL_LANGUAGE}'!"
+	exit 1
+fi
+exit 0
diff --git a/kernel-julia/container/eventloop.jl b/kernel-julia/container/eventloop.jl
@@ -0,0 +1,70 @@
+__precompile__(false)
+
+# FIXME: vprintln and unknown_request are defined in stdio.jl and handlers.jl
+
+# from stdio.jl
+macro vprintln(x...)
+    quote
+        if verbose::Bool
+            println(orig_stdout[], get_log_preface(), $(map(esc, x)...))
+        end
+    end
+end
+
+# from handlers.jl
+function unknown_request(socket, msg)
+    @vprintln("UNKNOWN MESSAGE TYPE $(msg.header["msg_type"])")
+end
+
+
+function eventloop(socket)
+    task_local_storage(:IJulia_task, "write task")
+    try
+        while true
+            msg = recv_ipython(socket)
+            try
+                send_status("busy", msg)
+                invokelatest(get(handlers, msg.header["msg_type"], unknown_request), socket, msg)
+            catch e
+                # Try to keep going if we get an exception, but
+                # send the exception traceback to the front-ends.
+                # (Ignore SIGINT since this may just be a user-requested
+                #  kernel interruption to interrupt long calculations.)
+                if !isa(e, InterruptException)
+                    content = error_content(e, msg="KERNEL EXCEPTION")
+                    map(s -> println(orig_stderr[], s), content["traceback"])
+                    send_ipython(publish[], msg_pub(execute_msg, "error", content))
+                end
+            finally
+                flush_all()
+                send_status("idle", msg)
+            end
+        end
+    catch e
+        # the Jupyter manager may send us a SIGINT if the user
+        # chooses to interrupt the kernel; don't crash on this
+        if isa(e, InterruptException)
+            eventloop(socket)
+        else
+            rethrow()
+        end
+    end
+end
+
+const requests_task = Ref{Task}()
+function waitloop()
+    @async eventloop(control[])
+    requests_task[] = @async eventloop(requests[])
+    while true
+        try
+            wait()
+        catch e
+            # send interrupts (user SIGINT) to the code-execution task
+            if isa(e, InterruptException)
+                @async Base.throwto(requests_task[], e)
+            else
+                rethrow()
+            end
+        end
+    end
+end