From c76ae5a675c9bc3b7ec56def7defdf28f20bd55e Mon Sep 17 00:00:00 2001 From: "david.fischak@eodc.eu" Date: Tue, 3 Dec 2024 13:10:24 +0100 Subject: [PATCH] Commit all --- README.md | 71 ++- kernel-julia/container/Dockerfile | 43 ++ kernel-julia/container/bootstrap-kernel.sh | 126 ++++++ kernel-julia/container/eventloop.jl | 70 +++ kernel-julia/container/init.jl | 123 ++++++ .../julia/scripts/launch_ijuliakernel.jl | 51 +++ .../julia/scripts/server_listener.py | 301 +++++++++++++ .../gateway/kernelspec-image/Dockerfile | 7 + .../julia_kubernetes/kernel.json | 26 ++ .../julia_kubernetes/logo-64x64.png | Bin 0 -> 31744 bytes .../scripts/kernel-pod.yaml.j2 | 97 ++++ .../scripts/launch_kubernetes.py | 414 ++++++++++++++++++ scripts/build_kernel_image.sh | 19 + scripts/build_kernelspec_image.sh | 11 + scripts/cmd.sh | 27 ++ scripts/pre-pull.sh | 13 + test/julia-testnb.ipynb | 111 +++++ 17 files changed, 1508 insertions(+), 2 deletions(-) create mode 100644 kernel-julia/container/Dockerfile create mode 100755 kernel-julia/container/bootstrap-kernel.sh create mode 100644 kernel-julia/container/eventloop.jl create mode 100644 kernel-julia/container/init.jl create mode 100644 kernel-julia/container/kernel-launchers/julia/scripts/launch_ijuliakernel.jl create mode 100644 kernel-julia/container/kernel-launchers/julia/scripts/server_listener.py create mode 100644 kernel-julia/gateway/kernelspec-image/Dockerfile create mode 100644 kernel-julia/gateway/kernelspec-image/julia_kubernetes/kernel.json create mode 100644 kernel-julia/gateway/kernelspec-image/julia_kubernetes/logo-64x64.png create mode 100644 kernel-julia/gateway/kernelspec-image/julia_kubernetes/scripts/kernel-pod.yaml.j2 create mode 100755 kernel-julia/gateway/kernelspec-image/julia_kubernetes/scripts/launch_kubernetes.py create mode 100755 scripts/build_kernel_image.sh create mode 100755 scripts/build_kernelspec_image.sh create mode 100755 scripts/cmd.sh create mode 100755 scripts/pre-pull.sh create mode 100644 test/julia-testnb.ipynb diff --git a/README.md b/README.md index a544b6f..8194d43 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,69 @@ -# julia-jeg-kernel -Julia JEG Kernel +# README + +## Components + +### container/ + +These scripts run inside the Julia/Jupyter container or are used to create it. + +* `kernel-launchers/` + + This directory is expected to be in the container at `/usr/local/bin/`. + + * `julia/scripts/launch_ijuliakernel.jl` + * Started by `bootstrap_kernel.sh` with the relevant environment variables (kernel ID, public key, etc.) as parameters, which are set by the JEG at container startup. This, in turn, invokes the `server_listener.py` as a subprocess. + * `julia/scripts/server_listener.py` + * This takes the initial connection parameters from the JEG, attempts to find (but not bind to) available ports accordingly for [ZeroMQ](https://zeromq.org/), and finally responds to JEG with a so-called `connection_file`, which includes all parameters to establish a connection between JEG and the Julia kernel. + +* `bootstrap-kernel.sh` + + * The script that is initially invoked when the kernel container starts. Depending on the chosen language, it calls the corresponding runtime. In the case of Julia, `launch_ijuliakernel.jl` with the necessary parameters from JEG is started. + +* `eventloop.jl` + + * The event loop at the kernel's core that continously receives messages from JEG. This file is slightly extend from recent commits to [IJulia](https://github.com/JuliaLang/IJulia.jl) and needs to be fixed such that no adaptations are required (see comments and FIXME). + +* `init.jl` + + * The last startup script for IJulia that is invoked by `launch_ijuliakernel.jl`. This takes the connection parameters created by `server_listener.py` in order to bind to ZeroMQ ports and set up corresponding message queues. Furthermore, all standard streams are redirected to IJulia. + +* `Dockerfile` + * The defining file for Julia kernels. It is based on `quay.io/jupyter/julia-notebook` but extended such that the modified files herein are included in the final image. The startup script is `bootstrap-kernel.sh`. + + +### gateway/kernelspec-image + +These files are meant to be placed at the Jupyter Enterprise Gateway. + +* `julia_kubernetes/` + * `kernel.json` + * This is the kernel specification that defines the image to be used and how Kubernetes is supposed to start the pod that holds the running Julia kernel container. + * `logo-64x64.png` + * The Julia logo. + * `scripts/kernel-pod.yaml.j2` + * The Jinja template that defines the kernel's pod can be modified to mount certain paths or set memory and compute limits, for example. + * `scripts/launch_kubernetes.py` + * With this script, Kubernetes is instructed to initiate a pod according to the kernel specification. This is unaltered, but must be included nonetheless. + + +* `Dockerfile` + * The Dockerfile for this kernelspec image that is mounted by the enterprise gateway pod. + +### scripts/ + +* `build_kernel_image.sh` + Builds and pushes the Julia kernel image `julia-kernel:alpha` + +* `build_kernelspec_image.sh` + Builds and pushes the kernel specification image `julia-kernelspec:alpha` that includes Julia. + +* `cmd.sh` + Bash commands to start JEG and JupyterLab in a Minikube cluster. + +* `pre-pull.sh` + Some commands that prematurely pull relevant images on the kernel image pullers. + +### test/ + +* `julia-testnb.ipynb` + * A Julia notebook that allows to test the basic functionalities, including multi-threaded code execution. diff --git a/kernel-julia/container/Dockerfile b/kernel-julia/container/Dockerfile new file mode 100644 index 0000000..0be09e1 --- /dev/null +++ b/kernel-julia/container/Dockerfile @@ -0,0 +1,43 @@ +# Ubuntu 18.04.1 LTS Bionic +ARG BASE_CONTAINER=quay.io/jupyter/julia-notebook:2024-05-27 +FROM ${BASE_CONTAINER} + +ENV PATH=${PATH}:${CONDA_DIR}/bin + +RUN pip install --upgrade ipykernel + +RUN conda install --quiet --yes \ + cffi \ + future \ + pycryptodomex && \ + conda clean --all && \ + fix-permissions ${CONDA_DIR} && \ + fix-permissions /home/${NB_USER} + +RUN conda update --all && \ + conda clean --all + +USER jovyan +ADD jupyter_enterprise_gateway_kernel_image_files*.tar.gz /usr/local/bin/ + +USER root +RUN mv /usr/local/bin/init.jl \ + $(julia --eval "using IJulia; println(pathof(IJulia))" | rev | cut --delimiter='/' --fields=2- | rev) +RUN mv /usr/local/bin/eventloop.jl \ + $(julia --eval "using IJulia; println(pathof(IJulia))" | rev | cut --delimiter='/' --fields=2- | rev) + +RUN apt-get update && apt-get install --yes --quiet --no-install-recommends \ + libkrb5-dev \ + && rm --recursive --force /var/lib/apt/lists/* + +RUN chown jovyan:users /usr/local/bin/bootstrap-kernel.sh && \ + chmod 0755 /usr/local/bin/bootstrap-kernel.sh && \ + chown --recursive jovyan:users /usr/local/bin/kernel-launchers + +USER jovyan + +ENV KERNEL_LANGUAGE=julia + +HEALTHCHECK NONE + +CMD /usr/local/bin/bootstrap-kernel.sh diff --git a/kernel-julia/container/bootstrap-kernel.sh b/kernel-julia/container/bootstrap-kernel.sh new file mode 100755 index 0000000..ce3bcaa --- /dev/null +++ b/kernel-julia/container/bootstrap-kernel.sh @@ -0,0 +1,126 @@ +#!/bin/bash + +PORT_RANGE=${PORT_RANGE:-${EG_PORT_RANGE:-0..0}} +RESPONSE_ADDRESS=${RESPONSE_ADDRESS:-${EG_RESPONSE_ADDRESS}} +PUBLIC_KEY=${PUBLIC_KEY:-${EG_PUBLIC_KEY}} +KERNEL_LAUNCHERS_DIR=${KERNEL_LAUNCHERS_DIR:-/usr/local/bin/kernel-launchers} +KERNEL_SPARK_CONTEXT_INIT_MODE=${KERNEL_SPARK_CONTEXT_INIT_MODE:-none} +KERNEL_CLASS_NAME=${KERNEL_CLASS_NAME} + +echo $0 env: `env` + +launch_python_kernel() { + # Launch the python kernel launcher - which embeds the IPython kernel and listens for interrupts + # and shutdown requests from Enterprise Gateway. + + export JPY_PARENT_PID=$$ # Force reset of parent pid since we're detached + + if [ -z "${KERNEL_CLASS_NAME}" ] + then + kernel_class_option="" + else + kernel_class_option="--kernel-class-name ${KERNEL_CLASS_NAME}" + fi + + set -x + python ${KERNEL_LAUNCHERS_DIR}/python/scripts/launch_ipykernel.py --kernel-id ${KERNEL_ID} \ + --port-range ${PORT_RANGE} --response-address ${RESPONSE_ADDRESS} --public-key ${PUBLIC_KEY} \ + --spark-context-initialization-mode ${KERNEL_SPARK_CONTEXT_INIT_MODE} ${kernel_class_option} + { set +x; } 2>/dev/null +} + +launch_R_kernel() { + # Launch the R kernel launcher - which embeds the IRkernel kernel and listens for interrupts + # and shutdown requests from Enterprise Gateway. + + set -x + Rscript ${KERNEL_LAUNCHERS_DIR}/R/scripts/launch_IRkernel.R --kernel-id ${KERNEL_ID} --port-range ${PORT_RANGE} \ + --response-address ${RESPONSE_ADDRESS} --public-key ${PUBLIC_KEY} \ + --spark-context-initialization-mode ${KERNEL_SPARK_CONTEXT_INIT_MODE} + { set +x; } 2>/dev/null +} + +launch_julia_kernel() { + # Launch the Julia kernel launcher - which embeds the IJulia kernel and listens for interrupts + # and shutdown requests from Enterprise Gateway. + + set -x + export JULIA_NUM_THREADS=$(( ($(nproc) - 2) > 0 ? $(nproc) - 2 : 1 )) # Use all but 2 cores for Julia + + julia --debug-info=1 --optimize=1 \ + ${KERNEL_LAUNCHERS_DIR}/julia/scripts/launch_ijuliakernel.jl \ + --kernel-id ${KERNEL_ID} --port-range ${PORT_RANGE} \ + --response-address ${RESPONSE_ADDRESS} --public-key ${PUBLIC_KEY} + + { set +x; } 2>/dev/null +} + +launch_scala_kernel() { + # Launch the scala kernel launcher - which embeds the Apache Toree kernel and listens for interrupts + # and shutdown requests from Enterprise Gateway. This kernel is currenly always launched using + # spark-submit, so additional setup is required. + + PROG_HOME=${KERNEL_LAUNCHERS_DIR}/scala + KERNEL_ASSEMBLY=`(cd "${PROG_HOME}/lib"; ls -1 toree-assembly-*.jar;)` + TOREE_ASSEMBLY="${PROG_HOME}/lib/${KERNEL_ASSEMBLY}" + if [ ! -f ${TOREE_ASSEMBLY} ]; then + echo "Toree assembly '${PROG_HOME}/lib/toree-assembly-*.jar' is missing. Exiting..." + exit 1 + fi + + # Toree launcher jar path, plus required lib jars (toree-assembly) + JARS="${TOREE_ASSEMBLY}" + # Toree launcher app path + LAUNCHER_JAR=`(cd "${PROG_HOME}/lib"; ls -1 toree-launcher*.jar;)` + LAUNCHER_APP="${PROG_HOME}/lib/${LAUNCHER_JAR}" + if [ ! -f ${LAUNCHER_APP} ]; then + echo "Scala launcher jar '${PROG_HOME}/lib/toree-launcher*.jar' is missing. Exiting..." + exit 1 + fi + + SPARK_OPTS="--name ${KERNEL_USERNAME}-${KERNEL_ID}" + TOREE_OPTS="--alternate-sigint USR2" + + set -x + eval exec \ + "${SPARK_HOME}/bin/spark-submit" \ + "${SPARK_OPTS}" \ + --jars "${JARS}" \ + --class launcher.ToreeLauncher \ + "${LAUNCHER_APP}" \ + "${TOREE_OPTS}" \ + "--kernel-id ${KERNEL_ID} --port-range ${PORT_RANGE} --response-address ${RESPONSE_ADDRESS} --public-key ${PUBLIC_KEY} --spark-context-initialization-mode ${KERNEL_SPARK_CONTEXT_INIT_MODE}" + { set +x; } 2>/dev/null +} + +# Ensure that required envs are present, check language before the dynamic values +if [ -z "${KERNEL_LANGUAGE+x}" ] +then + echo "KERNEL_LANGUAGE is required. Set this value in the image or when starting container." + exit 1 +fi +if [ -z "${KERNEL_ID+x}" ] || [ -z "${RESPONSE_ADDRESS+x}" ] || [ -z "${PUBLIC_KEY+x}" ] +then + echo "Environment variables, KERNEL_ID, RESPONSE_ADDRESS, and PUBLIC_KEY are required." + exit 1 +fi + +# Invoke appropriate launcher based on KERNEL_LANGUAGE (case-insensitive) + +if [[ "${KERNEL_LANGUAGE,,}" == "python" ]] +then + launch_python_kernel +elif [[ "${KERNEL_LANGUAGE,,}" == "julia" ]] +then + launch_julia_kernel +elif [[ "${KERNEL_LANGUAGE,,}" == "scala" ]] +then + launch_scala_kernel +elif [[ "${KERNEL_LANGUAGE,,}" == "r" ]] +then + launch_R_kernel +else + echo "Unrecognized value for KERNEL_LANGUAGE: '${KERNEL_LANGUAGE}'!" + exit 1 +fi +exit 0 diff --git a/kernel-julia/container/eventloop.jl b/kernel-julia/container/eventloop.jl new file mode 100644 index 0000000..44b64c1 --- /dev/null +++ b/kernel-julia/container/eventloop.jl @@ -0,0 +1,70 @@ +__precompile__(false) + +# FIXME: vprintln and unknown_request are defined in stdio.jl and handlers.jl + +# from stdio.jl +macro vprintln(x...) + quote + if verbose::Bool + println(orig_stdout[], get_log_preface(), $(map(esc, x)...)) + end + end +end + +# from handlers.jl +function unknown_request(socket, msg) + @vprintln("UNKNOWN MESSAGE TYPE $(msg.header["msg_type"])") +end + + +function eventloop(socket) + task_local_storage(:IJulia_task, "write task") + try + while true + msg = recv_ipython(socket) + try + send_status("busy", msg) + invokelatest(get(handlers, msg.header["msg_type"], unknown_request), socket, msg) + catch e + # Try to keep going if we get an exception, but + # send the exception traceback to the front-ends. + # (Ignore SIGINT since this may just be a user-requested + # kernel interruption to interrupt long calculations.) + if !isa(e, InterruptException) + content = error_content(e, msg="KERNEL EXCEPTION") + map(s -> println(orig_stderr[], s), content["traceback"]) + send_ipython(publish[], msg_pub(execute_msg, "error", content)) + end + finally + flush_all() + send_status("idle", msg) + end + end + catch e + # the Jupyter manager may send us a SIGINT if the user + # chooses to interrupt the kernel; don't crash on this + if isa(e, InterruptException) + eventloop(socket) + else + rethrow() + end + end +end + +const requests_task = Ref{Task}() +function waitloop() + @async eventloop(control[]) + requests_task[] = @async eventloop(requests[]) + while true + try + wait() + catch e + # send interrupts (user SIGINT) to the code-execution task + if isa(e, InterruptException) + @async Base.throwto(requests_task[], e) + else + rethrow() + end + end + end +end diff --git a/kernel-julia/container/init.jl b/kernel-julia/container/init.jl new file mode 100644 index 0000000..0bdaff4 --- /dev/null +++ b/kernel-julia/container/init.jl @@ -0,0 +1,123 @@ +import Random: seed! + +using Logging + + +# use our own random seed for msg_id so that we +# don't alter the user-visible random state (issue #336) +const IJulia_RNG = seed!(Random.MersenneTwister(0)) +import UUIDs +uuid4() = string(UUIDs.uuid4(IJulia_RNG)) + +const orig_stdin = Ref{IO}() +const orig_stdout = Ref{IO}() +const orig_stderr = Ref{IO}() +const SOFTSCOPE = Ref{Bool}() +function __init__() + seed!(IJulia_RNG) + orig_stdin[] = stdin + orig_stdout[] = stdout + orig_stderr[] = stderr + SOFTSCOPE[] = lowercase(get(ENV, "IJULIA_SOFTSCOPE", "yes")) in ("yes", "true") +end + +# the following constants need to be initialized in init(). +const publish = Ref{Socket}() +const raw_input = Ref{Socket}() +const requests = Ref{Socket}() +const control = Ref{Socket}() +const heartbeat = Ref{Socket}() +const profile = Dict{String,Any}() +const read_stdout = Ref{Base.PipeEndpoint}() +const read_stderr = Ref{Base.PipeEndpoint}() +const socket_locks = Dict{Socket,ReentrantLock}() + +# similar to Pkg.REPLMode.MiniREPL, a minimal REPL-like emulator +# for use with Pkg.do_cmd. We have to roll our own to +# make sure it uses the redirected stdout, and because +# we don't have terminal support. +import REPL +struct MiniREPL <: REPL.AbstractREPL + display::TextDisplay +end +REPL.REPLDisplay(repl::MiniREPL) = repl.display +const minirepl = Ref{MiniREPL}() + +function init(args) + inited && error("IJulia is already running") + if length(args) > 0 + merge!(profile, open(JSON.parse,args[1])) + verbose && println("PROFILE = $profile") + global connection_file = args[1] + else + # generate profile and save + let port0 = 5678 + merge!(profile, Dict{String,Any}( + "ip" => "127.0.0.1", + "transport" => "tcp", + "stdin_port" => port0, + "control_port" => port0+1, + "hb_port" => port0+2, + "shell_port" => port0+3, + "iopub_port" => port0+4, + "key" => uuid4() + )) + fname = "profile-$(getpid()).json" + global connection_file = "$(pwd())/$fname" + println("connect ipython with --existing $connection_file") + open(fname, "w") do f + JSON.print(f, profile) + end + end + end + + if !isempty(profile["key"]) + signature_scheme = get(profile, "signature_scheme", "hmac-sha256") + isempty(signature_scheme) && (signature_scheme = "hmac-sha256") + sigschm = split(signature_scheme, "-") + if sigschm[1] != "hmac" || length(sigschm) != 2 + error("unrecognized signature_scheme $signature_scheme") + end + hmacstate[] = MbedTLS.MD(getfield(MbedTLS, Symbol("MD_", uppercase(sigschm[2]))), + profile["key"]) + end + + profile["ip"] = "*" + publish[] = Socket(PUB) + raw_input[] = Socket(ROUTER) + requests[] = Socket(ROUTER) + control[] = Socket(ROUTER) + heartbeat[] = Socket(ROUTER) + sep = profile["transport"]=="ipc" ? "-" : ":" + + #sleep(0.2) # wait for the sockets to be created + bind(heartbeat[], "$(profile["transport"])://$(profile["ip"])$(sep)$(profile["hb_port"])") + bind(publish[], "$(profile["transport"])://$(profile["ip"])$(sep)$(profile["iopub_port"])") + bind(control[], "$(profile["transport"])://$(profile["ip"])$(sep)$(profile["control_port"])") + bind(raw_input[], "$(profile["transport"])://$(profile["ip"])$(sep)$(profile["stdin_port"])") + bind(requests[], "$(profile["transport"])://$(profile["ip"])$(sep)$(profile["shell_port"])") + + # associate a lock with each socket so that multi-part messages + # on a given socket don't get inter-mingled between tasks. + for s in (publish[], raw_input[], requests[], control[], heartbeat[]) + socket_locks[s] = ReentrantLock() + end + + start_heartbeat(heartbeat[]) + if capture_stdout + read_stdout[], = redirect_stdout() + redirect_stdout(IJuliaStdio(stdout,"stdout")) + end + if capture_stderr + read_stderr[], = redirect_stderr() + redirect_stderr(IJuliaStdio(stderr,"stderr")) + end + redirect_stdin(IJuliaStdio(stdin,"stdin")) + minirepl[] = MiniREPL(TextDisplay(stdout)) + + logger = ConsoleLogger(Base.stderr) + Base.CoreLogging.global_logger(logger) + + send_status("starting") + global inited = true +end diff --git a/kernel-julia/container/kernel-launchers/julia/scripts/launch_ijuliakernel.jl b/kernel-julia/container/kernel-launchers/julia/scripts/launch_ijuliakernel.jl new file mode 100644 index 0000000..c9afca7 --- /dev/null +++ b/kernel-julia/container/kernel-launchers/julia/scripts/launch_ijuliakernel.jl @@ -0,0 +1,51 @@ +using IJulia + + +# Set up the kernel path +path = pathof(IJulia) +kernel_path = join(vcat(split(path, '/')[1:end-1], "kernel.jl"), '/') +@assert isfile(kernel_path) +pid = getpid() + + +# Determine the directory this script resides in +script_path = abspath(PROGRAM_FILE) +listener_file = joinpath(dirname(script_path), "server_listener.py") + + +# Set startup arguments +lower_port = 49152 +upper_port = 65535 +kernel_id = ARGS[2] +response_address = ARGS[6] +public_key = ARGS[8] + + +# Launch the server listener +svr_listener_cmd = `python3 -v -c +"import os, sys, importlib.util; +spec = importlib.util.spec_from_file_location('setup_server_listener', '$listener_file'); +gl = importlib.util.module_from_spec(spec); +spec.loader.exec_module(gl); +gl.setup_server_listener(conn_filename='/tmp/confile.json', parent_pid=$pid, + lower_port=$lower_port, upper_port=$upper_port, + response_addr='$response_address', kernel_id='$kernel_id', + public_key='$public_key')"` + +println("Starting server_listener.py") +@async Base.run(pipeline(svr_listener_cmd)) + + +# Wait for the connection file to be created +while !isfile("/tmp/confile.json") + sleep(0.2) +end +@assert isfile("/tmp/confile.json") + + +# Set ARGS for kernel.jl +empty!(ARGS) +push!(ARGS, "/tmp/confile.json") + +println("Starting kernel.jl") +include(kernel_path) diff --git a/kernel-julia/container/kernel-launchers/julia/scripts/server_listener.py b/kernel-julia/container/kernel-launchers/julia/scripts/server_listener.py new file mode 100644 index 0000000..98034c8 --- /dev/null +++ b/kernel-julia/container/kernel-launchers/julia/scripts/server_listener.py @@ -0,0 +1,301 @@ +"""A server listener for Julia.""" + +import base64 +import json +import logging +import os +import random +import socket +import uuid +from threading import Thread + +from Cryptodome.Cipher import AES, PKCS1_v1_5 +from Cryptodome.PublicKey import RSA +from Cryptodome.Random import get_random_bytes +from Cryptodome.Util.Padding import pad +from jupyter_client.connect import write_connection_file + +LAUNCHER_VERSION = 1 # Indicate to server the version of this launcher (payloads may vary) + +max_port_range_retries = int( + os.getenv("MAX_PORT_RANGE_RETRIES", os.getenv("EG_MAX_PORT_RANGE_RETRIES", "5")) +) + +log_level = os.getenv("LOG_LEVEL", os.getenv("EG_LOG_LEVEL", "10")) +log_level = int(log_level) if log_level.isdigit() else log_level + +logging.basicConfig(format="[%(levelname)1.1s %(asctime)s.%(msecs).03d %(name)s] %(message)s") + +logger = logging.getLogger("server_listener for Julia launcher") +logger.setLevel(log_level) +fileHandler = logging.FileHandler("{0}/{1}.log".format("/tmp", "server_listener.log")) +logger.addHandler(fileHandler) + + +def _encrypt(connection_info_str, public_key): + """Encrypt the connection information using a generated AES key that is then encrypted using + the public key passed from the server. Both are then returned in an encoded JSON payload. + + This code also exists in the Python kernel-launcher's launch_ipykernel.py script. + """ + aes_key = get_random_bytes(16) + cipher = AES.new(aes_key, mode=AES.MODE_ECB) + + # Encrypt the connection info using the aes_key + encrypted_connection_info = cipher.encrypt(pad(connection_info_str, 16)) + b64_connection_info = base64.b64encode(encrypted_connection_info) + + # Encrypt the aes_key using the server's public key + imported_public_key = RSA.importKey(base64.b64decode(public_key.encode())) + cipher = PKCS1_v1_5.new(key=imported_public_key) + encrypted_key = base64.b64encode(cipher.encrypt(aes_key)) + + # Compose the payload and Base64 encode it + payload = { + "version": LAUNCHER_VERSION, + "key": encrypted_key.decode(), + "conn_info": b64_connection_info.decode(), + } + b64_payload = base64.b64encode(json.dumps(payload).encode(encoding="utf-8")) + return b64_payload + + +def return_connection_info( + connection_file, response_addr, lower_port, upper_port, kernel_id, public_key, parent_pid +): + """Returns the connection information corresponding to this kernel. + + This code also exists in the Python kernel-launcher's launch_ipykernel.py script. + """ + response_parts = response_addr.split(":") + if len(response_parts) != 2: + logger.error( + f"Invalid format for response address '{response_addr}'. Assuming 'pull' mode..." + ) + return + + response_ip = response_parts[0] + try: + response_port = int(response_parts[1]) + except ValueError: + logger.error( + f"Invalid port component found in response address '{response_addr}'. " + "Assuming 'pull' mode..." + ) + return + + with open(connection_file) as fp: + cf_json = json.load(fp) + fp.close() + + # add process and process group ids into connection info + cf_json["pid"] = parent_pid + cf_json["pgid"] = os.getpgid(parent_pid) + + # prepare socket address for handling signals + comm_sock = prepare_comm_socket(lower_port, upper_port) + cf_json["comm_port"] = comm_sock.getsockname()[1] + cf_json["kernel_id"] = kernel_id + + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.connect((response_ip, response_port)) + json_content = json.dumps(cf_json).encode(encoding="utf-8") + logger.debug(f"JSON Payload '{json_content}") + payload = _encrypt(json_content, public_key) + logger.debug(f"Encrypted Payload '{payload}") + s.send(payload) + + return comm_sock + + +def prepare_comm_socket(lower_port, upper_port): + """Prepares the socket to which the server will send signal and shutdown requests. + + This code also exists in the Python kernel-launcher's launch_ipykernel.py script. + """ + sock = _select_socket(lower_port, upper_port) + logger.info( + f"Signal socket bound to host: {sock.getsockname()[0]}, port: {sock.getsockname()[1]}" + ) + sock.listen(1) + sock.settimeout(5) + return sock + + +def _select_ports(count, lower_port, upper_port): + """Select and return n random ports that are available and adhere to the given port range, if applicable. + + This code also exists in the Python kernel-launcher's launch_ipykernel.py script. + """ + ports = [] + sockets = [] + for _ in range(count): + sock = _select_socket(lower_port, upper_port) + ports.append(sock.getsockname()[1]) + sockets.append(sock) + with open("/tmp/info.txt", "a") as file: + file.write("Selected and opened port: " + str(sock.getsockname()[1]) + "\n") + with open("/tmp/info.txt", "a") as file: + file.write(f"Selected and opened {count} ports") + for sock in sockets: + with open("/tmp/info.txt", "a") as file: + file.write("Trying to close socket " + str(sock.getsockname()[1]) + "\n") + sock.close() + + with open("/tmp/info.txt", "a") as file: + file.write("Returning ports\n") + return ports + + +def _select_socket(lower_port, upper_port): + """Create and return a socket whose port is available and adheres to the given port range, if applicable. + + This code also exists in the Python kernel-launcher's launch_ipykernel.py script. + """ + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + found_port = False + retries = 0 + while not found_port: + try: + sock.bind(("0.0.0.0", _get_candidate_port(lower_port, upper_port))) # noqa + found_port = True + except Exception: + retries = retries + 1 + if retries > max_port_range_retries: + msg = "Failed to locate port within range {}..{} after {} retries!".format( + lower_port, upper_port, max_port_range_retries + ) + raise RuntimeError(msg) from None + return sock + + +def _get_candidate_port(lower_port, upper_port): + """Returns a port within the given range. If the range is zero, the zero is returned. + + This code also exists in the Python kernel-launcher's launch_ipykernel.py script. + """ + range_size = upper_port - lower_port + if range_size == 0: + return 0 + return random.randint(lower_port, upper_port) + + +def get_server_request(sock): + """Gets a request from the server and returns the corresponding dictionary. + + This code also exists in the Python kernel-launcher's launch_ipykernel.py script. + """ + conn = None + data = "" + request_info = None + try: + conn, addr = sock.accept() + while True: + buffer = conn.recv(1024).decode("utf-8") + if not buffer: # send is complete + request_info = json.loads(data) + break + data = data + buffer # append what we received until we get no more... + except Exception as e: + if type(e) is not socket.timeout: + raise e + finally: + if conn: + conn.close() + + return request_info + + +def server_listener(sock, parent_pid): + """Waits for requests from the server and processes each when received. Currently, + these will be one of a sending a signal to the corresponding kernel process (signum) or + stopping the listener and exiting the kernel (shutdown). + + This code also exists in the Python kernel-launcher's launch_ipykernel.py script. + """ + shutdown = False + while not shutdown: + request = get_server_request(sock) + if request: + signum = -1 # prevent logging poll requests since that occurs every 3 seconds + if request.get("signum") is not None: + signum = int(request.get("signum")) + os.kill(parent_pid, signum) + if request.get("shutdown") is not None: + shutdown = bool(request.get("shutdown")) + if signum != 0: + logger.info(f"server_listener got request: {request}") + + +#def setup_server_listener(conn_filename): +def setup_server_listener( + conn_filename, parent_pid, lower_port, upper_port, response_addr, kernel_id, public_key +): + with open("/tmp/info.txt", "w") as file: + file.write("Setting up server listener\n") + + """Set up the server listener.""" + #ip = "0.0.0.0" # noqa + key = str(uuid.uuid4()).encode() # convert to bytes + + #kernel_id = os.getenv('KERNEL_ID') + #port_range = os.getenv('PORT_RANGE') + response_address = response_addr #os.getenv('RESPONSE_ADDRESS') + #public_key = os.getenv('PUBLIC_KEY') + #port_range_array = port_range.split('..') + lower_port = 49152 #port_range_array[0] + upper_port = 65535 #port_range_array[1] + + #with open("/tmp/info.txt", "a") as file: + # file.write("Selecting ports\n") + ports = _select_ports(5, lower_port, upper_port) + + #with open("/tmp/info.txt", "a") as file: + # file.write(f"conn_filename: {conn_filename}\n") + # file.write(f"parent_pid: {parent_pid}\n") + # file.write(f"lower_port: {lower_port}\n") + # file.write(f"upper_port: {upper_port}\n") + # file.write(f"response_addr: {response_addr}\n") + # file.write(f"kernel_id: {kernel_id}\n") + # file.write(f"public_key: {public_key}\n") + + write_connection_file( + fname=conn_filename, + ip=response_address, + key=key, + shell_port=ports[0], + iopub_port=ports[1], + stdin_port=ports[2], + hb_port=ports[3], + control_port=ports[4], + ) + print(f"Connection file written to: {conn_filename}") + if response_address: + comm_socket = return_connection_info( + conn_filename, + response_address, + int(lower_port), + int(upper_port), + kernel_id, + public_key, + int(parent_pid) + ) + if comm_socket: + print(f"Server listener started for kernel {kernel_id}") + + if comm_socket: # socket in use, start server listener thread + server_listener_thread = Thread( + target=server_listener, + args=( + comm_socket, + int(parent_pid) + ), + ) + server_listener_thread.start() + + return + + +__all__ = [ + "setup_server_listener", +] diff --git a/kernel-julia/gateway/kernelspec-image/Dockerfile b/kernel-julia/gateway/kernelspec-image/Dockerfile new file mode 100644 index 0000000..2f12772 --- /dev/null +++ b/kernel-julia/gateway/kernelspec-image/Dockerfile @@ -0,0 +1,7 @@ +FROM busybox:stable + +RUN mkdir --parents /kernels/julia_kubernetes + +COPY ../julia_kubernetes /kernels/julia_kubernetes + +CMD ["sh"] diff --git a/kernel-julia/gateway/kernelspec-image/julia_kubernetes/kernel.json b/kernel-julia/gateway/kernelspec-image/julia_kubernetes/kernel.json new file mode 100644 index 0000000..662a7d9 --- /dev/null +++ b/kernel-julia/gateway/kernelspec-image/julia_kubernetes/kernel.json @@ -0,0 +1,26 @@ +{ + "language": "julia", + "display_name": "Julia on Kubernetes", + "metadata": { + "process_proxy": { + "class_name": "enterprise_gateway.services.processproxies.k8s.KubernetesProcessProxy", + "config": { + "image_name": "ghcr.io/eodcgmbh/julia-jeg-kernel:beta" + } + }, + "debugger": true + }, + "env": {}, + "argv": [ + "python", + "/usr/local/share/jupyter/kernels/julia_kubernetes/scripts/launch_kubernetes.py", + "--RemoteProcessProxy.kernel-id", + "{kernel_id}", + "--RemoteProcessProxy.port-range", + "{port_range}", + "--RemoteProcessProxy.response-address", + "{response_address}", + "--RemoteProcessProxy.public-key", + "{public_key}" + ] + } diff --git a/kernel-julia/gateway/kernelspec-image/julia_kubernetes/logo-64x64.png b/kernel-julia/gateway/kernelspec-image/julia_kubernetes/logo-64x64.png new file mode 100644 index 0000000000000000000000000000000000000000..2aca843c152f89e87276f4c477d63d6fa565f8d9 GIT binary patch literal 31744 zcmcG$c{J7E_c(m+HDsPLB{HQ<z0aQae!cG)8R{}K@G&5Sn2#UR zGDe6x7yen$(ZUnM!pb;2+Y>@AkjuEs6Lx(6`1WY2 zj%>@G8`G^j=sAt$P0AeAjq?UM)oxz!@AT_m{FO4EscC;CyJWF*Z{zi*5QGv46OC4K zzcpujbNKu=JJzOq44hQ8pRO+l>7L=A40>L+kY5r0BO*z#Vy@_*Xm~@p8mIP)#gSH> zHZ^TTe6X7xp=+`g2NYH0JkJXcC!9!sp%@|i7Zg1D zG*~GRCGaj#t-geQhyEmKv)*+i4 z5u_QU5s~<`DeI6DL&A)0os*el%$}U`pQ-jwVN@JOd+S+TsUuw^s#VyXPKlK1J)=QM z$#_OIa<){iEi%D6GB(g9jO$gL$)vWSi!ztZ%inISOrhJ*N-7`{b#3swew*=&X~Xb$ z4xwDZaf+T*!B(6$f$L#=SqE^4UmHlWR{!C|jqh#adA2`1s)B2mXKz1ID9uNthZqLy zFK!f1cND~apE6FNxkGSO_E3DarmM@0BLVd(1IA}gals=Ge{0U^-fvlVv$YU$Mm!f0 zZ;EH79a70h!~+0@JTzoJ_j;}lX1=x8Jz=)rrlqTD{g=E zXcUMV#-1+0uC%W-50S$;EszloyLqJVo$#0HszmLizj(tG)#EpMq?ju-tZYunl{0Hmx*U%M^=I2FM* zTq-TEEYWkiWL!-NmC9g%Q(3 z?2*+9#XeY?GAVwlfy<8z$n z`||Sg+3Ehltf%s*cMUrQGFSEc6U;e!w3_R$reQIUwtwFHx6{!u><`24Eas}^SF^rd z%+yMGDNm{#Y)I-~SZV&00W5q88>f9Z^_KhPOItCFgd-=dzZAbKe?NEO1UGU&1Qb-F zq3XCXD>ov54E2FG0uBr_@N&++LA_V8#W%f%n{87*>4m=)VkYL!I5bwDsA0hM`d|xh z_TQwW4cy)q9c1B^$%a_k*l?(9tj^IbfH3=MuBc9m$Z1?vaM2(re{npT7V)oSQzL8l zQX!0)i+MIIafgeP(UUnTH*ox{zaKL~3;7%5url|n|kUUoQw1c3h0F{qLi^2xMQ zKbkuQ_v2Jl=RU9gp%-U&o^wJ*=FIHypRG&z6ylRsh9A2$wL?)M?tQ!8&~Yl#%KIrq zc3`0`s0ftNwu+%8Sk82Of2GnUAVE!b#bAePOi)t3NGv`V&))f$Mg57ACY}0aekSK+ zUx2m8vd+nC%JS#?UuS@31=lO?AX3I-n@5k!eXgtoYMqItrJVE_a2VA>zHN%!DY~;!=*(^C0m!YzGGvJ zGe^VGiGEGZAbLkZ-b3(=B)yB?X9N}aOhOB&m?$5tGOy+Y6cvcTR@0LlIArh8%|*$5 z6hX5k(c)QNj@dh$R^E#Sqe{^65nb<`_H!e9P@Lr0g|Gsf+hI{m_(&a$)vp8ujDAj( zN8&r>a*_BG*wM~eGT=oSp7%*`v4u%!DV32P@S(Rtpmiah@2fyT6HB-S;cThd#}&G zUOebpha;T_)kU~!K2|bkYZ$u zBUk~!LN%|`5S(Hy5Ys)ly&ERhI0H#BZdCbAt^C3PruD!fz?9B+Jt8CC|$6y%%6|qoW=*O45cW z(Z(~fc=hDwk2yiJud}A!FTIyyM225jq>ogS0@p2UwuPHo5qDNQ+^~!fV(T|Wg zaU_-+If7y=e*gB3B>k3wCA(otjgRN|d9d7kD5+-BHE{J?7IO_fZX^P3a84VbsjF%2 zuc|Efj@XSx!UQZ((#1e~*M;BJqi!O`CVYrV4;ZYv@z*~&>Q#c9bkF>cM!|TTsbeF% zTRtFY>YOMuiURkf*FuiRrsAQ9@^2i)XuRBxB~TnkdI2<2YHlgb@{{xbL3QnbB02yPtjp z97hD_|6zt+d$^>)p_wPGn*ExGI|VZtkzH=- z$Zq{uF&pczC(1=V_`lyvHKx+Pcj#u(H^jJ#GBX=8p~wYvz|?$SWClV^TQF|T?TZ7;(7``fhfoGN>DNfiI>$JpDl{TEudJwOQw!>Z_-<0KclW{&~RrD7y< z>{=KCpe94-aR#-Dcj?erRzQ(GOi_qW#9-Ir$U`Un*wGL3EjK4U>;WQAhTAT#Y?&Gk zG!M!&Eq#lR6kk?cyUIp7ejl8VH?v@rTVKkfu^M(wQetB9Qc1BnGKehqw$}T;xEp75 zl0llpS2ELnRxstZ3uuG9li*slV*e3fs?WwLL92gy)+7TKs~26GQQDV$0n!=w zt86H+@suWM#r+l=c@Mt;HK}6t=apDC@NK40HeBy%4A|%+7WGu+Lv*Od69!kX zK_zU^$He4~2o1>@^pEM!xRWyHB928oK&1YZYN_P%`5XL5a&0#pT;YE>q{|$+=OuZ_ zgg+^2dsuid6=!q=M9%aF<`seJ7-`y2kt?4R7gj%=+%JOCp8gNpp;s}plcj(3Pj>3j z8OdU6PB|~p>_Ck(3@7gT%}RNC5vWWf@r-2KEd|+pb2sXhHuiLI=ur12WgIEjMn(dd zd&&2`Cn10a4adx~y77&$7ztpYj{A1?dLiB_DTNa~F?jWBz$=orLGFH43da#e2t&qt z#+oLLj)q&^zO|jUhzit7C|`FLio5O*fyirHoP<3lr#gTSbBqOX`PjlhC+2xOb|-AK zj!PllvLp|;)f^Q!RqO7t<5c9baSC3%7(8KCP_+bpzp0-QJg)@AsKr-j&rqYmmdSbr z6W{f-a02&<$)O^_u8t8sJUIzll|NTva|=hLe);mF3Juts#f4A zRlqOSRPQcpDnc*(mk*AA#EVM+9`xsy=b8&{zZIuPyiYito^?5O-KSPx2F&Z0++CMw zc+n@})Lbe)a%9gjfc`MpCeXx$0})u#uVdQmK9zbgHn;H7!WS77xp#8Y2jV1|Cg5qFzeKqk54ZZGU0+bFCQz$(QVNN0E6qiE z3s&4X%F=&%Wun{Gk`EcNC=Yfo*m!sQmzA}|&{BNbZ_pUw<>bB#f{i3wVd$^|I?7K! zO)=t7-E!1HMn*EzJv;O%15nCcWk7YG_e*a%`;mU<$A>hK_^Nr#ISXVL#7RXsIAQoF z^zk+vxzpG%Kc`~4gZVg$1Y<}{Vzey<_C~`Dli2m0I@Xv_T5fuJE+ZrHN6#q{xU2sv zl@rPhdQ>PwYFMz_%BG_umYQ<$=M5Ty=6^}U^3tsq5i$Q=G}d_Qlo2u#HFP+gm7g@k z-7MK%h9{0AP>@Luzn;VcJw&n7w@|ee{6a=D7|FgJ@xbwxfS2~D`A?4?9*x=mug0mg z-^NW5PjRaFidSboaiTqhhQI&>Id$y34lVjID;8uIXy3?OiH^Pmpn2f1i4I;k&xjP} zS9azsKVu-)!m*CV|2tOyy^JJ&QEJ|Wd^2iP`5o9^wE6jNL~b;m1Qo>A<0rf}dx^+j}Dcy5q39DIy0+oK-c^A1BdXBd!DQBsx!GK5@>dr8g9XRi^ZI5iaZ zezn4}9Y>MieoXt03I$u3B&5^3O9BQawV!RbBi>sV%oQmTK%SK!m<2J+e}E8eAF!w3 zC$s`?$wiO?z>}+uXkEpDys>;~V3V69A-gliyDu_^95TY@O z0>He_eMa7hn9Jm`Z3KKiGAnlUtphmRS5XfF`~wmm$x;Ns%EgAiw`5?QTqkqWi)ZYj@^h_g2#ZI>yFfy3^A-1R6w zml}n>YkfBmc7U`Uurj==0SOAG+5Gx5RJ|#CQ|XYQzv;?p&aD&+<+zPKF!0lzbU2c* z%)y}$c=R2-$eQt4)N|P5}-=b0{S*TIIy=O9FLAFR2*Xz_r9IQXgTlTA7{w;SYC4JPBu+$P&py%}ji>a%h^|XGc3v#81fPGClsh%)P^9sh zSlquU`-Ce`gZ4xAp#I)}rlY1BN>a2|^FhV7$SM*e|#o7C)tf3{;&s zgz9GQ?!lI!Is-Ty-T33Ozio@`Ahucz5h;%W(YRLk)l7}9mnq=iNd`C<%PJ4|I49s^=#fiN)C+7=L4Lugkfa; z!h)O%a4$wP9Ycpx;A~F&T>nx+)k_B`*ro6X($k>O{b9@Is0Pd3qe#neaL#&pvC#p3 z8$A$5T)yIe1uZ|xPM-nk!e#Nlwur$Y0v_XYOch2xznGwlLy@*E82e!I#0x3JckFbC zw`5|%v6UayV46dGRU1@7M$MPAUIA2F3l9K&30m#-C;q9CGsZeUn^_Ulm8@v+Rh2OH zK5%-_<`zOsZduWSj9ckV(sr)Gc`9@?#`AoUB847^yw}%(7U0NBgg}B}?-40b1bMH$ zH-+`#{GmDBtByVPXfH&NWEt0OcodrVDfjzMGAkgtVPORt_MzybAeyg`7gAS?_`-h{ z%1WWRweD2H>Vg0};L3JkMDTGHq8IDmG2vw^TYp4 zsKymo76i`9h)G=^k1-;WIz5=a$^hsJ{rZ0d{n;V1rbPSiS%kq+Gk~>!|3~IqD8Lv~ z==V<};{EWC;Ay_EsL?`S)q*>5Bn?pF_4?m(L$S?K&@LF>QJ8Kyjp9zLi;tuMa%kI7$ zuroRlOqrqRtIo5V>z`!AaPEh&W8Sqf#F4&#@v^}sK@PH4+NEg=d8K(i5w2D?pEYJUFwMHPgrpS*qMUl~JCz*Zhf4~E+7li`BazPrj|}~A*D}^iA-qAJ2J#-Htg*mIgyWcROh*Wp#T=g(UOlv zLA5fcPGa*E0~!YN!%H$$9RQN z?~)LO0#u`|g{_461*v(JU@8J<6Dg_E5uj>7(vYzPRYm%2&4@j5$3S}m=sl17gHbSc z&|F}ahTCO(=uyI9Wsf%4-~;FaCcEjxb?}U~7t<2-5K$bCe>AteydO3-5PAWA0w@Rq z^%gy@p9Z&F87gYQr|^gglt1xU`12(z21L;CSeV?#`v9(?uh_R}{xVb(Ahy}}KU_uN z2d-lBl-+rrg%%l*V#{Z7q2R0$46YA@c&e{YOon=^QrG~&&=KN&7wZ^m$O1)m25jZz zNPU>o?RV;N|2<=pxVcRMdk`)F=Fyq1cBko( zks1`&fT;r0@Vua^DB`?BEU4guj|$}?+@%f?SWGVLPXUf>J(CE5&KNac) z(Ab=-_bEn7E!1-(8zBqnCTa|!y3u8boL8n41Ho(mh9$Lbg~v90Mx--kKN{dvVz3F_ z`ro#StfAO0Q)ISJyRv zHX>$oL<@m?c0^Bn#FO;uT(DZsAjGsV_lk6YXJ5x`BAo zEi`wdPhhr;LaX0Ey*imf3jYpAvT$f-22`txP3~BvcIw{(a2TB_# z$m#y&W6pvG0aNwkz>08Gf~GPLc35yi$SJ!ZOQbV!8YYVm6kzpA4wEC4yL6>i zL3&VCGxETs>lE(@3#z;?+S47>hBtCPPK{dJ=UxAt-%5Kh>2n!{*-*>*+bZfb1yhCZk4Hla8@;l3S5fD}sHIWu%6o_r zITGQ40?4T32vo0Jqt@O)I#7Nl`@Gm%3;Ax ztaG1qCobXG)+6vx6k7i0-~HQD=OWqz6=m-Q=WPefJyoMbBH3K@J*`8Xc z*8PG6z0ozlsk~Pwz2n7j1my>GXpcVDoAGH*ajOn$q_FCl^R&7YTgHtoi@Xk{(nLtR zZ-+RHM1NryH)^>s2~{x6I*G-?MO!c0zx_Pa`E?2V1mLyD!57eG-}CR$T1SMt?Z~ zc#egQZ{CoDCeq0|_MfUa2U;taU`Or=SQ&2hPtaP+rSPk#p)O`)eJ*I~$!y`yZ8)Vnz`{uvH!DRjzF4IoqR>LO1QeUkU8hl30|e*nfsXk; z+*6WokWx;$_vdbvL0}HuA5hu@6yHw-TolJf%FcG0TI1aLAX+rE-wb|n+x|Fm=Y|TJ z&1TPZgN@^Gti+Lxe&M4?!OMDrJ)+3n?4pNHZtuL6h#Z!y%k z8SgSq69(m_m`P($;C_DF9tiUS;!&bB5T@mU$s||P?=Ib3Lb~b1~?l=JFq0@Mfu6c5j5_q%7fP3`&_sBy;5-MT%hmL zV*j;01a380eE$+%TJ72?KgB?je8d6W7&D+^N7o+*ZRTvwy5yW#{{^M`q`KMb=53o_ zHdlJ;OQA&r=ts_jBG=v6W~l5XWn!X?-9M-jtc%KYt7|1a{heTRVl%MpzjCa=4r+28 zp9pk{EU4Ckyk1w)<<(;P8(vnp-u6Th3f-npfRlcFifig#AxH0&dVp@%D)(61bSxd? zKtCWlCY`yG!Y1T4Ir-^YX*NpRj?p2GN0&i#CPSWdJVtca_z@}qvt^rpv|^<|eUrO> zTPpZkJl1#h1*lZ#2fYbE-7GA19{X}iJ4M?GU5kYdzbPvlEOMbm*C4tgia-tep0S>w z9zJb*a0f1d6U2MZr~hj=j_G5K31q+x61Ba?RhnGqF>tRad9pKbt4M@mG_hu$yq%2D z0QAW7iYL}SB8?K{F))c;7V2C-uhCFa;JOIY&H4ewpVXk~tP< z?tKTHiv$HOsVFXdu){GGGzU&O3UKy5qxhU1dpHaohl9066&vfxs`0cash(XEjePgE z@$czUGxIq=78VPqXxyW_;RBL4zd2Lk@6yJF<~~ElQYjeYz|r|Q&MPPQ)Pp+r(``kx zB+rZ9`}gTMXM#)Gj94BjdhT~k^rS(K5RQ>>(AcAB9L@@H1)uuiWKS+_FlLuCq5g!0 zpor?nXLrs1fuWqB^+MWGc6QLxge9$;Rcv-way{ z5FI@L1Af!L=k&BngqhPy%MxcBFH?2kh)O_l!iE3z19Dj!>4rlAl?JXm^0o-$7_`-) z7mzz3v721bh?B@hL@dom*n`b)o?DIn9eRGs$O~HnzAuV=be8d15`+ojzqc1L;3-Ss z7m{~B<*{u0hmr_k|8=?Hr5A9hIxMd$2I#n^a8^JqyP5Q@Ho7#rKW0Al?rA5qVI!^=g@@@#PO!0x;K0fI0m=l$XK@3 zNH&V+W4rPmx-B=96khW0o~MR6+s5l|zWCQ-+10f)@UNSb?2E0rBGYAk(VV+%pxX5{ z)}-mWoqDU{u6}pCsZQ_v)T`SrSR>`0yX|HO>mWdB>04b|7v%&0+!l@z&f(UcUjA>t zq#8C}%%jUBn%c8#y#A(dEjz2oaFYN#HE9I|fTi-cnL8@Y%BCTDPvB|sF}qk6;D(-i1hP31yZsyNUAYp|Jblab~_P2&GBdM>WI z*%#*M;sA?eU>8(_A^!DZRl%|8Ur<65VbY80yvv`=qkY&1i=)|5FV=lac9UVg5wur? z8~9FgPR32A>P+Kw>SHR%_kg`bjtetFOUJH%yK>YT(=b~NxI>Ok4f~}~Zzd3qLcuy% zkMYddioUw8@^;Uu?)vBw1}Q}RiVc3nnkjCNRv(j17U33Yer`Hu7ZiHB>db)8c%eEX zCSY@p6{tMq_7vSG?~&NV&PZEneiZ9#?h+ZSNeAZ;?%WuMbg(Yo$gT)usX=+iS7ti~ z3U2>2HG_6Hpc>K}?Oo~%anQ1pO-+cn{nFOuD@%ArlAY9ZiwK{>QC=Zr1XU|UHr}$B z4*R9GE*6Uh7LjFDAy>ghg-MGLF-NNe|oU@3L~;498oyHd2gZ1Fa3xK!^^MELk`#QyOR` zEHFOiyIi5FThHR#yEgk_q&#`ET+mxzU~PWZc&ysyu`ZT-hmY8I=<0-zJvamK=RdW8 z{|`xZY;o%UpNo04(ENgXVBmty`fH1&(J6Uun>7jT6>USAO~0niz^IU<=5gX?%s|1> zjp3`F0~KXUf1Y2wy7R7MdAXXJANS+Uu9f~1o78Zfj)Q95Zq~Oi?vn1=Uw_bey8n!< zZa>f6SQ|@~A%4baTX&Zl{_@!8tBXOUwds$aFF8%{R^R*c#qrm-a~o^EO==oxIlJ*` zFOzvpWc1QM`KKIy>`^}y%k4Z%t}A|QGAHuNT5DR#TC#Lw(B}E-WPc5}6hbu?%MHg5jFM~T#PQ-eIh(7?A#XQb z%r@TcC?(%oY;3Jd#&eQX)_M(`S4_Oc|-Ht?RFg?$as9lIb{X$ zCjD%l(vooTRzB@H*Y)JKEP-d4J^mu3X}>wOdYPfaEd~;bqN(1-Q4WUb5|404Mm)Of zB@T)X%Xk_-mY8t0-l*7>bDS2XJ>TqC>$bU6Why74XrebYaBMXwzMf)s@391Q2rqvr zSL1n8x}3Z*d(B0ccX87(9O(>iIyEWoyvxXVSLY4ew}g|?Je;TP{Ly>UOI+G7&T=-D z47uXarkY<+5r50I$%sw6dRID2_JX(By)rf$jJ9O;QpskTpI^5XdT=XsRdHRVIcP3q zWwB{eYk5IXzL)X7alsAzB_o>~1v$j|{jprqC4Osq%!Sw6&Q{bvY#Nrf*NuOwoVq;c z9W@4@yIUkl}~rsj7yXA?z)bhCLDSA~(gg-jvNVOF?Ghwi+!#p#U6E4ju}sehX|{dN`#e!L%; zABz^IingQtWw!og&D*mX#&6V~1`qb1FK@fIxt6>+nyo_5j(n0o(32`&H`VZNaBe#v z_iT)7t+!m&B9&|9kD=x*s?A+Ezv+wFvVun^{OQhjOOVRuYfQ;BsBCwxt+|A23R|nm zNOz$ouIWs^@R$8% zyoI~wqQ)14mwhV)U2A@vtynu}GDSDFaRm4A%jFwEOGhU(xxd4id6=vmF9uz-Y5b}u z(KR{m`zQ1xmHPBPnL>xC&<=mPzOo%=Os5&1uvKQ=|EN}!Jn!1>N++XcE`dAkwP|4Y z(mLl{Ze94BjlL63q9Ip*jX`sNyp5S<$o(fp`|isIUKGr1^xfrGl;nP_I*5mS20OgN zn}vsLA8Gc|9GF)7C@~=~Tx2}bW#!T)~G+`*RtyuLw z%becXl|yZM<7>F)nucNecFmHOKaNFBotx9~;fQjji@z#gv6NCGcKB`T_KjSoBPf2by^p-B{q#%KW4h;C;J8|%0chH<6wU4 zrMj`8JHqEU&zyQbc`89$dFqlFd8C!^?r3^?M>X$98{^1OC*w)A4vEVz?{9QZEVvzN zU;`GY9=*|C&6Z!0;G{j8`Yzgdr4;u!_ILh$4kJ_6#*Jys_>hWQAJk8zUC_;Yy3#kb zxne%ms#!YWV~>394bEnbzNm5Lv1JN6F06DtF}yIU&y22bVX)<-nJC)2+*k-4Cwr@r8(dCJ%0!rlJ6N^H)?X+Kk4 z6vt`*{oDCDLY6UzUf9=;R&eFT(`FsDkesYKJUJuK@-mm^+S%Kmedw3mg>s)7FO{CQ zq|EbBTSXziVo1xJ-c_N{#K*w%{H7M;|t?kl*>JfQf-fF?MhkJQlFVE++ zThpmWm;c<8aPGod)Z3^Rvl*iVs~C=c5cZqD=97J|EObpa^a}{Kc5;y=va1 zdhx6Xi~4zKj4eNPJS?RD$lGpx%v!ZZ92fKVua(`wi1r@i`u!QS25*InxilC5n0WvC z6OL22K4){O!SUIGqUavtZ`V-ksXt$O6dEk-$EhtMTV51@^1odm^JEX*^_bSx@}>Dl zt7?hYkh<#@Hx)>)H~34{KKziB`J;5D+WEV_(WmP@A4cf2$wI6lu?JTrtQ7tVkdB`L z8pu{Hr7oM@pY8re@&4pj@nPifVegseAI6O+VZlO-BziV0jq{~74fmT{zPwHz6WGyp z?j?2P&0Yq*nfq)Pa0J?$QsO5T#cOS^dOzeK7{IU5O3nK;6gs+W^4qQ#LJ@sEr}ZcV zOjp$yoi&tg%%F{3dL~?pC(==y8?j5c&K>psoa-GrW2r^2#8lb->^bYb*n=h`=MG&f z%G!Y=Hf75c4&_@^2*0}-J|AVc?}E}I`*_c~+}%ej(H+bmxJ=KTVNLfw4LgQe0lnI6 zC%$mr>}Hpg4$a;096NXB%w^XM+HLQGh*SSK(b48*!Z}msUUP}gF|Sg6N5QuWqG-+e z`WppL`5*B|nRra8e5xBJew0eq+Nx1ojDJ4!LiUJ4+_^i#8~du2vP384c7Xaak~X=ak9n~-MA~;g%Gj5+z|B&IT+VIx<_Ng@@4q!cinsT=9-Z<8MK7X_tVw>nO|NHR~Rz|3R7%*2faS* zl+e+Cf8*PctHvGU^Zm`hQr*VPq~9eQ%ZogD($c^7(?!1X>l}YIDfWwj9xsjQmVT4= zyos9>i&Xd5zb5v#o6#9b7c%0vdw8z)l#9LMGJA|(e@xm>XE2A8SIQy;1dL5?*RRAe zP)2W3v{Jtwm#|v@Q0FqZ6#UD1zwKU{CIl`B*2*%Z=qdXen=(6GnFEsa)v zV@q9FuIt?X{P_oEz^{g_HOR1BKeWtzro%++-jJa`m#xFsJ@+fN8SAw@b)C=Vn_;=n zx`lOAOGTvYxl_zS-o_=)wSLvAGU6HvhYzX_>rU(&rEPQnzb-ceJ4G3hC@oDSm`{QxTD(d6iw18_m4tGdJ?Xe!d$-}4;i$j zF}s7w=I~5kL$$d? zo6Lg(ET#iZNYBVBI7?o3)mV&Fr&?uPvT*ga_qAtT*sfInD>`{kXw$M|v#qU?3LM|7+N_HtpcxV z%uT0XQm(GA$CE1c`dVAf79SK_s9Q1*ofCN7MJLO9jLF?cv1qgFY+foMh}uGVR-}yY z(hhleCDVenH1^<1y7%Z@d7*uudisAPDD=8HIvV@?(?b5_O*SdVBjp?eVem-QzZc$^ zijBm+Uf6FonQ$X+J4W4$@Gcdd-+h~5+b8nBka(G!YK3QYnG}iLSDDAfemd%#YH{&y zgHBe@y$?+yRgG|wfBGJ-D5||*9eALJwBODQt0-#HVb722O?TLst63V~puzH zCLQCitiQW^3r*2hcDT%&(K+cTpgmRYi=P)J^$3CRM%ciJMuPFHWM4*2X`vkJBzcRo za#IXK+nBtQd5?D1jDNGTv-Y96{{p*%gqCCwrsl&tguH1hgIx;tev~35{ap4-YH7tR z50_mTwS}Ghk~`-g7&yEjVp-OAOH=6N`{lL9IJQdWI8=MKpZ8_H#aU0yzIR_5Etpg} ztKVgEAmt#!p=WHhxV_AH&ZA>tsTF~%qY81nN0YOuUo2L3-q|VPz{f^#LR|KRymcA>~qWF z#bi?)QGQFR`wPtIz7Z8k&sBE=yaYO^png>W3piWh*8oCX-hXlIOKz3|-}jnVw0oJI z)lv8cnbQ_0x(D`&=2R?9zQ%LQLUiOy9P!;Pb0p?(@$zl^$?~C5zX|3R6zYU|muW3$ zg;RF4AaQ!(){DL#V!XThxYX|mCP-XcNiPzr!iUE+YrTc%xPw z`Q(6Z2Ns2{9EvC?2pi_xXrv0mb}deDxVP(d4)aF0T+#!{4}BB<3GK^9`Llg+mv!9{ z7I$d)qy93GHH}AZ1-kXkcmoc?mN%{)18Cv%LT0wOFrhMhTEdfxMau@gOkp;nz)hH* zc*}yK0oY6G-B#elXHs!k<4hm3G5%zQvsTl;VL3s%hCWK6oA5}qj|)+0W*&{d7f>AZ zQr0uY+D`;tA8QLs|1e4M0~kILA11fi(zL2(0^M}S3gZ7VklK9Nq~4Ic+2Ex#!{9iJ z`JG^1p0chAQ952=PUF%K;x97mUR0)io2E?9hlaAPTHbHYBH+aBf_o1nK1>2_$IE$+ zKY?4l$K5dz<*cqS;m@n=o4F7Jz|W<=d$XQdntQc0-Mce{%c2yv$*tJ6o8{?Fl_u5} z(%5x&>lUow;+9#}A%|<-|1yfz!5VnyXa1AnP&USl;iA8iqXM{}r3nr}i>qh!g;fnm z^EkEoC-2xblg#PSg8prq77R(FayzSlh&CdG<9QxVa(;CsM7t4{q%x)-QR<_ z)}6bas$7??8o8_toou#V)89Jj@6yG7{ldBf$R4`)lZ-j{luu!qCAZj8!W*P4h2k2pNJ!Q@cO^I{eeLr={)bb*C6wLeIQ=1Oo-w;+E@GuQ}+@&m0mdBCqK?9 zT`MZjW;43f#&dR7rTm2+B9y$By{cJJJIg3LB7oz~;ym=L$yab+m#bHXb$DG9#dn9bCA`k(<9na?RlbGG#Ni-Q zs&L0;f0si)#z09UJNC-CZ+7zqA7)?umJ}qOIML#k!Ygo{iSjdr*D>m3OY1EQBWV@E zU0$kMx9&TbQjy-=aatI8b7s(Szu6(m%K6CwkK27WduY)oa4G%#B|XRXan^q8X^Zz( z)k=fcK6xrb;?#Q5uiif0j{fu&$MBd$ny*17KtjTSN3h|^lAf5*CU}lhYuSgI`ANr1 zl$3Bq&n1<&jIEEAw}K$uf4!^f^R4^( zhWyk*4{Z}v`Npy`o)}o9qd!Zgh2yYzM!Z@{?pj%^F@5d5p7p?C@8&zfXu!xIKe$p` zGUWiS^+j=vxp6&HH?>gHliIc3TUNy1?=9B?l|u+Q9BANYH5e9?N0pL?ni4cAW?D+!k?}ev;LI<~^?aPK|yej34i~P%bCtCs!EHaQj9)`kq*c}X+kH??o zuV*fPYo2g;Rj=kSTEw<1IU@^%MBJ2h#x`x0`DoYqGTKV_{1s-n$q)|gCI@@2nT?@8 z;FN_HQW+gnY061YFWF~6(9ylu;(9A#E(&p;G{^^E&1TFpBb7N9b3sBl`?_(_tfu; zyLwFNd86X0Rd|jvvFIamENJb&)y|&f(rPE2@R?iM%XDJz*=;!S*9peM*NeP=j|VTj zbf)xe1+JkTF>Jxz>?9Ap|0OSznxRu6n;~#NXiM>HtQ`Gw(Y_6tyLfMY@Mi&i?Ti#oKMtsrftWeoy3R8aTYf8wJW*Ee*}8nAc6e;OwmQ zyxT6_+M4+u%8$n0eLuPuN;D}OSZp^n7bS^Pzqaz}EB$)+$HA$@o!T3+3>A2(2n9HH zS`x3bVk;kWliJTP%)gUvWxl+P?0mQXYHj%Dn$%U&{Kd&Ca|xX<6)=^vzkWKs-jB#>8Gwt=PaE6=j7_?Y4b7MMHC@14{{8JRtd4RVV?WvLv0&$p>21C+dLY{>eUtl`NM zk4C8k4=8CKrqz%^oIXjrM5J?mHD|8m#cNVqK=foyFWgvKfO;#9 zBjrp?+PUJ$by2&1Eejlz13;y7QNr~>v&&C3=c0;xG<_g_Hgv3fVR!Y&aH|$vx<&_ud1WSOREFJuXwayEs9U;DtvEU57#$LE#q% zhB_=~+cf+=fs{s$s)pKAb)|>Hr9biO8j}JWHkB|%W#qAwxD|PYf5WDcIs3tluHWV> z**7xf@e0c#V4!dTvY>Gv+e6ZX-SO7NIRMIfK2OnR~Xz zA6Cqm@5NS(u=-PVNeE+KP0Q^zXiJ7ADZcXFGvUxAM(~&T1Ub6#a6O|vSbUBFioSYJ zuE#%cuX(whYv{Y2EyvAS?Ge08`r>;)_xcDXntDc8zeHp`dhuo<22OZ8*~c( zul$%Wx4nu|xi4SlNrBPj%@Uoj&vnu3OT+d$11dz>Z1aFxk0DDZ*no7wbiZI))PKK( z@zrEu8`sU}|3cr(C&UnOuFz!a1t_K}^S{HkH1S0r5JAT*+sIvEk$@skSeqR==}!Yn z9WqhZ+`b}W{Pi-_P=AieK3 zUOU$QLpdYrpA_)C-(?_A;mdr_a0T9Ug~kipr0YJD1KM%qV`kh_R!qVTU>75cy_wE$ ztk}9dUvAK%%H(zKCgUh6i6q|4n=dVFuQuT+DZVvEHc!Wcf;6|>kZPqr-0;HPG1*3N zU8dvojHsm|iYoF}7|6uz=++|j2qt}pSE~ip7U&89eR2!5;=TTJAqs{6Kq~_1?HK6Au6_&i^LO|C z11)oAS~GA`XHf(VY+O0=9)QztSuPrE>Lx_;Tahz|L38ir>=tumbZhd;(8_R+4R1}5 zlQhoEOdrUru3@3jP<5jBf+*@GI=A~ANRIxGlvZ7T$ce&*tCj4yxMOB)<8|^p1W|ai zN+=-+#Ac?`kx43d%*+O-{^5eGPoSKz1AOg+>)E+Hs}eO6(esb@HV2IH>bTU&kLjfo z*FuO^|B`)YP3V#M(}^Fe7o7@rxyId(DAvX~Xo6s}Ko2VqX0&UArpA>IY< z5qx2iZGK|nl^yeE&mtmI(=1%-sep-S8VlK~ig#C$(begXXI}J$aEccAzgG}2#>~6X zT&wI(OK$lG|MktH)M!aJ^)VISvOk0R#nl1tf(T(+4GF|qv87++9&}&XjWesiiBXwD zmO8j#$Hb9%?L}S`J6OC+H-?`12*npN2s<^Cyg$QhJD`9e^ZspNzs_&$KInw8kj{1PJ`@CUNaxr8D6<&`(Jt(3R071MpdT* zuP$i#nTYYmZ>2CSR*#Dz@&)6OOKN`t-d-(j-_iPze~ckhZS9$th7a@`Urn*juS@Lf zHhDJr6i+%3etu=-O^1=L5t9O??zwOdGO{%ky--_z+2bTm{d)4;(T3Zh+{7c0O*-#t%teABlq`PF?Li~x z%RxN6^v|0~?jtogW)^!RvTdlWs*s95mCH^XGCf&FrsY4zFe!l2tp4~cJxmT`$i+%P zFM0Rjwvob*%NnXd-azuwg1!B8V>MfFa1bK+)C9*QI%1$LCBxw*?W!w}Ev+RABkw~djlnB|INVbq!vda#UO=jkM zTzm0{TK&dYgcSL`$y(l(Mb zHnf&WPc+xU>|m}lpudrVkB_k3C8ILjalHgzY@}_ zHt()ux$@KPX^x^xrHnbDVWo~)NV&$+=(vNDL(n1r0!F;@uOI4lT;CO`5#4?Zj-Pk& zVm(wilG%6lIy)O6!BR5hgw84BUQGL;iY)ISTmPxA3cz33@5J#F_kqw2#Wq^^K<4!y z@Uvu;)I+_(N!P zhbNJP*FMi1*c@65g|jv{&2XKVj>j_mP=yTHuRY#Q_N=e5o+x+JLTL7>mA<0&mzlBc zoGe)5`Q<`VXzuW5x;gz>w+h`rMEY~q$lGjRo6iS=M+hYZ`@r3iU_Z^>Rs7w+*?op7^3#Fsw2zxS@@e{m8=sQ8CtdG-nt6EQKYu%&g6ds>7A03@a(4;bp)nTi5AmpYt&00r!6Ut1m^mCGGFn8Gm3{Bl4SGI3 zJ=r0Kqe9F4A>&%9+JCLu6%d-ru}z*8cI=+qqk>ISX#ubPz$Ft}tfc*$VJ_(;6EaLo zKVEYq5Qnl4g3bWe6p#DCmq8|QSV(x_(f8L8R#Xept~u7ECGPEuAK?7>ZpOXadhLF& zW~Ktl)>;~vj#ijvCw53-q~y)uwfm{2?ih2JX_3)0@~$OTak(Hrf0{#TV?K1ROa60T zUJ@gv;YzcN6-Xsb8Z>qPm|@2?jD1`&bKA;2XQ^^y+z+S!qC++sRosp6)2r-klchoo z$p#I@#2IUcFEpPo8G)_4U%i4o5S9W&G$c}0e(rZ|;)PnXE; zTi_{$oyn`(O3GG&p&(WR@i&2fsb z)bhvyJRC0eD#>Wys2L93pXO8wJ5{Pw5loqdNB1rFpS0xfVn;`>jji4ln;;X-j$;ML zO)Om3yV&U+pJ>`iWE~L1L4dNte#vRaU9oq8wwxW2X5_>D$^ZuCy5u=VhCuiGE9n(d z{=j?Q<0yO9H^Ag7BHjo+Wjl-0-=nHpgGjv2ylcU&A(@g$f@NI6yGEWYo#hZ26$<2| z%zM({`-I*B`|a6m{AI<4R|X?6KZ0Fwa^{yT+aMQq1wpDHq1>(uF6omjF-o)z<;RyV zaWl}DQtPmFd41x(K?X$kns*?fG&l1Ql31jSFYx>)>%GHjz85cJEa|GH%#$+RAjLzvC-f5qyROoMwaFcf$Wp&>qB*e!A?PtxGiBm5rH8SmJdtR1HGL=iv@)6|p|b=DNzl_`uQ*zKd8Ha_oBNvZ=DCR2Qd z)vw`Kc;I7>o~g}(L)3pe2B4;VOv5oOu{JN;Whsz4U%-@-+2#N|6;`*09r+|k__wrX zm*E5_4j>Q<{GX;mKeuA6W&RQ&(O-Jb=ONi?-?)Nwc9b%ZhP$gA>BSp0Eq>*_u%-bD z630QNEq3CuyFd-j51-JV3RhDqdVwl#lzQwoU%sL`&UOtE-{+>!1{fJ-S~?xVk(gFE zett-w=t7Mxoj`LrHeYyduQXO9DB%NHIw3dk6Yx}RPtx5H4`quIfhJ0ZS7p=I4pjv2 zu+rZj(%9`0ev0;!1|n9MUZt~Vy#2TyII6_==@lX!9asx3E*#?~s=Zg=1HhB*BkWSs zSFF7a*AI0;A4}y7h&TChq?{$*ESUo=x=N1pIMV#;uNtGoC*l7;!Pk;>_ zzt|X(_wow0wLaj_t}?~{I8!smjz?D~ICZB#up06n#ok$$dH0ZSs!C2m*vCKw7OUC_ zpDTA0u&M-39pv6?vgesK3|f+_*WV$rHVq6=6?VG3JA!QRRSZq%ixPl8VT(FIkkR?qc7D(UvCSGjA&$rH84^1$rQ zJ?%&@O$S9$UBYBYhC3!6b0$|N3u0OI?o1=nJTtt&u)T6_neB~Jr7kV`WJvIvpMTg7 zAb~@Bhb?*Y0toPn4nTU)zpMV51uX+A-pJnHync zi!rbfPZv3CeH8qsNvyB%x6{=BTVV;ND_6`_S%VWeJ|&7E z^o$28610fzn@Lv50q_t7yt^9p|J~M$ns70Soi|GwvU(9`9<|e?F5S&G2)RYvIIp2e zS-4k^Hj@*E33^SO(xKx>)jqc~FUtRR-%`>Y&R2L5#U#%g&xMh__7tVd8&lno1lW3 zVk@_?7PY}B`3SkUd+#}t8{kqL&w?tBy5H3wq?97xSxA?Q`lh~P;R4?5y+L8eWAJ*X z%chiu^=E)U4P&WEXC7vRt=)%_d#_u<=0cw>3B>=7fkA8I-`77Hx95Fp)uZF<{E_bUP-ij^D{)q-ej61Tu(yj6e~En!Pe zWdf|<*O0tFSOa`Z4I#~Da5DI*L#@K%O$|S^p+E0jy#&%4e5_UIhRKn7`%1xl|1q<&RG<^{$Mc%a=u{qbyTk9|N>`^&=nGD6-om5#ZPt|@A5@~!E z1d~r=st?D|jD|J*UfXOc$jdcu9J!1~d^2(?v(w1hx^X|*j$E`JwK7t}s>_(k8^}#v zolsf@^R#agThenNbp+^ckJTpA{>lL#-WgVq)f%ym?n`akYrlN0vNolY>Y;}gBv0Y2 zRBKxFFtO2%FTOzrMCs9YzuSviWnTcQ0X1`9IY7d6q3HBfhB{46xRolW3MSJ+o0pPMo@O{u$c>d{{G zz1SV{+8CZBuL;j^5tnpth_WJykt5jYS)UJa14PQ=%iTOFUf@6l2wPf@)H`ljTCnq^ zkGSY!cGr2r&eJ3T4#{Pn&CIa)cSS8&smVXCU;4<>=PbUZV$&K7Ce$gv0@i+k{P*3x zbiz)TK;8x~&iqLE;kjjt38;5m)i_DZz5nVE-JYZlt@#)cNG z|A#@zbqf7v(3prCYopdLHff7L1E82aq`%*Q_@=CD4~QStoP0}9qaV&OAK{Thi8aa7 zZJRcov;H-xS=ttwuXjxs;06Rfz*Ucim%W`4?ay?j`&nw#zD_2VM~6tN3o1?dl<50a zxp`2IN^$z-F-Eiukc);UCxe#tW9Q}|EV28oTA*uz7Pi^E?NoY-3^52KMEk@Yh=b52 z`*l{vX0|>M4!`et6Y*3JWYXR0@Z$`DDUT{EI#`@{Kb5ok7y%v{9n%a=+BX82v*pK&m_TQu9)CL(D9{9zx4M6aYqAdyw1v z_duP?_5o~Ah-OXYsf#t`5apmkks}DqpeAHu7*C%}h8NR`|C)>%AXL()Jk_r>#ZKIu zG*iGdmCYrd`u$&0gLic4o@STkXcziyaN}J#S5MGKL@f)=Hy{K+B`?}AapZgV?IH~% z>hu~G_4Vev)-A98Dmg=jmb(KCOHG_x_JiPkyrb7EP#Bh$L9CsM{)m_}5>&g;ck7<) z+A?!3TqhuKuxrlLnXK!SIvuuDD+p!FnR%_(M@uEyG@G?NsY4ZqEi^d*#U!T}*Nquk zc~3chgWE~Z>CW)!;k3&zk5iu;I82x0XDG1Sj5+sJs^$Ge=VFDOON<aCoQm18!6cu46`%>$KTxDJ!l5l&HV&pxT0zd%QIL48$>b zV^Ln5^$@!Xf{QscLg#cOjwA=;JZkx;X!<&qgHz|X@mA)?Eas3rvJC^71E}Kl`ML~n zg`Yianuo1qxP+oXdMY;Q>{*o7)f@%qNQq#=JEcMIUJg`Yy|!66yG0z>8N>eEP`G{S zWp!OKpy;=hF9g#!;0^x{^=PS=<0pUle$jCNQefiy+dF0wx0jeNM&q`s03z-EGFUS>6*Vc(gZ;&iGc+p&lUDp{$ukQ=Lxp0V~%v)l=*DmvH zD3a|RrjUa*-g1I#!)XYv>;+7Cw8>JYPq8crywJB3rNg)njDc0;I}f(kUS^n2eOZ=X zp#^RgZvi~8<)won%K5LuuD~{aX*5+2o~`R+(XhCZlidozUST}Y?HWf1QLr`ZkK&da z703{4hgPYor3M%*FiUoAIGr5@Dfoe~vkmvPa>^cR-RxV8D78z$&L^kaSgFaNE^yA| zhAHmtou|`&Q$0iu^LFrF{5oFC`;h4TBP!1#>U_E7?po)E@4BT0S#n!J7#-pFX?oL=gwJo=OFBhH*G{d{xH zeXNDCzz70X#>l*|^r{n>yNqHMHf7o$-bls)3`dI@j|5%X^HP8a43LgaQaNif#Gj7xnql~dBr zemMJ$$BZlzT3a~r@bE5nBa$=h7imG>J&48N@_>7E8rg6*oOTa+(?LG4bbfg=_BXlA z?MTCHFBn&aBt$LP@}fN%5Uq84TFP*;FX{C5o3Ubsow`%>gHef-tt$yUqwt`-a%g;OsD0pFntI^R@3=>3;?LUV{S&*Y+dvzyl zY?q=AG^r_Ue{_}Jk>qvWD~0Z$`{XP>`cde^pC^Oa&^64&-}l@j6TAgp7wOCoAHh-d zs&@WM=Kd=rOIYS3Js}+io7N&HxxmzFhIQ#qpTZ;CB|Wz5B-V6@V_^(kJ&^+s@=(5l7E3?;pLBOyt~LOD3CP2}8xf3j!(C*9qet4&Sn%>(j(YKLSdL#+03u7kOdUF6@||C8>1ldG*dH|hZe zmc8N{H2MU~&OFO?umG}4RdlyZ)(XfWbWWZ2XKIH5OAUymgGQB%h`DX{{sXI{w_7RP z7RxVW1`x*jAPUN7N}hx-*k;oZ5PJ1o$#+{95L@70dI%_#$?bLSn8WA(c-9G;2DC2- z(xcB!Vw$wp_wwY>+fbWJCM#lAhg%d~k`4Jm$TC{LTNeMkAsTK2m&u*mm zHlHWu1NaxFzgQKtRtwFpuZlP9Q^_r{<$q4t6xL+Wi9^qWfw7-;rtfpON7OpKbc~tgly5*oRrs z#;ZAW5fr4KXJW5AODAAMa}?2?F=f=Ow}Nmh0ZMzL=ql1wc@!MC%NUB!uwfTlnz z#URikpFX@|O-L+IK@~F|n}ydLZh|`hQBZv#0Kd%+e_as zM2WSoM6qVnVIEpr0dkT7KuKmTMR`$4%kf0FX>hH2w3N-J2h|ij2e3i5Gr{FLJlyD3 z*kd<2C{DYN_yg(|heE}GH9)39L!Fgo*G?TO%J4^*qBH!muNUmbYya)cd3%M*_X+FA zRqg@r5*FQV2M)ZpYyrckEhw&RnNWDKJ;SfOF4%{DtzI8O{eSoKH*M=Y`8}pm|I;_? zR5+~L#Al#p7R3r+^prB`Y^N48ky3+%P2@4C*6wUy47{To{8<|gd0~^%4U@dN;;vjN z;}Bif7%x_p);cn64@nI2s32}PNT9qTq$3lEC;eH0Wo%}uu&uWbf^>=PO;|5+6;8Y+ zN`VqOwG@&VKJn5QXfJ*VBYQid^LxWMF6V3lO*Xb4m)9PhrhMCUUgI*m%^NC)X}F<< zwlkgu2#L{>ZpnMts$z}J&z_GDMZr{NL_Ce8gqS_FA1?HHPwk^pUF@<~DlJ$jPS}mY zGo8=PckdnW(Mi2-tX~ka2nc4`#F1$kl)$9h-P@hpN26}OZZxYyeOp%m35BxAQ7&+u zO?2AX1>*-7;5z~r8Fx38-n(c7wv5GRBsNm3%p<>JkNw;q3p{&1?*Z4i{@(|s>5-5s zb8EQ=V9?JLOQ>L@rK)*hh5Rku3s~J^zcSw&j;1aDgB{)5SB_X(i8CFK&j*m^4UJ0L zXeEsS;zc`w=`WYN^=gbFvwGVyww^)=5|$?)4J3T9@Fj+TBHu1Hn0Rv|XR2#%EAn#_ zfXG3C>*l0aHunhgS*afPv=uieHep_vz{}GQ02UMVS<NJEP}n@(rkD4t11a6ARt1?ki8B)62QpL)qEr2OHgO>X;ucDV>tOBY4{!X8}RGoOh=Qzw!x;s0C(smi=|&l24c1e~_3IDND(>r=hy<1sK9 z068NWFYyExsMsez3n<1x0q4UHHHdhufSHsFYNN{{-qJHRA8a>b2 zs0C#51Ve?|)Cu2$f}#R}lXvL?e#!OMe6#|YFUNtwr`0V2^ySK-)vY3gst z&Vc%<$igj{{2MY*-vQ$)Y+yv3$YesRRJ&vHgLq!X1z70Ra(a30!!C-~0Lf96yI}lxb%t47JXM`PXR)34GXl3!H zgzJ!c@Q+i^(Vzny)7uMfCpq8tZf*OP<-Jge02nL7_-NxxzLHAX^cA)eVXdD4jdRcj z!I6R8usqv*&3Fo5@> z;dIgt+8La2cA(%L7VEQ{;8d!hO*fhHTE!+S8@^Og8Uz68`UcVD)XMvhee8MP4cdbz zk=b;*v|0aNsr$#>lSwS`AK)7*>#=K6B^VVlf$9=?3o?F$b#$h~$1O_#an$pdbaRMi zFW-pmN&%*5$ukZu6!j95DR&yKq^*$m1-6v}`#_T;a|6P4YZg)rc>1pue#XumV^ z?g8L*76zQARuLoJ0vsq6Z#6zy=x2vEbvDD9APVh6pF{^G{2`tdUkN)qu9HBWgHi!U zK<7u;_4a_jfvP|S`);V^BQ$Cb)rLpS-TF;#sQP{(c63~SHa_Ah2g_SEe}f7kD5xn? z6FPmp^s&D|6%@nl!3$2-*}EG;4V7NBH z+++m{2#@N&-j8k9vb#mvA0lkRY>IQwUN9F*a0ArU9kBq|Qif8t$X$ll2h&yo<|E1r2d2#hqHWuJ-~#ulH0Td|z2Y6Wzp%5% zrrS8kV$LGmNbCPnbt2xg9FHBiQn+E#7Rxa_)AO7tWg2KGr}OZUCS^psszkyCrKLT< zzOq&(8DGKN%(-CVhHr4_^iprm2>k5u_iz=6hytgy$Anu3tBO+tT}2@`N!S*X$474x zgfK`R@Ixo8c?HFlaR7FPx$O(s;Um0TgQ6!6vfv*U<7hr$n-e%oEhW@Bx8zhX4j$1a zN+yB~pC07~C^^lb+eQLNePfU#N@msVi~#Aty4C#pFSa(;=U}*oGu?M{SvTp^XzXKy zdmC;tT@jCCJEeL#iR6qWl_hD}BkxYW6{9@J5}0js7~&AV`!=5BHXdTHbo-l@>6*Jh z-QKD`+?LYv%=MH|d744Dm4{K+n3Lk<4FA%oyX?_Da=(s|$j523Oc}-5@S7$(T3K7{ z`e4ByF8(2^Y_E!NUf*|e-^%XlyvSpdX}h99deBJDch+LdODW03tRU4jV%#jvw~uf2 zyJR}2@;S*~PNYsUpN<_N+;cc~+_-dnb*0*0L>$po&Sx)kuw|ljI!q+PY$SAnmj76dYJDyczTbmmrQ;??wErHcXP`Dr$$3))B;z5B3xM^1_r@@;Oo1WA74z+ zyIzM6A{j55OIuTn?k$v}=1K8z%;~+PS1qMRh7J?EkA~UB8VbLsRNOE*{D@x;_p^ZqdC@{{Qc&b! z?its!H5vGiSII9{1RpJ6!E21vh8*%k5(Mlwe*xX+VLD~%xY>ZQge>1l{ZlfhWl;52 zm1QS&Q_L7tM3?9a1jURH_F*AZt>ns*QZDTqeaJ6ob$j>i964v}itk^buiqpw$f+!*q;GYI+1JOS*ytcTk~onj z-4PX^;!IgW7Q{-KU%PNlnR@5}ZWdWA#O4x*&mR<$N`|EpFE_YpZ)c=78bnD-mTvpS zExHn-b|7#JuR<{M)HS`3&=p!EehbB|+?BFkzx!2dX5&-#c?go-ak_coQTK7Oew><& zrwv8t8?$7V-`^h#-zQXeRlDSEB@PcSoJ`*VwY z#0?a5Y2rqa*IfC``Cx;2bt-+8Z=n=CySW95G7%5U@6UL`lc^_}Y2>kFqP!Q1a;op` zZb5{G#VVUf2yH}t|Cybbbiy-k<)e|$e-~+fcfU*EXM9bI^Pf+zbl{LY?J&JS*&!dE31yAKrn zDU{j_C-O5>r2F?oE=r|bN);1vkHZlk?$hYQm%FX&Ju~}?cvL#M^z$Z)ESc+>=+SW8 zfhE_`=Dnf;ds;LW9Z^t8*@Ab!b9z&lE8^Sy9g$D=Y-rk`+>Stxf->_XV@B29?L7L* zb>$>aO~Cb)$S1Uch4J=x#$K|k_TwJPY#!xmy)aLU;%0CEs-(n{+Uvtyf(?!y(<0!# zl3g?AUKJmXgO_1bU#@gMiGn0ES@CRwnY)I>qv;2SmlI2(H0`hOcG2RDx5%z`(w@_F zGR7%OEoQPV2bSbz5Gs^^Touip*-%NSu)5{7LBmU4NZkXOC#ozukL0idzgGEAl$|Jx z_a`qg6yraAZ?)Z#lD8ts?Wg;=eOB>V*bXu%YW-pkaX6bJe5S;%^$$Iw(T`5CD${m7 zv$K{KBF6vx^!ae9S7a{!;9*5;0^V5NUQu5}Z_o1+EyzJ|GdbjM!h>nSLj#DUi!If( zncWLHXycf3c=z4aQ=`T|rG$v)JlGQSC{G0zjmF~X>U7JYX5B(O+IZd(dFyLS`|*Vw zlKZ& z=(L*WBMy3F-(gH_@+AifoE8$)#xKCq}$!4p4JyQBUxMaw;+{8Nvv1Y zywEER$J0-WB5fdFM2tLf1Do08Vm01fg}3w>y-Xr|{{}q&j7Z-udoP}tJ-%G6R>yf) z^c^-HtttuH2iXhnjWHG;mw#qDdbzE> z#}DVtkMbEUQBr~ zn{{GKEB2%^p<#iZSNZ!=*4`pVkw5;E0TmSD&Ur5$*t)=;?$bvk{s~vhVQ{o)GJMnFbek&Ay>AWxoKQlO@jABl1f2PpYqbddP9e&3>t6p=9{G z3>4c}+oyl`?`s=owRc}n_@j%9pB7pNyoN_Pr=2Y=A5ic9i3rSKLH8u+Wr5MzciSO9 zrVY-IG9%=0#+otB$WbRR_32nJ@9r>#f~I#aLVeFI5-RrGQd_?%NFKyYSA+vb+2fTmHDYQ<|0Yx1wb6Nx2Y9_l zjaa_XxU&`RNh8O)V+OwOvYSgxFH?>hZr1~4$z4pEinpu^cIG9M6+rP_0~WtWPhERu zu}39ZPUpQ!oOXrEP?WIMuZYM1QC}KI1p*>z55MA735a8T>g95PTqAd@Nz7GW?T(Ff z^>@>fTEFk-H!@&Xr-k59P!Md=3j(rVklVxOH>=bp*q!7eZw9Ev#8czGbq!cSON&(9 zHIf_Es<#d-=k_dFyPC@VKx9a{`uX+LNbAv2y?|aOzIx6tBfnzTWDBV9-YnT9$Eow~ z$5>ro&4|%0-jTHq(&%Gc6fhuz=E~tn@~jbKt$)D&k?-(WVPnI);(Tr4Z?TMCQx8v3 zlR`@8N*24!lI}Ep{q#}1tv_EmjVe*xdmyXLk^(8;!P*qB{_>!K@$^T&F8cW=Q#mp6 zltmrhT^V~n(2C9C4zH#0l2`niKIgHo%h~Z!@ag;dDrV6v=?GOEjC&i{H6OvhD_g;m7wX)~l-yH3- zFTa&L4N#zm?cG(kI@`ab@I^;hMaHwHs5&c1*d`n5;>#-&m5JAUtm|oWh5MM7ez()Ab_>~d#Fsp=V0B;>BH|3DFy+PU zWqQyv-IiG)n^3XgD zK$~@gYyYoD>#c+AJP!F#7q7F(yH^w^T3FLQ}+@h3odQy zC-Kpb>fsQnaPmvXnr7~!!pN;Kx1~!0wt_|%d0jWY-G0(=uKPeION%~_PlyUDVMv;r z7scIN$$Z&iDm7$vO3b^pv}3*Mk>!)di902mSp%mMY?;bmI!VMsHwUva6w+62lmsIdwrTQ-yf% I|IhyaKTXjVy8r+H literal 0 HcmV?d00001 diff --git a/kernel-julia/gateway/kernelspec-image/julia_kubernetes/scripts/kernel-pod.yaml.j2 b/kernel-julia/gateway/kernelspec-image/julia_kubernetes/scripts/kernel-pod.yaml.j2 new file mode 100644 index 0000000..73f682e --- /dev/null +++ b/kernel-julia/gateway/kernelspec-image/julia_kubernetes/scripts/kernel-pod.yaml.j2 @@ -0,0 +1,97 @@ +# This file defines the Kubernetes objects necessary for kernels to run witihin Kubernetes. +# Substitution parameters are processed by the launch_kubernetes.py code located in the +# same directory. Some values are factory values, while others (typically prefixed with 'kernel_') can be +# provided by the client. +# +# This file can be customized as needed. No changes are required to launch_kubernetes.py provided kernel_ +# values are used - which be automatically set from corresponding KERNEL_ env values. Updates will be required +# to launch_kubernetes.py if new document sections (i.e., new k8s 'kind' objects) are introduced. +# +apiVersion: v1 +kind: Pod +metadata: + name: "{{ kernel_pod_name }}" + namespace: "{{ kernel_namespace }}" + labels: + kernel_id: "{{ kernel_id }}" + app: enterprise-gateway + component: kernel + source: kernel-pod.yaml +spec: + restartPolicy: Never + serviceAccountName: "{{ kernel_service_account_name }}" +# NOTE: that using runAsGroup requires that feature-gate RunAsGroup be enabled. +# WARNING: Only using runAsUser w/o runAsGroup or NOT enabling the RunAsGroup feature-gate +# will result in the new kernel pod's effective group of 0 (root)! although the user will +# correspond to the runAsUser value. As a result, BOTH should be uncommented AND the feature-gate +# should be enabled to ensure expected behavior. In addition, 'fsGroup: 100' is recommended so +# that /home/jovyan can be written to via the 'users' group (gid: 100) irrespective of the +# "kernel_uid" and "kernel_gid" values. + {% if kernel_uid is defined or kernel_gid is defined %} + securityContext: + {% if kernel_uid is defined %} + runAsUser: {{ kernel_uid | int }} + {% endif %} + {% if kernel_gid is defined %} + runAsGroup: {{ kernel_gid | int }} + {% endif %} + fsGroup: 100 + {% endif %} + containers: + - image: "{{ kernel_image }}" + name: "{{ kernel_pod_name }}" + env: +# Add any custom envs here that aren't already configured for the kernel's environment +# - name: MY_CUSTOM_ENV +# value: "my_custom_value" + {% if kernel_cpus is defined or kernel_memory is defined or kernel_gpus is defined or kernel_cpus_limit is defined or kernel_memory_limit is defined or kernel_gpus_limit is defined %} + resources: + {% if kernel_cpus is defined or kernel_memory is defined or kernel_gpus is defined %} + requests: + {% if kernel_cpus is defined %} + cpu: "{{ kernel_cpus }}" + {% endif %} + {% if kernel_memory is defined %} + memory: "{{ kernel_memory }}" + {% endif %} + {% if kernel_gpus is defined %} + nvidia.com/gpu: "{{ kernel_gpus }}" + {% endif %} + {% endif %} + {% if kernel_cpus_limit is defined or kernel_memory_limit is defined or kernel_gpus_limit is defined %} + limits: + {% if kernel_cpus_limit is defined %} + cpu: "{{ kernel_cpus_limit }}" + {% endif %} + {% if kernel_memory_limit is defined %} + memory: "{{ kernel_memory_limit }}" + {% endif %} + {% if kernel_gpus_limit is defined %} + nvidia.com/gpu: "{{ kernel_gpus_limit }}" + {% endif %} + {% endif %} + {% endif %} + {% if kernel_working_dir is defined %} + workingDir: "{{ kernel_working_dir }}" + {% endif %} + volumeMounts: +# Define any "unconditional" mounts here, followed by "conditional" mounts that vary per client + - mountPath: /tmp/kernel-logs + name: log-volume + readOnly: false + {% if kernel_volume_mounts %} + {% for volume_mount in kernel_volume_mounts %} + - {{ volume_mount }} + {% endfor %} + {% endif %} + volumes: +# Define any "unconditional" volumes here, followed by "conditional" volumes that vary per client + - name: log-volume + hostPath: + path: /host + type: DirectoryOrCreate + {% if kernel_volumes %} + {% for volume in kernel_volumes %} + - {{ volume }} + {% endfor %} + {% endif %} diff --git a/kernel-julia/gateway/kernelspec-image/julia_kubernetes/scripts/launch_kubernetes.py b/kernel-julia/gateway/kernelspec-image/julia_kubernetes/scripts/launch_kubernetes.py new file mode 100755 index 0000000..2582d66 --- /dev/null +++ b/kernel-julia/gateway/kernelspec-image/julia_kubernetes/scripts/launch_kubernetes.py @@ -0,0 +1,414 @@ +#!/opt/conda/bin/python +"""Launch on kubernetes.""" +import argparse +import os +import sys +from typing import Dict, List + +import urllib3 +import yaml +from jinja2 import Environment, FileSystemLoader, select_autoescape +from kubernetes import client, config +from kubernetes.client.rest import ApiException + +urllib3.disable_warnings() + +KERNEL_POD_TEMPLATE_PATH = "/kernel-pod.yaml.j2" + + +def generate_kernel_pod_yaml(keywords): + """Return the kubernetes pod spec as a yaml string. + + - load jinja2 template from this file directory. + - substitute template variables with keywords items. + """ + j_env = Environment( + loader=FileSystemLoader(os.path.dirname(__file__)), + trim_blocks=True, + lstrip_blocks=True, + autoescape=select_autoescape( + disabled_extensions=( + "j2", + "yaml", + ), + default_for_string=True, + default=True, + ), + ) + # jinja2 template substitutes template variables with None though keywords doesn't + # contain corresponding item. Therefore, no need to check if any are left unsubstituted. + # Kubernetes API server will validate the pod spec instead. + k8s_yaml = j_env.get_template(KERNEL_POD_TEMPLATE_PATH).render(**keywords) + + return k8s_yaml + + +def extend_pod_env(pod_def: dict) -> dict: + """Extends the pod_def.spec.containers[0].env stanza with current environment.""" + env_stanza = pod_def["spec"]["containers"][0].get("env") or [] + + # Walk current set of template env entries and replace those found in the current + # env with their values (and record those items). Then add all others from the env + # that were not already. + processed_entries: List[str] = [] + for item in env_stanza: + item_name = item.get("name") + if item_name in os.environ: + item["value"] = os.environ[item_name] + processed_entries.append(item_name) + + for name, value in os.environ.items(): + if name not in processed_entries: + env_stanza.append({"name": name, "value": value}) + + pod_def["spec"]["containers"][0]["env"] = env_stanza + return pod_def + + +# a popular reason that lasts many APIs but is not constantized in the client lib +K8S_ALREADY_EXIST_REASON = "AlreadyExists" + + +def _parse_k8s_exception(exc: ApiException) -> str: + """Parse the exception and return the error message from kubernetes api + + Args: + exc (Exception): Exception object + + Returns: + str: Error message from kubernetes api + """ + # more exception can be parsed, but at the time of implementation we only need this one + if exc.status == 409: + if exc.reason == "Conflict" and f'"reason":{K8S_ALREADY_EXIST_REASON}' in exc.body: + return K8S_ALREADY_EXIST_REASON + return "" + + +def launch_kubernetes_kernel( + kernel_id, + port_range, + response_addr, + public_key, + spark_context_init_mode, + pod_template_file, + spark_opts_out, + kernel_class_name, +): + """Launches a containerized kernel as a kubernetes pod.""" + + if os.getenv("KUBERNETES_SERVICE_HOST"): + config.load_incluster_config() + else: + config.load_kube_config() + + # Capture keywords and their values. + keywords = dict() + + # Factory values... + # Since jupyter lower cases the kernel directory as the kernel-name, we need to capture its case-sensitive + # value since this is used to locate the kernel launch script within the image. + # Ensure these key/value pairs are reflected in the environment. We'll add these to the container's env + # stanza after the pod template is generated. + if port_range: + os.environ["PORT_RANGE"] = port_range + if public_key: + os.environ["PUBLIC_KEY"] = public_key + if response_addr: + os.environ["RESPONSE_ADDRESS"] = response_addr + if kernel_id: + os.environ["KERNEL_ID"] = kernel_id + if spark_context_init_mode: + os.environ["KERNEL_SPARK_CONTEXT_INIT_MODE"] = spark_context_init_mode + if kernel_class_name: + os.environ["KERNEL_CLASS_NAME"] = kernel_class_name + + os.environ["KERNEL_NAME"] = os.path.basename( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + ) + + # Walk env variables looking for names prefixed with KERNEL_. When found, set corresponding keyword value + # with name in lower case. + for name, value in os.environ.items(): + if name.startswith("KERNEL_"): + keywords[name.lower()] = yaml.safe_load(value) + + # Substitute all template variable (wrapped with {{ }}) and generate `yaml` string. + k8s_yaml = generate_kernel_pod_yaml(keywords) + + # For each k8s object (kind), call the appropriate API method. Too bad there isn't a method + # that can take a set of objects. + # + # Creation for additional kinds of k8s objects can be added below. Refer to + # https://github.com/kubernetes-client/python for API signatures. Other examples can be found in + # https://github.com/jupyter-server/enterprise_gateway/tree/main/enterprise_gateway/services/processproxies/k8s.py + # + pod_template = None + pod_created = None + kernel_namespace = keywords["kernel_namespace"] + k8s_objs = yaml.safe_load_all(k8s_yaml) + for k8s_obj in k8s_objs: + if k8s_obj.get("kind"): + if k8s_obj["kind"] == "Pod": + # print("{}".format(k8s_obj)) # useful for debug + pod_template = extend_pod_env(k8s_obj) + if pod_template_file is None: + try: + pod_created = client.CoreV1Api(client.ApiClient()).create_namespaced_pod( + body=k8s_obj, namespace=kernel_namespace + ) + except ApiException as exc: + if _parse_k8s_exception(exc) == K8S_ALREADY_EXIST_REASON: + pod_created = ( + client.CoreV1Api(client.ApiClient()) + .list_namespaced_pod( + namespace=kernel_namespace, + label_selector=f"kernel_id={kernel_id}", + watch=False, + ) + .items[0] + ) + else: + raise exc + elif k8s_obj["kind"] == "Secret": + if pod_template_file is None: + client.CoreV1Api(client.ApiClient()).create_namespaced_secret( + body=k8s_obj, namespace=kernel_namespace + ) + elif k8s_obj["kind"] == "PersistentVolumeClaim": + if pod_template_file is None: + try: + client.CoreV1Api( + client.ApiClient() + ).create_namespaced_persistent_volume_claim( + body=k8s_obj, namespace=kernel_namespace + ) + except ApiException as exc: + if _parse_k8s_exception(exc) == K8S_ALREADY_EXIST_REASON: + pass + else: + raise exc + elif k8s_obj["kind"] == "PersistentVolume": + if pod_template_file is None: + client.CoreV1Api(client.ApiClient()).create_persistent_volume(body=k8s_obj) + elif k8s_obj["kind"] == "Service": + if pod_template_file is None: + if pod_created is not None: + # Create dependency between pod and service, useful to delete service when kernel stops + k8s_obj["metadata"]["ownerReferences"] = [ + { + "apiVersion": "v1", + "kind": "pod", + "name": str(pod_created.metadata.name), + "uid": str(pod_created.metadata.uid), + } + ] + client.CoreV1Api(client.ApiClient()).create_namespaced_service( + body=k8s_obj, namespace=kernel_namespace + ) + elif k8s_obj["kind"] == "ConfigMap": + if pod_template_file is None: + if pod_created is not None: + # Create dependency between pod and configmap, useful to delete service when kernel stops + k8s_obj["metadata"]["ownerReferences"] = [ + { + "apiVersion": "v1", + "kind": "pod", + "name": str(pod_created.metadata.name), + "uid": str(pod_created.metadata.uid), + } + ] + client.CoreV1Api(client.ApiClient()).create_namespaced_config_map( + body=k8s_obj, namespace=kernel_namespace + ) + else: + sys.exit( + f"ERROR - Unhandled Kubernetes object kind '{k8s_obj['kind']}' found in yaml file - " + f"kernel launch terminating!" + ) + else: + sys.exit( + f"ERROR - Unknown Kubernetes object '{k8s_obj}' found in yaml file - kernel launch terminating!" + ) + + if pod_template_file: + # TODO - construct other --conf options for things like mounts, resources, etc. + # write yaml to file... + stream = open(pod_template_file, "w") + yaml.dump(pod_template, stream) + + # Build up additional spark options. Note the trailing space to accommodate concatenation + additional_spark_opts = ( + f"--conf spark.kubernetes.driver.podTemplateFile={pod_template_file} " + f"--conf spark.kubernetes.executor.podTemplateFile={pod_template_file} " + ) + + additional_spark_opts += _get_spark_resources(pod_template) + + if spark_opts_out: + with open(spark_opts_out, "w+") as soo_fd: + soo_fd.write(additional_spark_opts) + else: # If no spark_opts_out was specified, print to stdout in case this is an old caller + print(additional_spark_opts) + + +def _get_spark_resources(pod_template: Dict) -> str: + # Gather up resources for cpu/memory requests/limits. Since gpus require a "discovery script" + # we'll leave that alone for now: + # https://spark.apache.org/docs/latest/running-on-kubernetes.html#resource-allocation-and-configuration-overview + # + # The config value names below are pulled from: + # https://spark.apache.org/docs/latest/running-on-kubernetes.html#container-spec + spark_resources = "" + containers = pod_template.get("spec", {}).get("containers", []) + if containers: + # We're just dealing with single-container pods at this time. + resources = containers[0].get("resources", {}) + if resources: + requests = resources.get("requests", {}) + if requests: + cpu_request = requests.get("cpu") + if cpu_request: + spark_resources += ( + f"--conf spark.driver.cores={cpu_request} " + f"--conf spark.executor.cores={cpu_request} " + ) + memory_request = requests.get("memory") + if memory_request: + spark_resources += ( + f"--conf spark.driver.memory={memory_request} " + f"--conf spark.executor.memory={memory_request} " + ) + + limits = resources.get("limits", {}) + if limits: + cpu_limit = limits.get("cpu") + if cpu_limit: + spark_resources += ( + f"--conf spark.kubernetes.driver.limit.cores={cpu_limit} " + f"--conf spark.kubernetes.executor.limit.cores={cpu_limit} " + ) + memory_limit = limits.get("memory") + if memory_limit: + spark_resources += ( + f"--conf spark.driver.memory={memory_limit} " + f"--conf spark.executor.memory={memory_limit} " + ) + return spark_resources + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--kernel-id", + dest="kernel_id", + nargs="?", + help="Indicates the id associated with the launched kernel.", + ) + parser.add_argument( + "--port-range", + dest="port_range", + nargs="?", + metavar="..", + help="Port range to impose for kernel ports", + ) + parser.add_argument( + "--response-address", + dest="response_address", + nargs="?", + metavar=":", + help="Connection address (:) for returning connection file", + ) + parser.add_argument( + "--public-key", + dest="public_key", + nargs="?", + help="Public key used to encrypt connection information", + ) + parser.add_argument( + "--spark-context-initialization-mode", + dest="spark_context_init_mode", + nargs="?", + help="Indicates whether or how a spark context should be created", + ) + parser.add_argument( + "--pod-template", + dest="pod_template_file", + nargs="?", + metavar="template filename", + help="When present, yaml is written to file, no launch performed.", + ) + parser.add_argument( + "--spark-opts-out", + dest="spark_opts_out", + nargs="?", + metavar="additional spark options filename", + help="When present, additional spark options are written to file, " + "no launch performed, requires --pod-template.", + ) + parser.add_argument( + "--kernel-class-name", + dest="kernel_class_name", + nargs="?", + help="Indicates the name of the kernel class to use. Must be a subclass of 'ipykernel.kernelbase.Kernel'.", + ) + + # The following arguments are deprecated and will be used only if their mirroring arguments have no value. + # This means that the default value for --spark-context-initialization-mode (none) will need to come from + # the mirrored args' default until deprecated item has been removed. + parser.add_argument( + "--RemoteProcessProxy.kernel-id", + dest="rpp_kernel_id", + nargs="?", + help="Indicates the id associated with the launched kernel. (deprecated)", + ) + parser.add_argument( + "--RemoteProcessProxy.port-range", + dest="rpp_port_range", + nargs="?", + metavar="..", + help="Port range to impose for kernel ports (deprecated)", + ) + parser.add_argument( + "--RemoteProcessProxy.response-address", + dest="rpp_response_address", + nargs="?", + metavar=":", + help="Connection address (:) for returning connection file (deprecated)", + ) + parser.add_argument( + "--RemoteProcessProxy.public-key", + dest="rpp_public_key", + nargs="?", + help="Public key used to encrypt connection information (deprecated)", + ) + parser.add_argument( + "--RemoteProcessProxy.spark-context-initialization-mode", + dest="rpp_spark_context_init_mode", + nargs="?", + help="Indicates whether or how a spark context should be created (deprecated)", + default="none", + ) + + arguments = vars(parser.parse_args()) + kernel_id = arguments["kernel_id"] or arguments["rpp_kernel_id"] + port_range = arguments["port_range"] or arguments["rpp_port_range"] + response_addr = arguments["response_address"] or arguments["rpp_response_address"] + public_key = arguments["public_key"] or arguments["rpp_public_key"] + spark_context_init_mode = ( + arguments["spark_context_init_mode"] or arguments["rpp_spark_context_init_mode"] + ) + pod_template_file = arguments["pod_template_file"] + spark_opts_out = arguments["spark_opts_out"] + kernel_class_name = arguments["kernel_class_name"] + + launch_kubernetes_kernel( + kernel_id, + port_range, + response_addr, + public_key, + spark_context_init_mode, + pod_template_file, + spark_opts_out, + kernel_class_name, + ) diff --git a/scripts/build_kernel_image.sh b/scripts/build_kernel_image.sh new file mode 100755 index 0000000..f10f2ad --- /dev/null +++ b/scripts/build_kernel_image.sh @@ -0,0 +1,19 @@ +#! /usr/bin/env bash + + +set -x + +cd ../kernel-julia/container/ + +# Create the tarball of the kernel files +tar --gzip --create --dereference \ + --file=jupyter_enterprise_gateway_kernel_image_files_docker-julia.tar.gz \ + kernel-launchers/ bootstrap-kernel.sh eventloop.jl init.jl + +# Build the Docker image +docker build --tag "ghcr.io/eodcgmbh/julia-jeg-kernel:beta" . + +# Cleanup +rm jupyter_enterprise_gateway_kernel_image_files_docker-julia.tar.gz + +set +x diff --git a/scripts/build_kernelspec_image.sh b/scripts/build_kernelspec_image.sh new file mode 100755 index 0000000..78a7652 --- /dev/null +++ b/scripts/build_kernelspec_image.sh @@ -0,0 +1,11 @@ +#! /usr/bin/env bash + + +set -x + +cd ../kernel-julia/gateway/kernelspec-image + +# Build the Docker image +docker build --tag "ghcr.io/eodcgmbh/julia-jeg-kernelspec:beta" . + +set +x diff --git a/scripts/cmd.sh b/scripts/cmd.sh new file mode 100755 index 0000000..baab140 --- /dev/null +++ b/scripts/cmd.sh @@ -0,0 +1,27 @@ +#! /usr/bin/env bash + + +exit 0 # Remove this line to run the script + +( + ./build_kernel_image.sh & + ./build_kernelspec_image.sh & +) & +pid_build=$! + +minikube start --embed-certs --memory=4g --cpus=4 & +pid_mkb=$! + +wait $pid_build $pid_mkb +helm upgrade --install enterprise-gateway ./enterprise_gateway/etc/kubernetes/helm/enterprise-gateway/ \ + --kube-context minikube --create-namespace --namespace jeg \ + --set kernel.shareGatewayNamespace=true \ + --set kernel.launchTimeout=600 \ + --set kernelspecs.image=docker.io/uberdavid/julia-kernelspec:alpha \ + --set kernel.allowedKernels="{r_kubernetes,python_kubernetes,julia_kubernetes}" + +# New shell +kubectl port-forward --namespace jeg svc/enterprise-gateway 8888:8888 & + +# New shell +jupyter lab --gateway-url=http://localhost:8888 --no-browser --GatewayClient.request_timeout=600.0 diff --git a/scripts/pre-pull.sh b/scripts/pre-pull.sh new file mode 100755 index 0000000..44b68ed --- /dev/null +++ b/scripts/pre-pull.sh @@ -0,0 +1,13 @@ +#! /usr/bin/env bash + + +python3 -c "import docker; dc=docker.from_env(); dc.images.pull('ghcr.io/eodcgmbh/julia-jeg-kernel', tag='beta')" +python3 -c "import docker; dc=docker.from_env(); dc.images.pull('ghcr.io/eodcgmbh/julia-jeg-kernelspec', tag='beta')" + + +# Manual image management on running kernel-image-puller pod +#import docker +#client = docker.from_env() +#client.images.list() +#client.images.pull() +#client.images.remove() diff --git a/test/julia-testnb.ipynb b/test/julia-testnb.ipynb new file mode 100644 index 0000000..9665c40 --- /dev/null +++ b/test/julia-testnb.ipynb @@ -0,0 +1,111 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "1b601f64-cba6-407c-b62c-258629fad7e7", + "metadata": {}, + "outputs": [], + "source": [ + "print(0123)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9176c0fc-8464-4194-b7a5-ec8cae2ddfdc", + "metadata": {}, + "outputs": [], + "source": [ + "using Base.Threads\n", + "\n", + "\n", + "results = zeros(100)\n", + "Threads.@threads for i in 1:100\n", + " results[i] = i^2\n", + " println(\"Computed for $i on thread $(threadid())\")\n", + "end\n", + "\n", + "println(\"Results: $results\")" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "489153ed-aa5d-48c6-92c8-7d1197ead780", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Number of threads: 6\n", + "Starting block matrix multiplication...\n", + " 2.038268 seconds (22.79 k allocations: 1.490 MiB, 5.16% compilation time)\n", + "Computation completed.\n", + "Checksum of result matrix: -4415.359418725857\n" + ] + } + ], + "source": [ + "using Base.Threads\n", + "\n", + "\n", + "function block_multiply!(C, A, B, block_size)\n", + " n = size(A, 1) # Square matrices\n", + "\n", + " #for ii in 1:block_size:n # single-threaded\n", + " @threads for ii in 1:block_size:n # multi-threaded\n", + " for jj in 1:block_size:n\n", + " for kk in 1:block_size:n\n", + " for i in ii:min(ii + block_size - 1, n)\n", + " for j in jj:min(jj + block_size - 1, n)\n", + " for k in kk:min(kk + block_size - 1, n)\n", + " C[i, j] += A[i, k] * B[k, j]\n", + " end\n", + " end\n", + " end\n", + " end\n", + " end\n", + " end\n", + "end\n", + "\n", + "\n", + "function test_multithreaded_matmul()\n", + " n = 1024 # Matrix dimensions n^2\n", + " block_size = 64 # Decomposition block size\n", + "\n", + " A = randn(n, n)\n", + " B = randn(n, n)\n", + " C = zeros(n, n)\n", + "\n", + " println(\"Number of threads: $(nthreads())\")\n", + " println(\"Starting block matrix multiplication...\")\n", + "\n", + " @time block_multiply!(C, A, B, block_size)\n", + "\n", + " println(\"Computation completed.\")\n", + " println(\"Checksum of result matrix: \", sum(C))\n", + "end\n", + "\n", + "\n", + "test_multithreaded_matmul()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Julia on Kubernetes", + "language": "julia", + "name": "julia_kubernetes" + }, + "language_info": { + "file_extension": ".jl", + "mimetype": "application/julia", + "name": "julia", + "version": "1.10.3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}