diff --git a/workspace/README.md b/workspace/README.md index e77b4900..44d1c116 100644 --- a/workspace/README.md +++ b/workspace/README.md @@ -5,7 +5,13 @@ Write and develop solutions in RStudio and Jupyter using Peak workspaces. We provide Python and R versions of the images: - [Python](./python) - [ds-pack](./python/ds-pack) - - [workspace-python-ds-pack-v1](./python/ds-pack/v1/) + - [workspace-python-ds-pack-1.0.0-base-python-3.8.13](./python/ds-pack/1.0.0-base-python-3.8.13/) + - [workspace-python-ds-pack-2.0.0-base-python-3.11.2](./python/ds-pack/2.0.0-base-python-3.11.2/) + - [workspace-python-ds-pack-2.1.0-base-python-3.8.17](./python/ds-pack/2.1.0-base-python-3.8.18/) + - [workspace-python-ds-pack-2.1.0-base-python-3.9.18](./python/ds-pack/2.1.0-base-python-3.9.18/) + - [workspace-python-ds-pack-2.1.0-base-python-3.10.13](./python/ds-pack/2.1.0-base-python-3.10.13/) + - [workspace-python-ds-pack-2.1.0-base-python-3.11.5](./python/ds-pack/2.1.0-base-python-3.11.5/) - [R](./r) - [ds-pack](./r/ds-pack) - - [workspace-r-ds-pack-v1](./r/ds-pack/v1/) + - [workspace-r-ds-pack-1.0.0-base-r-4.2.0](./r/ds-pack/1.0.0-base-r-4.2.0/) + - [workspace-r-ds-pack-1.1.0-base-r-4.3.1](./r/ds-pack/1.1.0-base-r-4.3.1/) diff --git a/workspace/python/ds-pack/v1/Dockerfile b/workspace/python/ds-pack/1.0.0-base-python-3.8.13/Dockerfile similarity index 100% rename from workspace/python/ds-pack/v1/Dockerfile rename to workspace/python/ds-pack/1.0.0-base-python-3.8.13/Dockerfile diff --git a/workspace/python/ds-pack/v1/README.md b/workspace/python/ds-pack/1.0.0-base-python-3.8.13/README.md similarity index 97% rename from workspace/python/ds-pack/v1/README.md rename to workspace/python/ds-pack/1.0.0-base-python-3.8.13/README.md index 80f66660..d0f2e67b 100644 --- a/workspace/python/ds-pack/v1/README.md +++ b/workspace/python/ds-pack/1.0.0-base-python-3.8.13/README.md @@ -1,4 +1,4 @@ -# workspace-python-ds-pack-v1 +# workspace-python-ds-pack-1.0.0-base-python-3.8.13 This is the Python Data Science pack image made for Workspaces. The image runs a [Jupyterhub](https://jupyter.org/hub) server that has [PyCharm](https://lp.jetbrains.com/projector/) and [VSCode](https://github.com/coder/code-server) pre-installed with all of the essential and most used packages. diff --git a/workspace/python/ds-pack/v1/codeserver-proxy/jupyter_codeserver_proxy/__init__.py b/workspace/python/ds-pack/1.0.0-base-python-3.8.13/codeserver-proxy/jupyter_codeserver_proxy/__init__.py similarity index 100% rename from workspace/python/ds-pack/v1/codeserver-proxy/jupyter_codeserver_proxy/__init__.py rename to workspace/python/ds-pack/1.0.0-base-python-3.8.13/codeserver-proxy/jupyter_codeserver_proxy/__init__.py diff --git a/workspace/python/ds-pack/v1/codeserver-proxy/jupyter_codeserver_proxy/icons/vs_code_icon.svg b/workspace/python/ds-pack/1.0.0-base-python-3.8.13/codeserver-proxy/jupyter_codeserver_proxy/icons/vs_code_icon.svg similarity index 100% rename from workspace/python/ds-pack/v1/codeserver-proxy/jupyter_codeserver_proxy/icons/vs_code_icon.svg rename to workspace/python/ds-pack/1.0.0-base-python-3.8.13/codeserver-proxy/jupyter_codeserver_proxy/icons/vs_code_icon.svg diff --git a/workspace/python/ds-pack/v1/codeserver-proxy/setup.py b/workspace/python/ds-pack/1.0.0-base-python-3.8.13/codeserver-proxy/setup.py similarity index 100% rename from workspace/python/ds-pack/v1/codeserver-proxy/setup.py rename to workspace/python/ds-pack/1.0.0-base-python-3.8.13/codeserver-proxy/setup.py diff --git a/workspace/python/ds-pack/v1/projector-proxy/jupyter_projector_proxy/__init__.py b/workspace/python/ds-pack/1.0.0-base-python-3.8.13/projector-proxy/jupyter_projector_proxy/__init__.py similarity index 100% rename from workspace/python/ds-pack/v1/projector-proxy/jupyter_projector_proxy/__init__.py rename to workspace/python/ds-pack/1.0.0-base-python-3.8.13/projector-proxy/jupyter_projector_proxy/__init__.py diff --git a/workspace/python/ds-pack/v1/projector-proxy/jupyter_projector_proxy/icons/pycharm_icon.svg b/workspace/python/ds-pack/1.0.0-base-python-3.8.13/projector-proxy/jupyter_projector_proxy/icons/pycharm_icon.svg similarity index 100% rename from workspace/python/ds-pack/v1/projector-proxy/jupyter_projector_proxy/icons/pycharm_icon.svg rename to workspace/python/ds-pack/1.0.0-base-python-3.8.13/projector-proxy/jupyter_projector_proxy/icons/pycharm_icon.svg diff --git a/workspace/python/ds-pack/v1/projector-proxy/setup.py b/workspace/python/ds-pack/1.0.0-base-python-3.8.13/projector-proxy/setup.py similarity index 100% rename from workspace/python/ds-pack/v1/projector-proxy/setup.py rename to workspace/python/ds-pack/1.0.0-base-python-3.8.13/projector-proxy/setup.py diff --git a/workspace/python/ds-pack/v1/requirements.txt b/workspace/python/ds-pack/1.0.0-base-python-3.8.13/requirements.txt similarity index 100% rename from workspace/python/ds-pack/v1/requirements.txt rename to workspace/python/ds-pack/1.0.0-base-python-3.8.13/requirements.txt diff --git a/workspace/python/ds-pack/v1/spark/config.ini b/workspace/python/ds-pack/1.0.0-base-python-3.8.13/spark/config.ini similarity index 100% rename from workspace/python/ds-pack/v1/spark/config.ini rename to workspace/python/ds-pack/1.0.0-base-python-3.8.13/spark/config.ini diff --git a/workspace/python/ds-pack/2.0.0-base-python-3.11.2/README.md b/workspace/python/ds-pack/2.0.0-base-python-3.11.2/README.md index 061bd0e1..5830faec 100644 --- a/workspace/python/ds-pack/2.0.0-base-python-3.11.2/README.md +++ b/workspace/python/ds-pack/2.0.0-base-python-3.11.2/README.md @@ -1,7 +1,8 @@ # workspace-python-ds-pack-2.0.0-base-python-3.11.2 This is the Python Data Science pack image made for Workspaces. The image runs a [Jupyterhub](https://jupyter.org/hub) server that has [VSCode](https://github.com/coder/code-server) pre-installed with all of the essential and most used packages. -Note: It doesn't support [PyCharm](https://lp.jetbrains.com/projector/) unlike previous Python workspace image versions. + +Note: The **doesn't support** [PyCharm](https://lp.jetbrains.com/projector/) unlike previous Python workspace image versions. ## Image details ### Base image diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.10.13/Dockerfile b/workspace/python/ds-pack/2.1.0-base-python-3.10.13/Dockerfile new file mode 100644 index 00000000..0cf35eb6 --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.10.13/Dockerfile @@ -0,0 +1,129 @@ +FROM python:3.10.13-slim-bookworm + +# Set shell to be bash and fail any pipe if any step in the pipe fails +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +# install system packages +RUN apt-get update \ + # installation requirements + && apt-get install -y --no-install-recommends \ + apt-transport-https \ + ca-certificates \ + software-properties-common \ + curl \ + gnupg2 \ + # docker + && mkdir -p /etc/apt/keyrings \ + && curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg \ + && echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \ + $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null \ + && apt-key adv --keyserver keyserver.ubuntu.com --recv-key '95C0FAF38DB3CCAD0C080A7BDC78B2DDEABC47B7' \ + && add-apt-repository 'deb http://cloud.r-project.org/bin/linux/debian bookworm-cran40/' \ + # node + && curl -sL https://deb.nodesource.com/setup_lts.x | bash - \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + # install system packages + # jq is needed for the Peak Platform + jq \ + nano \ + vim \ + fish \ + zsh \ + r-base=4.2.2.20221110-2 \ + r-recommended=4.2.2.20221110-2 \ + build-essential \ + sudo \ + nodejs \ + git \ + htop \ + pandoc \ + texlive \ + texlive-science \ + texlive-latex-extra \ + texlive-luatex \ + texlive-xetex \ + texlive-pictures \ + latexmk \ + lmodern \ + docker-ce \ + less \ + libxext6 \ + libxrender1 \ + libxtst6 \ + libfreetype6 \ + libxi6 \ + openssl \ + unzip \ + && rm -rf /var/lib/apt/lists/* + +# install AWS CLI v2 +RUN curl -fsSL "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.7.4.zip" -o "awscliv2.zip" \ + && unzip -q awscliv2.zip \ + && ./aws/install \ + && rm -rf aws awscliv2.zip + +# Needed for jupyterhub +RUN npm install -g configurable-http-proxy \ + && rm -rf ~/.npm + +# Install Python packages +COPY requirements.txt /tmp/requirements.txt + +RUN pip install --no-cache-dir --upgrade \ + pip \ + setuptools \ + wheel + +RUN pip install -r /tmp/requirements.txt + +# SPARK: copy over spark config +COPY ./spark /usr/local/spark + +# VS Code +COPY codeserver-proxy codeserver-proxy +RUN curl -fsSL https://code-server.dev/install.sh \ + | sh -s -- --version 4.16.1 \ + && pip install --no-cache-dir ./codeserver-proxy + +# vscode cli +RUN curl -Lk 'https://code.visualstudio.com/sha/download?build=stable&os=cli-alpine-x64' --output vscode_cli.tar.gz \ + && tar -xf vscode_cli.tar.gz \ + && cp ./code /usr/local/bin \ + && rm -rf vscode_cli.tar.gz + +COPY vscode-tunnel vscode-tunnel + +RUN mkdir /vscode-tunnel/logs \ + && mkdir /vscode-tunnel/docs \ + && chmod -R 777 vscode-tunnel + +RUN cp vscode-tunnel/scripts/start.sh /usr/bin/start-tunnel \ + && cp vscode-tunnel/scripts/stop.sh /usr/bin/stop-tunnel \ + && cp vscode-tunnel/scripts/refresh.sh /usr/bin/restart-tunnel \ + && cp vscode-tunnel/scripts/status.sh /usr/bin/tunnel-status \ + && chmod +x /usr/bin/start-tunnel /usr/bin/stop-tunnel /usr/bin/restart-tunnel /usr/bin/tunnel-status + +COPY TUNNELING.md /vscode-tunnel/docs/README.md + +RUN git config --global credential.helper store \ + && git config --global core.filemode false + +# Install Projector & PyCharm Community +COPY projector-proxy projector-proxy +RUN pip install --no-cache-dir projector-installer==1.7.0 --upgrade \ + && projector --accept-license --config-directory /usr/local/share/projector \ + ide autoinstall --config-name pycharm --ide-name "PyCharm Community Edition 2021.3.1" \ + && chmod -R a=rwx /usr/local/share/projector \ + && pip install --no-cache-dir ./projector-proxy + +# force shell to bash as Peak Platform isn't respecting jupyterhub option +RUN ln -sf /bin/bash /bin/sh + +EXPOSE 8000 +CMD ["jupyterhub", \ + "--Spawner.default_url=/lab", \ + "--Spawner.http_timeout=90", \ + "--NotebookApp.terminado_settings={'shell_command':'[/bin/bash]'}" \ + ] diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.10.13/README.md b/workspace/python/ds-pack/2.1.0-base-python-3.10.13/README.md new file mode 100644 index 00000000..302ba30a --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.10.13/README.md @@ -0,0 +1,79 @@ +# workspace-python-ds-pack-2.1.0-base-python-3.10.13 +This is the Python Data Science pack image made for Workspaces. +The image runs a [Jupyterhub](https://jupyter.org/hub) server that has [PyCharm](https://lp.jetbrains.com/projector/) and [VSCode](https://github.com/coder/code-server) pre-installed with all of the essential and most used packages. +The images also comes pre installed with [peak-sdk](https://docs.peak.ai/sdk/). + +## Image details +### Base image +This image uses [3.10.13-slim-bookworm](https://hub.docker.com/layers/library/python/3.10.13-slim-bookworm/images/sha256-fb6e9cde0d7ae6ea8cd516a76ba78dad72e6a09465a8e93fe847d63fbcebc9a6?context=explore) as its base which is maintained by [the Docker Community](https://github.com/docker-library/python). + +### OS and other details +``` +Debian GNU/Linux 12 (bookworm) +Linux Kernel 5.10.186-179.751.amzn2.x86_64 +Python 3.10.13 +``` + +### Important Linux packages installed +``` +aws-cli 2.7.4 +curl 7.88.1 +git 2.39.2 +jq 1.6 +nano 7.2 +vim 9.0 +fish 3.6.0 +zsh 5.9 +R 4.3.1 +node 18.17.1 +docker 24.0.6 +htop 3.2.0 +pandoc 2.17.1.1 +less 590 +latex 3.141592653-2.6-1.40.24 +vscode-cli 1.82.2 +``` + +### Python libraries installed +``` +ipywidgets 8.1.0 +jupyter-server-proxy 4.0.0 +jupyterhub 4.0.2 +jupyterlab-git 0.42.0 +jupyterlab-lsp 4.2.0 +jupyterlab 3.6.5 +jupyterlab_widgets 3.0.8 +jupytext 1.15.1 +lckr-jupyterlab-variableinspector 3.0.9 +mypy-ls 0.5.1 +nbconvert 7.8.0 +notebook 6.5.5 +pyls-black 0.4.7 +pyls-flake 80.4.0 +pyls-isort 0.2.2 +pyls-mypy 0.1.8 +python-lsp-black 1.3.0 +python-lsp-server[all] 1.4.1 +virtualenv 20.24.5 +peak-sdk 1.0.0 +jupysql 0.10.1 +``` + +### VS Code extensions installed +``` +cweijan.vscode-database-client2 6.6.3 +``` + +### VS Code Tunneling + +The image supports creating [remote tunnels](https://code.visualstudio.com/docs/remote/tunnels). For simplicity the image comes in handy with some node/bash scripts which can be used to `initialise`, `start`, `stop`, and `restart` the remote tunnels. More info can be found [here](./TUNNELING.md). + +## Building the image +To build the image locally, run the docker build command and pass in the required build arguments: +``` +docker build . -t workflow-python-ds-pack-2.1.0-base-python-3.10.13 +``` + +## Using the image +To use the image, select it when configuring the Workspace. +If you need to install additional dependencies or add some use case specific environment variables, it can be easily extended. diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.10.13/TUNNELING.md b/workspace/python/ds-pack/2.1.0-base-python-3.10.13/TUNNELING.md new file mode 100644 index 00000000..32f42936 --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.10.13/TUNNELING.md @@ -0,0 +1,65 @@ +# Tunneling with VS Code + +This README provides instructions for setting up and managing remote tunnels using this image, making it easier for you to access your workspace remotely within VS Code. + +## What is VS Code Tunneling ? + +VS Code Tunneling is a feature that allows users to connect to a remote machine, such as a desktop PC or virtual machine, via a secure tunnel. This feature securely transmits data from one network to another, allowing users to develop against any machine of their choosing from a VS Code desktop or web client without the need for SSH or HTTPS setup. For more information, please refer (this)[https://code.visualstudio.com/docs/remote/tunnels] link. + +## Initializing a Remote Tunnel + +To create a new remote tunnel for your workspace, you can use the provided bash script: + +```bash +start-tunnel +``` + +Running this script initializes and starts a new tunnel in the background, which can be accessed remotely. To ensure its functionality, a brief authentication step using GitHub is required. + +After executing the above command, you will see the following message in the console: + +``` +To grant access to the server, please log into https://github.com/login/device and use code +``` + +Follow the link provided and enter the `` displayed in the console. + +That's it! Your new, secure tunnel should now be active and accessible at the following URL: `https://vscode.dev/tunnel/workspace--0` + +## Managing Tunnels + +### Restarting a Tunnel + +If you need to restart your tunnel for smooth operation, you can use the following command: + +```bash +restart-tunnel +``` + +### Stopping a Tunnel + +To stop an active tunnel, use the following command: + +```bash +stop-tunnel +``` + +### Starting a Tunnel + +To start a previously stopped tunnel, use this command: + +```bash +start-tunnel +``` + +### Verifying the status of tunnel + +To check the status of the tunnel, use this command: + +```bash +tunnel-status +``` + +## Additional Notes + +To view the logs of any running or closed tunnel, you can refer to the `output.log` file located at `/vscode-tunnel/logs/output.log`. Similarly, any errors related to running or closed tunnels can be found in the `/vscode-tunnel/logs/error.log` file. diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.10.13/codeserver-proxy/jupyter_codeserver_proxy/__init__.py b/workspace/python/ds-pack/2.1.0-base-python-3.10.13/codeserver-proxy/jupyter_codeserver_proxy/__init__.py new file mode 100644 index 00000000..9739bd3a --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.10.13/codeserver-proxy/jupyter_codeserver_proxy/__init__.py @@ -0,0 +1,28 @@ +import os +import subprocess + +bash_command = "code-server --install-extension cweijan.vscode-database-client2@6.6.3" + +def setup_codeserver(): + _install_vscode_extension() + return { + 'command': ['code-server', + '--auth', + 'none', + '--disable-telemetry', + '--port={port}'], + 'launcher_entry': { + 'title': 'VS Code', + 'icon_path': os.path.join(os.path.dirname(os.path.abspath(__file__)), 'icons', 'vs_code_icon.svg') + } + } + +def _install_vscode_extension(): + # Run the Bash command and capture the output + try: + result = subprocess.check_output(bash_command, shell=True, text=True) + print("Bash command output:") + print(result) + except subprocess.CalledProcessError as e: + print("Error running the Bash command:") + print(e) diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.10.13/codeserver-proxy/jupyter_codeserver_proxy/icons/vs_code_icon.svg b/workspace/python/ds-pack/2.1.0-base-python-3.10.13/codeserver-proxy/jupyter_codeserver_proxy/icons/vs_code_icon.svg new file mode 100644 index 00000000..37cf0a52 --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.10.13/codeserver-proxy/jupyter_codeserver_proxy/icons/vs_code_icon.svg @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.10.13/codeserver-proxy/setup.py b/workspace/python/ds-pack/2.1.0-base-python-3.10.13/codeserver-proxy/setup.py new file mode 100644 index 00000000..644aa275 --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.10.13/codeserver-proxy/setup.py @@ -0,0 +1,18 @@ +import setuptools + +setuptools.setup( + name="jupyter-codeserver-server", + version="1.0", + packages=setuptools.find_packages(), + keywords=['Jupyter'], + classifiers=['Framework :: Jupyter'], + install_requires=['jupyter-server-proxy'], + entry_points={ + 'jupyter_serverproxy_servers': [ + 'codeserver = jupyter_codeserver_proxy:setup_codeserver', + ] + }, + package_data={ + 'jupyter_codeserver_proxy': ['icons/*'] + } +) diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.10.13/projector-proxy/jupyter_projector_proxy/__init__.py b/workspace/python/ds-pack/2.1.0-base-python-3.10.13/projector-proxy/jupyter_projector_proxy/__init__.py new file mode 100644 index 00000000..da8ef670 --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.10.13/projector-proxy/jupyter_projector_proxy/__init__.py @@ -0,0 +1,41 @@ +import os +import shutil + +SYS_PROJECTOR_CONFIG_DIR = "/usr/local/share/projector" +USER_PROJECTOR_CONFIG_DIR = os.path.join(os.environ["HOME"], ".projector") + + +def setup_projector(): + _copy_projector_config() + return { + "command": ["projector", "run", "pycharm"], + "launcher_entry": { + "title": "PyCharm", + "icon_path": os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "icons", + "pycharm_icon.svg", + ), + }, + # default projector config runs on port 9999 + "port": 9999, + # set timeout for projector to become responsive to 60s, as it can sometimes + # take a while + "timeout": 60, + } + + +def _copy_projector_config(): + """ + Copy the projector config folder (containing a pre-downloaded copy of PyCharm + Community) from the system location in /usr/local/share to the user's home + directory. + + Project by default looks for config files in ~/.projector. Copying config folder + from /usr/local/share/ means that each user does not individually have to install + Projector and PyCharm, but can also add their own configurations if they want to + install other Jetbrains IDEs. + """ + + if not os.path.exists(USER_PROJECTOR_CONFIG_DIR): + shutil.copytree(SYS_PROJECTOR_CONFIG_DIR, USER_PROJECTOR_CONFIG_DIR) diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.10.13/projector-proxy/jupyter_projector_proxy/icons/pycharm_icon.svg b/workspace/python/ds-pack/2.1.0-base-python-3.10.13/projector-proxy/jupyter_projector_proxy/icons/pycharm_icon.svg new file mode 100644 index 00000000..694bce26 --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.10.13/projector-proxy/jupyter_projector_proxy/icons/pycharm_icon.svg @@ -0,0 +1,67 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + PyCharm_comm_64 + + + + + + + + + + + + + + + + diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.10.13/projector-proxy/setup.py b/workspace/python/ds-pack/2.1.0-base-python-3.10.13/projector-proxy/setup.py new file mode 100644 index 00000000..fa667057 --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.10.13/projector-proxy/setup.py @@ -0,0 +1,16 @@ +import setuptools + +setuptools.setup( + name="jupyter-projector-server", + version="1.0", + packages=setuptools.find_packages(), + keywords=["Jupyter"], + classifiers=["Framework :: Jupyter"], + install_requires=["jupyter-server-proxy"], + entry_points={ + "jupyter_serverproxy_servers": [ + "projector = jupyter_projector_proxy:setup_projector", + ] + }, + package_data={"jupyter_projector_proxy": ["icons/*"]}, +) diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.10.13/requirements.txt b/workspace/python/ds-pack/2.1.0-base-python-3.10.13/requirements.txt new file mode 100644 index 00000000..e758edd4 --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.10.13/requirements.txt @@ -0,0 +1,21 @@ +ipywidgets==8.1.0 +jupyter-server-proxy==4.0.0 +jupyterhub==4.0.2 +jupyterlab-git==0.42.0 +jupyterlab-lsp==4.2.0 +jupyterlab==3.6.5 +jupyterlab_widgets==3.0.8 +jupytext==1.15.1 +lckr-jupyterlab-variableinspector==3.0.9 +mypy-ls==0.5.1 +nbconvert==7.8.0 +notebook==6.5.5 +pyls-black==0.4.7 +pyls-flake8==0.4.0 +pyls-isort==0.2.2 +pyls-mypy==0.1.8 +python-lsp-black==1.3.0 +python-lsp-server[all]==1.4.1 +virtualenv==20.23.0 +peak-sdk==1.0.0 +jupysql==0.10.1 \ No newline at end of file diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.10.13/spark/config.ini b/workspace/python/ds-pack/2.1.0-base-python-3.10.13/spark/config.ini new file mode 100644 index 00000000..2eb378fd --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.10.13/spark/config.ini @@ -0,0 +1,3 @@ +[spark.credentials] +WORKSPACEID= +TENANTCLUSTER= diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.10.13/vscode-tunnel/scripts/init-or-start.js b/workspace/python/ds-pack/2.1.0-base-python-3.10.13/vscode-tunnel/scripts/init-or-start.js new file mode 100644 index 00000000..d214581c --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.10.13/vscode-tunnel/scripts/init-or-start.js @@ -0,0 +1,111 @@ +/** + * The script aims to setup a vscode tunnel for the workspace. It does by starting a tunnel in the background. + * Usage - node /vscode-tunnel/scripts setup.js + */ +const { spawn } = require('child_process'); +const fs = require('fs'); + +const outputLogFilePath = '/vscode-tunnel/logs/output.log'; +const errorLogFilePath = '/vscode-tunnel/logs/error.log'; +const scriptPath = '/vscode-tunnel/scripts/tunnel.sh'; + +function sleep(milliseconds) { + const start = new Date().getTime(); + while (new Date().getTime() - start < milliseconds); +} + +function waitForDeviceVerification() { + while (true) { + console.log('Waiting for device verification...'); + try { + const data = fs.readFileSync(outputLogFilePath, 'utf8'); + const index = data.lastIndexOf('Open this link in your browser'); + if (index >= 0) { + console.log(data.substring(index)); + break; + } + // check for errors if any + const errors = fs.readFileSync(errorLogFilePath, 'utf8'); + if (errors.length) { + console.error('There was an error creating the tunnel.'); + console.error(error); + break; + } + sleep(4000); + } catch (error) { + if (error.code === 'ENOENT') { + console.log(`File '${fileName}' not found.`); + } else { + console.error(`An error occurred: ${err}`); + } + break; + } + } +} + +function printOutput() { + // Read the file and print its contents + try { + const data = fs.readFileSync(outputLogFilePath, 'utf8'); + const index = data.lastIndexOf('Visual Studio Code Server'); + if (index >= 0) { + console.log(data.substring(index)); + } else { + console.log(data); + } + } catch (error) { + if (error.code === 'ENOENT') { + console.log(`File '${fileName}' not found.`); + } else { + console.error(`An error occurred: ${err}`); + } + } + + try { + const data = fs.readFileSync(errorLogFilePath, 'utf8'); + if (data.length) { + console.error('There was an error creating the tunnel.'); + console.error(data); + return false; + } + } catch (error) { + if (error.code === 'ENOENT') { + console.log(`File '${fileName}' not found.`); + } else { + console.error(`An error occurred: ${err}`); + } + } + + return true; +} + +function main() { + + // Open a file to store the child process's output + const outputLogFile = fs.openSync(outputLogFilePath, 'a'); + const errorLogFile = fs.openSync(errorLogFilePath, 'a'); + + // Spawn a child process to run the shell script with stdin, stdout, and stderr redirected + const childProcess = spawn('bash', [scriptPath], { + detached: true, + stdio: ['pipe', outputLogFile, errorLogFile], // Redirect output to log files + }); + + // Close the file descriptors to release them + fs.closeSync(outputLogFile); + fs.closeSync(errorLogFile); + + console.log('Starting tunnel...'); + // initial wait + sleep(3000); + const result = printOutput(); + + if (result) { + waitForDeviceVerification(); + } + + // Unref the child process to allow the Node.js process to exit + childProcess.unref(); +} + +main(); diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.10.13/vscode-tunnel/scripts/refresh.sh b/workspace/python/ds-pack/2.1.0-base-python-3.10.13/vscode-tunnel/scripts/refresh.sh new file mode 100644 index 00000000..dd1f76cd --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.10.13/vscode-tunnel/scripts/refresh.sh @@ -0,0 +1 @@ +code tunnel restart diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.10.13/vscode-tunnel/scripts/start.sh b/workspace/python/ds-pack/2.1.0-base-python-3.10.13/vscode-tunnel/scripts/start.sh new file mode 100644 index 00000000..233a2e6e --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.10.13/vscode-tunnel/scripts/start.sh @@ -0,0 +1 @@ +node /vscode-tunnel/scripts/init-or-start.js diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.10.13/vscode-tunnel/scripts/status.sh b/workspace/python/ds-pack/2.1.0-base-python-3.10.13/vscode-tunnel/scripts/status.sh new file mode 100644 index 00000000..4d8b473b --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.10.13/vscode-tunnel/scripts/status.sh @@ -0,0 +1 @@ +code tunnel status diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.10.13/vscode-tunnel/scripts/stop.sh b/workspace/python/ds-pack/2.1.0-base-python-3.10.13/vscode-tunnel/scripts/stop.sh new file mode 100644 index 00000000..35bca6ab --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.10.13/vscode-tunnel/scripts/stop.sh @@ -0,0 +1 @@ +code tunnel kill diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.10.13/vscode-tunnel/scripts/tunnel.sh b/workspace/python/ds-pack/2.1.0-base-python-3.10.13/vscode-tunnel/scripts/tunnel.sh new file mode 100644 index 00000000..996e7239 --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.10.13/vscode-tunnel/scripts/tunnel.sh @@ -0,0 +1 @@ +code tunnel diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.11.5/Dockerfile b/workspace/python/ds-pack/2.1.0-base-python-3.11.5/Dockerfile new file mode 100644 index 00000000..78e08e4e --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.11.5/Dockerfile @@ -0,0 +1,121 @@ +FROM python:3.11.5-slim-bookworm + +# Set shell to be bash and fail any pipe if any step in the pipe fails +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +# install system packages +RUN apt-get update \ + # installation requirements + && apt-get install -y --no-install-recommends \ + apt-transport-https \ + ca-certificates \ + software-properties-common \ + curl \ + gnupg2 \ + # docker + && mkdir -p /etc/apt/keyrings \ + && curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg \ + && echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \ + $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null \ + && apt-key adv --keyserver keyserver.ubuntu.com --recv-key '95C0FAF38DB3CCAD0C080A7BDC78B2DDEABC47B7' \ + && add-apt-repository 'deb http://cloud.r-project.org/bin/linux/debian bookworm-cran40/' \ + # node + && curl -sL https://deb.nodesource.com/setup_lts.x | bash - \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + # install system packages + # jq is needed for the Peak Platform + jq \ + nano \ + vim \ + fish \ + zsh \ + r-base=4.2.2.20221110-2 \ + r-recommended=4.2.2.20221110-2 \ + build-essential \ + sudo \ + nodejs \ + git \ + htop \ + pandoc \ + texlive \ + texlive-science \ + texlive-latex-extra \ + texlive-luatex \ + texlive-xetex \ + texlive-pictures \ + latexmk \ + lmodern \ + docker-ce \ + less \ + libxext6 \ + libxrender1 \ + libxtst6 \ + libfreetype6 \ + libxi6 \ + openssl \ + unzip \ + && rm -rf /var/lib/apt/lists/* + +# install AWS CLI v2 +RUN curl -fsSL "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.7.4.zip" -o "awscliv2.zip" \ + && unzip -q awscliv2.zip \ + && ./aws/install \ + && rm -rf aws awscliv2.zip + +# Needed for jupyterhub +RUN npm install -g configurable-http-proxy \ + && rm -rf ~/.npm + +# Install Python packages +COPY requirements.txt /tmp/requirements.txt + +RUN pip install --no-cache-dir --upgrade \ + pip \ + setuptools \ + wheel + +RUN pip install -r /tmp/requirements.txt + +# SPARK: copy over spark config +COPY ./spark /usr/local/spark + +# VS Code +COPY codeserver-proxy codeserver-proxy +RUN curl -fsSL https://code-server.dev/install.sh \ + | sh -s -- --version 4.16.1 \ + && pip install --no-cache-dir ./codeserver-proxy + +# vscode cli +RUN curl -Lk 'https://code.visualstudio.com/sha/download?build=stable&os=cli-alpine-x64' --output vscode_cli.tar.gz \ + && tar -xf vscode_cli.tar.gz \ + && cp ./code /usr/local/bin \ + && rm -rf vscode_cli.tar.gz + +COPY vscode-tunnel vscode-tunnel + +RUN mkdir /vscode-tunnel/logs \ + && mkdir /vscode-tunnel/docs \ + && chmod -R 777 vscode-tunnel + +RUN cp vscode-tunnel/scripts/start.sh /usr/bin/start-tunnel \ + && cp vscode-tunnel/scripts/stop.sh /usr/bin/stop-tunnel \ + && cp vscode-tunnel/scripts/refresh.sh /usr/bin/restart-tunnel \ + && cp vscode-tunnel/scripts/status.sh /usr/bin/tunnel-status \ + && chmod +x /usr/bin/start-tunnel /usr/bin/stop-tunnel /usr/bin/restart-tunnel /usr/bin/tunnel-status + +COPY TUNNELING.md /vscode-tunnel/docs/README.md + +RUN git config --global credential.helper store \ + && git config --global core.filemode false + +# force shell to bash as Peak Platform isn't respecting jupyterhub option +RUN ln -sf /bin/bash /bin/sh + +EXPOSE 8000 +CMD ["jupyterhub", \ + "--Spawner.default_url=/lab", \ + "--Spawner.http_timeout=90", \ + "--NotebookApp.terminado_settings={'shell_command':'[/bin/bash]'}" \ + ] diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.11.5/README.md b/workspace/python/ds-pack/2.1.0-base-python-3.11.5/README.md new file mode 100644 index 00000000..a5fbb0d0 --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.11.5/README.md @@ -0,0 +1,81 @@ +# workspace-python-ds-pack-2.1.0-base-python-3.11.5 +This is the Python Data Science pack image made for Workspaces. +The image runs a [Jupyterhub](https://jupyter.org/hub) server that has [VSCode](https://github.com/coder/code-server) pre-installed with all of the essential and most used packages. +The images also comes pre installed with [peak-sdk](https://docs.peak.ai/sdk/). + +Note: The image **doesn't support** [PyCharm](https://lp.jetbrains.com/projector/) unlike previous Python workspace image versions. + +## Image details +### Base image +This image uses [3.11.5-slim-bookworm](https://hub.docker.com/layers/library/python/3.11.5-slim-bookworm/images/sha256-948a09a8f42c0b2f4308ef563dc3b76de59ebfc5d83ccb7ed7a0fbbb6c5b4713?context=explore) as its base which is maintained by [the Docker Community](https://github.com/docker-library/python). + +### OS and other details +``` +Debian GNU/Linux 12 (bookworm) +Linux Kernel 5.10.186-179.751.amzn2.x86_64 +Python 3.11.5 +``` + +### VS Code extensions installed +``` +cweijan.vscode-database-client2 6.6.3 +``` + +### Important Linux packages installed +``` +aws-cli 2.7.4 +curl 7.88.1 +git 2.39.2 +jq 1.6 +nano 7.2 +vim 9.0 +fish 3.6.0 +zsh 5.9 +R 4.3.1 +node 18.17.1 +docker 24.0.6 +htop 3.2.0 +pandoc 2.17.1.1 +less 590 +latex 3.141592653-2.6-1.40.24 +vscode-cli 1.82.2 +``` + +### Python libraries installed +``` +ipywidgets 8.1.0 +jupyter-server-proxy 4.0.0 +jupyterhub 4.0.2 +jupyterlab-git 0.42.0 +jupyterlab-lsp 4.2.0 +jupyterlab 3.6.5 +jupyterlab_widgets 3.0.8 +jupytext 1.15.1 +lckr-jupyterlab-variableinspector 3.0.9 +mypy-ls 0.5.1 +nbconvert 7.8.0 +notebook 6.5.5 +pyls-black 0.4.7 +pyls-flake 80.4.0 +pyls-isort 0.2.2 +pyls-mypy 0.1.8 +python-lsp-black 1.3.0 +python-lsp-server[all] 1.4.1 +virtualenv 20.24.5 +peak-sdk 1.0.0 +jupysql 0.10.1 +``` + +### VS Code Tunneling + +The image supports creating [remote tunnels](https://code.visualstudio.com/docs/remote/tunnels). For simplicity the image comes in handy with some node/bash scripts which can be used to `initialise`, `start`, `stop`, and `restart` the remote tunnels. More info can be found [here](./TUNNELING.md). + +## Building the image +To build the image locally, run the docker build command and pass in the required build arguments: +``` +docker build . -t workflow-python-ds-pack-2.1.0-base-python-3.11.5 +``` + +## Using the image +To use the image, select it when configuring the Workspace. +If you need to install additional dependencies or add some use case specific environment variables, it can be easily extended. diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.11.5/TUNNELING.md b/workspace/python/ds-pack/2.1.0-base-python-3.11.5/TUNNELING.md new file mode 100644 index 00000000..32f42936 --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.11.5/TUNNELING.md @@ -0,0 +1,65 @@ +# Tunneling with VS Code + +This README provides instructions for setting up and managing remote tunnels using this image, making it easier for you to access your workspace remotely within VS Code. + +## What is VS Code Tunneling ? + +VS Code Tunneling is a feature that allows users to connect to a remote machine, such as a desktop PC or virtual machine, via a secure tunnel. This feature securely transmits data from one network to another, allowing users to develop against any machine of their choosing from a VS Code desktop or web client without the need for SSH or HTTPS setup. For more information, please refer (this)[https://code.visualstudio.com/docs/remote/tunnels] link. + +## Initializing a Remote Tunnel + +To create a new remote tunnel for your workspace, you can use the provided bash script: + +```bash +start-tunnel +``` + +Running this script initializes and starts a new tunnel in the background, which can be accessed remotely. To ensure its functionality, a brief authentication step using GitHub is required. + +After executing the above command, you will see the following message in the console: + +``` +To grant access to the server, please log into https://github.com/login/device and use code +``` + +Follow the link provided and enter the `` displayed in the console. + +That's it! Your new, secure tunnel should now be active and accessible at the following URL: `https://vscode.dev/tunnel/workspace--0` + +## Managing Tunnels + +### Restarting a Tunnel + +If you need to restart your tunnel for smooth operation, you can use the following command: + +```bash +restart-tunnel +``` + +### Stopping a Tunnel + +To stop an active tunnel, use the following command: + +```bash +stop-tunnel +``` + +### Starting a Tunnel + +To start a previously stopped tunnel, use this command: + +```bash +start-tunnel +``` + +### Verifying the status of tunnel + +To check the status of the tunnel, use this command: + +```bash +tunnel-status +``` + +## Additional Notes + +To view the logs of any running or closed tunnel, you can refer to the `output.log` file located at `/vscode-tunnel/logs/output.log`. Similarly, any errors related to running or closed tunnels can be found in the `/vscode-tunnel/logs/error.log` file. diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.11.5/codeserver-proxy/jupyter_codeserver_proxy/__init__.py b/workspace/python/ds-pack/2.1.0-base-python-3.11.5/codeserver-proxy/jupyter_codeserver_proxy/__init__.py new file mode 100644 index 00000000..9739bd3a --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.11.5/codeserver-proxy/jupyter_codeserver_proxy/__init__.py @@ -0,0 +1,28 @@ +import os +import subprocess + +bash_command = "code-server --install-extension cweijan.vscode-database-client2@6.6.3" + +def setup_codeserver(): + _install_vscode_extension() + return { + 'command': ['code-server', + '--auth', + 'none', + '--disable-telemetry', + '--port={port}'], + 'launcher_entry': { + 'title': 'VS Code', + 'icon_path': os.path.join(os.path.dirname(os.path.abspath(__file__)), 'icons', 'vs_code_icon.svg') + } + } + +def _install_vscode_extension(): + # Run the Bash command and capture the output + try: + result = subprocess.check_output(bash_command, shell=True, text=True) + print("Bash command output:") + print(result) + except subprocess.CalledProcessError as e: + print("Error running the Bash command:") + print(e) diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.11.5/codeserver-proxy/jupyter_codeserver_proxy/icons/vs_code_icon.svg b/workspace/python/ds-pack/2.1.0-base-python-3.11.5/codeserver-proxy/jupyter_codeserver_proxy/icons/vs_code_icon.svg new file mode 100644 index 00000000..37cf0a52 --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.11.5/codeserver-proxy/jupyter_codeserver_proxy/icons/vs_code_icon.svg @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.11.5/codeserver-proxy/setup.py b/workspace/python/ds-pack/2.1.0-base-python-3.11.5/codeserver-proxy/setup.py new file mode 100644 index 00000000..644aa275 --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.11.5/codeserver-proxy/setup.py @@ -0,0 +1,18 @@ +import setuptools + +setuptools.setup( + name="jupyter-codeserver-server", + version="1.0", + packages=setuptools.find_packages(), + keywords=['Jupyter'], + classifiers=['Framework :: Jupyter'], + install_requires=['jupyter-server-proxy'], + entry_points={ + 'jupyter_serverproxy_servers': [ + 'codeserver = jupyter_codeserver_proxy:setup_codeserver', + ] + }, + package_data={ + 'jupyter_codeserver_proxy': ['icons/*'] + } +) diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.11.5/requirements.txt b/workspace/python/ds-pack/2.1.0-base-python-3.11.5/requirements.txt new file mode 100644 index 00000000..e758edd4 --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.11.5/requirements.txt @@ -0,0 +1,21 @@ +ipywidgets==8.1.0 +jupyter-server-proxy==4.0.0 +jupyterhub==4.0.2 +jupyterlab-git==0.42.0 +jupyterlab-lsp==4.2.0 +jupyterlab==3.6.5 +jupyterlab_widgets==3.0.8 +jupytext==1.15.1 +lckr-jupyterlab-variableinspector==3.0.9 +mypy-ls==0.5.1 +nbconvert==7.8.0 +notebook==6.5.5 +pyls-black==0.4.7 +pyls-flake8==0.4.0 +pyls-isort==0.2.2 +pyls-mypy==0.1.8 +python-lsp-black==1.3.0 +python-lsp-server[all]==1.4.1 +virtualenv==20.23.0 +peak-sdk==1.0.0 +jupysql==0.10.1 \ No newline at end of file diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.11.5/spark/config.ini b/workspace/python/ds-pack/2.1.0-base-python-3.11.5/spark/config.ini new file mode 100644 index 00000000..2eb378fd --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.11.5/spark/config.ini @@ -0,0 +1,3 @@ +[spark.credentials] +WORKSPACEID= +TENANTCLUSTER= diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.11.5/vscode-tunnel/scripts/init-or-start.js b/workspace/python/ds-pack/2.1.0-base-python-3.11.5/vscode-tunnel/scripts/init-or-start.js new file mode 100644 index 00000000..d214581c --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.11.5/vscode-tunnel/scripts/init-or-start.js @@ -0,0 +1,111 @@ +/** + * The script aims to setup a vscode tunnel for the workspace. It does by starting a tunnel in the background. + * Usage - node /vscode-tunnel/scripts setup.js + */ +const { spawn } = require('child_process'); +const fs = require('fs'); + +const outputLogFilePath = '/vscode-tunnel/logs/output.log'; +const errorLogFilePath = '/vscode-tunnel/logs/error.log'; +const scriptPath = '/vscode-tunnel/scripts/tunnel.sh'; + +function sleep(milliseconds) { + const start = new Date().getTime(); + while (new Date().getTime() - start < milliseconds); +} + +function waitForDeviceVerification() { + while (true) { + console.log('Waiting for device verification...'); + try { + const data = fs.readFileSync(outputLogFilePath, 'utf8'); + const index = data.lastIndexOf('Open this link in your browser'); + if (index >= 0) { + console.log(data.substring(index)); + break; + } + // check for errors if any + const errors = fs.readFileSync(errorLogFilePath, 'utf8'); + if (errors.length) { + console.error('There was an error creating the tunnel.'); + console.error(error); + break; + } + sleep(4000); + } catch (error) { + if (error.code === 'ENOENT') { + console.log(`File '${fileName}' not found.`); + } else { + console.error(`An error occurred: ${err}`); + } + break; + } + } +} + +function printOutput() { + // Read the file and print its contents + try { + const data = fs.readFileSync(outputLogFilePath, 'utf8'); + const index = data.lastIndexOf('Visual Studio Code Server'); + if (index >= 0) { + console.log(data.substring(index)); + } else { + console.log(data); + } + } catch (error) { + if (error.code === 'ENOENT') { + console.log(`File '${fileName}' not found.`); + } else { + console.error(`An error occurred: ${err}`); + } + } + + try { + const data = fs.readFileSync(errorLogFilePath, 'utf8'); + if (data.length) { + console.error('There was an error creating the tunnel.'); + console.error(data); + return false; + } + } catch (error) { + if (error.code === 'ENOENT') { + console.log(`File '${fileName}' not found.`); + } else { + console.error(`An error occurred: ${err}`); + } + } + + return true; +} + +function main() { + + // Open a file to store the child process's output + const outputLogFile = fs.openSync(outputLogFilePath, 'a'); + const errorLogFile = fs.openSync(errorLogFilePath, 'a'); + + // Spawn a child process to run the shell script with stdin, stdout, and stderr redirected + const childProcess = spawn('bash', [scriptPath], { + detached: true, + stdio: ['pipe', outputLogFile, errorLogFile], // Redirect output to log files + }); + + // Close the file descriptors to release them + fs.closeSync(outputLogFile); + fs.closeSync(errorLogFile); + + console.log('Starting tunnel...'); + // initial wait + sleep(3000); + const result = printOutput(); + + if (result) { + waitForDeviceVerification(); + } + + // Unref the child process to allow the Node.js process to exit + childProcess.unref(); +} + +main(); diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.11.5/vscode-tunnel/scripts/refresh.sh b/workspace/python/ds-pack/2.1.0-base-python-3.11.5/vscode-tunnel/scripts/refresh.sh new file mode 100644 index 00000000..dd1f76cd --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.11.5/vscode-tunnel/scripts/refresh.sh @@ -0,0 +1 @@ +code tunnel restart diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.11.5/vscode-tunnel/scripts/start.sh b/workspace/python/ds-pack/2.1.0-base-python-3.11.5/vscode-tunnel/scripts/start.sh new file mode 100644 index 00000000..233a2e6e --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.11.5/vscode-tunnel/scripts/start.sh @@ -0,0 +1 @@ +node /vscode-tunnel/scripts/init-or-start.js diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.11.5/vscode-tunnel/scripts/status.sh b/workspace/python/ds-pack/2.1.0-base-python-3.11.5/vscode-tunnel/scripts/status.sh new file mode 100644 index 00000000..4d8b473b --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.11.5/vscode-tunnel/scripts/status.sh @@ -0,0 +1 @@ +code tunnel status diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.11.5/vscode-tunnel/scripts/stop.sh b/workspace/python/ds-pack/2.1.0-base-python-3.11.5/vscode-tunnel/scripts/stop.sh new file mode 100644 index 00000000..35bca6ab --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.11.5/vscode-tunnel/scripts/stop.sh @@ -0,0 +1 @@ +code tunnel kill diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.11.5/vscode-tunnel/scripts/tunnel.sh b/workspace/python/ds-pack/2.1.0-base-python-3.11.5/vscode-tunnel/scripts/tunnel.sh new file mode 100644 index 00000000..996e7239 --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.11.5/vscode-tunnel/scripts/tunnel.sh @@ -0,0 +1 @@ +code tunnel diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.8.18/Dockerfile b/workspace/python/ds-pack/2.1.0-base-python-3.8.18/Dockerfile new file mode 100644 index 00000000..d487f1a3 --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.8.18/Dockerfile @@ -0,0 +1,129 @@ +FROM python:3.8.18-slim-bookworm + +# Set shell to be bash and fail any pipe if any step in the pipe fails +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +# install system packages +RUN apt-get update \ + # installation requirements + && apt-get install -y --no-install-recommends \ + apt-transport-https \ + ca-certificates \ + software-properties-common \ + curl \ + gnupg2 \ + # docker + && mkdir -p /etc/apt/keyrings \ + && curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg \ + && echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \ + $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null \ + && apt-key adv --keyserver keyserver.ubuntu.com --recv-key '95C0FAF38DB3CCAD0C080A7BDC78B2DDEABC47B7' \ + && add-apt-repository 'deb http://cloud.r-project.org/bin/linux/debian bookworm-cran40/' \ + # node + && curl -sL https://deb.nodesource.com/setup_lts.x | bash - \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + # install system packages + # jq is needed for the Peak Platform + jq \ + nano \ + vim \ + fish \ + zsh \ + r-base=4.2.2.20221110-2 \ + r-recommended=4.2.2.20221110-2 \ + build-essential \ + sudo \ + nodejs \ + git \ + htop \ + pandoc \ + texlive \ + texlive-science \ + texlive-latex-extra \ + texlive-luatex \ + texlive-xetex \ + texlive-pictures \ + latexmk \ + lmodern \ + docker-ce \ + less \ + libxext6 \ + libxrender1 \ + libxtst6 \ + libfreetype6 \ + libxi6 \ + openssl \ + unzip \ + && rm -rf /var/lib/apt/lists/* + +# install AWS CLI v2 +RUN curl -fsSL "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.7.4.zip" -o "awscliv2.zip" \ + && unzip -q awscliv2.zip \ + && ./aws/install \ + && rm -rf aws awscliv2.zip + +# Needed for jupyterhub +RUN npm install -g configurable-http-proxy \ + && rm -rf ~/.npm + +# Install Python packages +COPY requirements.txt /tmp/requirements.txt + +RUN pip install --no-cache-dir --upgrade \ + pip \ + setuptools \ + wheel + +RUN pip install -r /tmp/requirements.txt + +# SPARK: copy over spark config +COPY ./spark /usr/local/spark + +# VS Code +COPY codeserver-proxy codeserver-proxy +RUN curl -fsSL https://code-server.dev/install.sh \ + | sh -s -- --version 4.16.1 \ + && pip install --no-cache-dir ./codeserver-proxy + +# vscode cli +RUN curl -Lk 'https://code.visualstudio.com/sha/download?build=stable&os=cli-alpine-x64' --output vscode_cli.tar.gz \ + && tar -xf vscode_cli.tar.gz \ + && cp ./code /usr/local/bin \ + && rm -rf vscode_cli.tar.gz + +COPY vscode-tunnel vscode-tunnel + +RUN mkdir /vscode-tunnel/logs \ + && mkdir /vscode-tunnel/docs \ + && chmod -R 777 vscode-tunnel + +RUN cp vscode-tunnel/scripts/start.sh /usr/bin/start-tunnel \ + && cp vscode-tunnel/scripts/stop.sh /usr/bin/stop-tunnel \ + && cp vscode-tunnel/scripts/refresh.sh /usr/bin/restart-tunnel \ + && cp vscode-tunnel/scripts/status.sh /usr/bin/tunnel-status \ + && chmod +x /usr/bin/start-tunnel /usr/bin/stop-tunnel /usr/bin/restart-tunnel /usr/bin/tunnel-status + +COPY TUNNELING.md /vscode-tunnel/docs/README.md + +RUN git config --global credential.helper store \ + && git config --global core.filemode false + +# Install Projector & PyCharm Community +COPY projector-proxy projector-proxy +RUN pip install --no-cache-dir projector-installer==1.7.0 --upgrade \ + && projector --accept-license --config-directory /usr/local/share/projector \ + ide autoinstall --config-name pycharm --ide-name "PyCharm Community Edition 2021.3.1" \ + && chmod -R a=rwx /usr/local/share/projector \ + && pip install --no-cache-dir ./projector-proxy + +# force shell to bash as Peak Platform isn't respecting jupyterhub option +RUN ln -sf /bin/bash /bin/sh + +EXPOSE 8000 +CMD ["jupyterhub", \ + "--Spawner.default_url=/lab", \ + "--Spawner.http_timeout=90", \ + "--NotebookApp.terminado_settings={'shell_command':'[/bin/bash]'}" \ + ] diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.8.18/README.md b/workspace/python/ds-pack/2.1.0-base-python-3.8.18/README.md new file mode 100644 index 00000000..e85e7773 --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.8.18/README.md @@ -0,0 +1,79 @@ +# workspace-python-ds-pack-2.1.0-base-python-3.8.18 +This is the Python Data Science pack image made for Workspaces. +The image runs a [Jupyterhub](https://jupyter.org/hub) server that has [PyCharm](https://lp.jetbrains.com/projector/) and [VSCode](https://github.com/coder/code-server) pre-installed with all of the essential and most used packages. +The images also comes pre installed with [peak-sdk](https://docs.peak.ai/sdk/). + +## Image details +### Base image +This image uses [python:3.8.18-slim-bookworm](https://hub.docker.com/layers/library/python/3.8.18-slim-bookworm/images/sha256-05928e4bd868a725705a451f59eab0638a3509899099955bb326624a3f9bb597?context=explore) as its base which is maintained by [the Docker Community](https://github.com/docker-library/python). + +### OS and other details +``` +Debian GNU/Linux 12 (bookworm) +Linux Kernel 5.10.186-179.751.amzn2.x86_64 +Python 3.8.18 +``` + +### Important Linux packages installed +``` +aws-cli 2.7.4 +curl 7.88.1 +git 2.39.2 +jq 1.6 +nano 7.2 +vim 9.0 +fish 3.6.0 +zsh 5.9 +R 4.3.1 +node 18.17.1 +docker 24.0.6 +htop 3.2.0 +pandoc 2.17.1.1 +less 590 +latex 3.141592653-2.6-1.40.24 +vscode-cli 1.82.2 +``` + +### Python libraries installed +``` +ipywidgets 8.1.0 +jupyter-server-proxy 4.0.0 +jupyterhub 4.0.2 +jupyterlab-git 0.42.0 +jupyterlab-lsp 4.2.0 +jupyterlab 3.6.5 +jupyterlab_widgets 3.0.8 +jupytext 1.15.1 +lckr-jupyterlab-variableinspector 3.0.9 +mypy-ls 0.5.1 +nbconvert 7.8.0 +notebook 6.5.5 +pyls-black 0.4.7 +pyls-flake 80.4.0 +pyls-isort 0.2.2 +pyls-mypy 0.1.8 +python-lsp-black 1.3.0 +python-lsp-server[all] 1.4.1 +virtualenv 20.24.5 +peak-sdk 1.0.0 +jupysql 0.10.1 +``` + +### VS Code extensions installed +``` +cweijan.vscode-database-client2 6.6.3 +``` + +### VS Code Tunneling + +The image supports creating [remote tunnels](https://code.visualstudio.com/docs/remote/tunnels). For simplicity the image comes in handy with some node/bash scripts which can be used to `initialise`, `start`, `stop`, and `restart` the remote tunnels. More info can be found [here](./TUNNELING.md). + +## Building the image +To build the image locally, run the docker build command and pass in the required build arguments: +``` +docker build . -t workflow-python-ds-pack-2.1.0-base-python-3.8.18 +``` + +## Using the image +To use the image, select it when configuring the Workspace. +If you need to install additional dependencies or add some use case specific environment variables, it can be easily extended. diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.8.18/TUNNELING.md b/workspace/python/ds-pack/2.1.0-base-python-3.8.18/TUNNELING.md new file mode 100644 index 00000000..32f42936 --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.8.18/TUNNELING.md @@ -0,0 +1,65 @@ +# Tunneling with VS Code + +This README provides instructions for setting up and managing remote tunnels using this image, making it easier for you to access your workspace remotely within VS Code. + +## What is VS Code Tunneling ? + +VS Code Tunneling is a feature that allows users to connect to a remote machine, such as a desktop PC or virtual machine, via a secure tunnel. This feature securely transmits data from one network to another, allowing users to develop against any machine of their choosing from a VS Code desktop or web client without the need for SSH or HTTPS setup. For more information, please refer (this)[https://code.visualstudio.com/docs/remote/tunnels] link. + +## Initializing a Remote Tunnel + +To create a new remote tunnel for your workspace, you can use the provided bash script: + +```bash +start-tunnel +``` + +Running this script initializes and starts a new tunnel in the background, which can be accessed remotely. To ensure its functionality, a brief authentication step using GitHub is required. + +After executing the above command, you will see the following message in the console: + +``` +To grant access to the server, please log into https://github.com/login/device and use code +``` + +Follow the link provided and enter the `` displayed in the console. + +That's it! Your new, secure tunnel should now be active and accessible at the following URL: `https://vscode.dev/tunnel/workspace--0` + +## Managing Tunnels + +### Restarting a Tunnel + +If you need to restart your tunnel for smooth operation, you can use the following command: + +```bash +restart-tunnel +``` + +### Stopping a Tunnel + +To stop an active tunnel, use the following command: + +```bash +stop-tunnel +``` + +### Starting a Tunnel + +To start a previously stopped tunnel, use this command: + +```bash +start-tunnel +``` + +### Verifying the status of tunnel + +To check the status of the tunnel, use this command: + +```bash +tunnel-status +``` + +## Additional Notes + +To view the logs of any running or closed tunnel, you can refer to the `output.log` file located at `/vscode-tunnel/logs/output.log`. Similarly, any errors related to running or closed tunnels can be found in the `/vscode-tunnel/logs/error.log` file. diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.8.18/codeserver-proxy/jupyter_codeserver_proxy/__init__.py b/workspace/python/ds-pack/2.1.0-base-python-3.8.18/codeserver-proxy/jupyter_codeserver_proxy/__init__.py new file mode 100644 index 00000000..9739bd3a --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.8.18/codeserver-proxy/jupyter_codeserver_proxy/__init__.py @@ -0,0 +1,28 @@ +import os +import subprocess + +bash_command = "code-server --install-extension cweijan.vscode-database-client2@6.6.3" + +def setup_codeserver(): + _install_vscode_extension() + return { + 'command': ['code-server', + '--auth', + 'none', + '--disable-telemetry', + '--port={port}'], + 'launcher_entry': { + 'title': 'VS Code', + 'icon_path': os.path.join(os.path.dirname(os.path.abspath(__file__)), 'icons', 'vs_code_icon.svg') + } + } + +def _install_vscode_extension(): + # Run the Bash command and capture the output + try: + result = subprocess.check_output(bash_command, shell=True, text=True) + print("Bash command output:") + print(result) + except subprocess.CalledProcessError as e: + print("Error running the Bash command:") + print(e) diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.8.18/codeserver-proxy/jupyter_codeserver_proxy/icons/vs_code_icon.svg b/workspace/python/ds-pack/2.1.0-base-python-3.8.18/codeserver-proxy/jupyter_codeserver_proxy/icons/vs_code_icon.svg new file mode 100644 index 00000000..37cf0a52 --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.8.18/codeserver-proxy/jupyter_codeserver_proxy/icons/vs_code_icon.svg @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.8.18/codeserver-proxy/setup.py b/workspace/python/ds-pack/2.1.0-base-python-3.8.18/codeserver-proxy/setup.py new file mode 100644 index 00000000..644aa275 --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.8.18/codeserver-proxy/setup.py @@ -0,0 +1,18 @@ +import setuptools + +setuptools.setup( + name="jupyter-codeserver-server", + version="1.0", + packages=setuptools.find_packages(), + keywords=['Jupyter'], + classifiers=['Framework :: Jupyter'], + install_requires=['jupyter-server-proxy'], + entry_points={ + 'jupyter_serverproxy_servers': [ + 'codeserver = jupyter_codeserver_proxy:setup_codeserver', + ] + }, + package_data={ + 'jupyter_codeserver_proxy': ['icons/*'] + } +) diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.8.18/projector-proxy/jupyter_projector_proxy/__init__.py b/workspace/python/ds-pack/2.1.0-base-python-3.8.18/projector-proxy/jupyter_projector_proxy/__init__.py new file mode 100644 index 00000000..da8ef670 --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.8.18/projector-proxy/jupyter_projector_proxy/__init__.py @@ -0,0 +1,41 @@ +import os +import shutil + +SYS_PROJECTOR_CONFIG_DIR = "/usr/local/share/projector" +USER_PROJECTOR_CONFIG_DIR = os.path.join(os.environ["HOME"], ".projector") + + +def setup_projector(): + _copy_projector_config() + return { + "command": ["projector", "run", "pycharm"], + "launcher_entry": { + "title": "PyCharm", + "icon_path": os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "icons", + "pycharm_icon.svg", + ), + }, + # default projector config runs on port 9999 + "port": 9999, + # set timeout for projector to become responsive to 60s, as it can sometimes + # take a while + "timeout": 60, + } + + +def _copy_projector_config(): + """ + Copy the projector config folder (containing a pre-downloaded copy of PyCharm + Community) from the system location in /usr/local/share to the user's home + directory. + + Project by default looks for config files in ~/.projector. Copying config folder + from /usr/local/share/ means that each user does not individually have to install + Projector and PyCharm, but can also add their own configurations if they want to + install other Jetbrains IDEs. + """ + + if not os.path.exists(USER_PROJECTOR_CONFIG_DIR): + shutil.copytree(SYS_PROJECTOR_CONFIG_DIR, USER_PROJECTOR_CONFIG_DIR) diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.8.18/projector-proxy/jupyter_projector_proxy/icons/pycharm_icon.svg b/workspace/python/ds-pack/2.1.0-base-python-3.8.18/projector-proxy/jupyter_projector_proxy/icons/pycharm_icon.svg new file mode 100644 index 00000000..694bce26 --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.8.18/projector-proxy/jupyter_projector_proxy/icons/pycharm_icon.svg @@ -0,0 +1,67 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + PyCharm_comm_64 + + + + + + + + + + + + + + + + diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.8.18/projector-proxy/setup.py b/workspace/python/ds-pack/2.1.0-base-python-3.8.18/projector-proxy/setup.py new file mode 100644 index 00000000..fa667057 --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.8.18/projector-proxy/setup.py @@ -0,0 +1,16 @@ +import setuptools + +setuptools.setup( + name="jupyter-projector-server", + version="1.0", + packages=setuptools.find_packages(), + keywords=["Jupyter"], + classifiers=["Framework :: Jupyter"], + install_requires=["jupyter-server-proxy"], + entry_points={ + "jupyter_serverproxy_servers": [ + "projector = jupyter_projector_proxy:setup_projector", + ] + }, + package_data={"jupyter_projector_proxy": ["icons/*"]}, +) diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.8.18/requirements.txt b/workspace/python/ds-pack/2.1.0-base-python-3.8.18/requirements.txt new file mode 100644 index 00000000..e758edd4 --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.8.18/requirements.txt @@ -0,0 +1,21 @@ +ipywidgets==8.1.0 +jupyter-server-proxy==4.0.0 +jupyterhub==4.0.2 +jupyterlab-git==0.42.0 +jupyterlab-lsp==4.2.0 +jupyterlab==3.6.5 +jupyterlab_widgets==3.0.8 +jupytext==1.15.1 +lckr-jupyterlab-variableinspector==3.0.9 +mypy-ls==0.5.1 +nbconvert==7.8.0 +notebook==6.5.5 +pyls-black==0.4.7 +pyls-flake8==0.4.0 +pyls-isort==0.2.2 +pyls-mypy==0.1.8 +python-lsp-black==1.3.0 +python-lsp-server[all]==1.4.1 +virtualenv==20.23.0 +peak-sdk==1.0.0 +jupysql==0.10.1 \ No newline at end of file diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.8.18/spark/config.ini b/workspace/python/ds-pack/2.1.0-base-python-3.8.18/spark/config.ini new file mode 100644 index 00000000..2eb378fd --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.8.18/spark/config.ini @@ -0,0 +1,3 @@ +[spark.credentials] +WORKSPACEID= +TENANTCLUSTER= diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.8.18/vscode-tunnel/scripts/init-or-start.js b/workspace/python/ds-pack/2.1.0-base-python-3.8.18/vscode-tunnel/scripts/init-or-start.js new file mode 100644 index 00000000..d214581c --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.8.18/vscode-tunnel/scripts/init-or-start.js @@ -0,0 +1,111 @@ +/** + * The script aims to setup a vscode tunnel for the workspace. It does by starting a tunnel in the background. + * Usage - node /vscode-tunnel/scripts setup.js + */ +const { spawn } = require('child_process'); +const fs = require('fs'); + +const outputLogFilePath = '/vscode-tunnel/logs/output.log'; +const errorLogFilePath = '/vscode-tunnel/logs/error.log'; +const scriptPath = '/vscode-tunnel/scripts/tunnel.sh'; + +function sleep(milliseconds) { + const start = new Date().getTime(); + while (new Date().getTime() - start < milliseconds); +} + +function waitForDeviceVerification() { + while (true) { + console.log('Waiting for device verification...'); + try { + const data = fs.readFileSync(outputLogFilePath, 'utf8'); + const index = data.lastIndexOf('Open this link in your browser'); + if (index >= 0) { + console.log(data.substring(index)); + break; + } + // check for errors if any + const errors = fs.readFileSync(errorLogFilePath, 'utf8'); + if (errors.length) { + console.error('There was an error creating the tunnel.'); + console.error(error); + break; + } + sleep(4000); + } catch (error) { + if (error.code === 'ENOENT') { + console.log(`File '${fileName}' not found.`); + } else { + console.error(`An error occurred: ${err}`); + } + break; + } + } +} + +function printOutput() { + // Read the file and print its contents + try { + const data = fs.readFileSync(outputLogFilePath, 'utf8'); + const index = data.lastIndexOf('Visual Studio Code Server'); + if (index >= 0) { + console.log(data.substring(index)); + } else { + console.log(data); + } + } catch (error) { + if (error.code === 'ENOENT') { + console.log(`File '${fileName}' not found.`); + } else { + console.error(`An error occurred: ${err}`); + } + } + + try { + const data = fs.readFileSync(errorLogFilePath, 'utf8'); + if (data.length) { + console.error('There was an error creating the tunnel.'); + console.error(data); + return false; + } + } catch (error) { + if (error.code === 'ENOENT') { + console.log(`File '${fileName}' not found.`); + } else { + console.error(`An error occurred: ${err}`); + } + } + + return true; +} + +function main() { + + // Open a file to store the child process's output + const outputLogFile = fs.openSync(outputLogFilePath, 'a'); + const errorLogFile = fs.openSync(errorLogFilePath, 'a'); + + // Spawn a child process to run the shell script with stdin, stdout, and stderr redirected + const childProcess = spawn('bash', [scriptPath], { + detached: true, + stdio: ['pipe', outputLogFile, errorLogFile], // Redirect output to log files + }); + + // Close the file descriptors to release them + fs.closeSync(outputLogFile); + fs.closeSync(errorLogFile); + + console.log('Starting tunnel...'); + // initial wait + sleep(3000); + const result = printOutput(); + + if (result) { + waitForDeviceVerification(); + } + + // Unref the child process to allow the Node.js process to exit + childProcess.unref(); +} + +main(); diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.8.18/vscode-tunnel/scripts/refresh.sh b/workspace/python/ds-pack/2.1.0-base-python-3.8.18/vscode-tunnel/scripts/refresh.sh new file mode 100644 index 00000000..dd1f76cd --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.8.18/vscode-tunnel/scripts/refresh.sh @@ -0,0 +1 @@ +code tunnel restart diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.8.18/vscode-tunnel/scripts/start.sh b/workspace/python/ds-pack/2.1.0-base-python-3.8.18/vscode-tunnel/scripts/start.sh new file mode 100644 index 00000000..233a2e6e --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.8.18/vscode-tunnel/scripts/start.sh @@ -0,0 +1 @@ +node /vscode-tunnel/scripts/init-or-start.js diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.8.18/vscode-tunnel/scripts/status.sh b/workspace/python/ds-pack/2.1.0-base-python-3.8.18/vscode-tunnel/scripts/status.sh new file mode 100644 index 00000000..4d8b473b --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.8.18/vscode-tunnel/scripts/status.sh @@ -0,0 +1 @@ +code tunnel status diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.8.18/vscode-tunnel/scripts/stop.sh b/workspace/python/ds-pack/2.1.0-base-python-3.8.18/vscode-tunnel/scripts/stop.sh new file mode 100644 index 00000000..35bca6ab --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.8.18/vscode-tunnel/scripts/stop.sh @@ -0,0 +1 @@ +code tunnel kill diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.8.18/vscode-tunnel/scripts/tunnel.sh b/workspace/python/ds-pack/2.1.0-base-python-3.8.18/vscode-tunnel/scripts/tunnel.sh new file mode 100644 index 00000000..996e7239 --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.8.18/vscode-tunnel/scripts/tunnel.sh @@ -0,0 +1 @@ +code tunnel diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.9.18/Dockerfile b/workspace/python/ds-pack/2.1.0-base-python-3.9.18/Dockerfile new file mode 100644 index 00000000..b527e491 --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.9.18/Dockerfile @@ -0,0 +1,129 @@ +FROM python:3.9.18-slim-bookworm + +# Set shell to be bash and fail any pipe if any step in the pipe fails +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +# install system packages +RUN apt-get update \ + # installation requirements + && apt-get install -y --no-install-recommends \ + apt-transport-https \ + ca-certificates \ + software-properties-common \ + curl \ + gnupg2 \ + # docker + && mkdir -p /etc/apt/keyrings \ + && curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg \ + && echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \ + $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null \ + && apt-key adv --keyserver keyserver.ubuntu.com --recv-key '95C0FAF38DB3CCAD0C080A7BDC78B2DDEABC47B7' \ + && add-apt-repository 'deb http://cloud.r-project.org/bin/linux/debian bookworm-cran40/' \ + # node + && curl -sL https://deb.nodesource.com/setup_lts.x | bash - \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + # install system packages + # jq is needed for the Peak Platform + jq \ + nano \ + vim \ + fish \ + zsh \ + r-base=4.2.2.20221110-2 \ + r-recommended=4.2.2.20221110-2 \ + build-essential \ + sudo \ + nodejs \ + git \ + htop \ + pandoc \ + texlive \ + texlive-science \ + texlive-latex-extra \ + texlive-luatex \ + texlive-xetex \ + texlive-pictures \ + latexmk \ + lmodern \ + docker-ce \ + less \ + libxext6 \ + libxrender1 \ + libxtst6 \ + libfreetype6 \ + libxi6 \ + openssl \ + unzip \ + && rm -rf /var/lib/apt/lists/* + +# install AWS CLI v2 +RUN curl -fsSL "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.7.4.zip" -o "awscliv2.zip" \ + && unzip -q awscliv2.zip \ + && ./aws/install \ + && rm -rf aws awscliv2.zip + +# Needed for jupyterhub +RUN npm install -g configurable-http-proxy \ + && rm -rf ~/.npm + +# Install Python packages +COPY requirements.txt /tmp/requirements.txt + +RUN pip install --no-cache-dir --upgrade \ + pip \ + setuptools \ + wheel + +RUN pip install -r /tmp/requirements.txt + +# SPARK: copy over spark config +COPY ./spark /usr/local/spark + +# VS Code +COPY codeserver-proxy codeserver-proxy +RUN curl -fsSL https://code-server.dev/install.sh \ + | sh -s -- --version 4.16.1 \ + && pip install --no-cache-dir ./codeserver-proxy + +# vscode cli +RUN curl -Lk 'https://code.visualstudio.com/sha/download?build=stable&os=cli-alpine-x64' --output vscode_cli.tar.gz \ + && tar -xf vscode_cli.tar.gz \ + && cp ./code /usr/local/bin \ + && rm -rf vscode_cli.tar.gz + +COPY vscode-tunnel vscode-tunnel + +RUN mkdir /vscode-tunnel/logs \ + && mkdir /vscode-tunnel/docs \ + && chmod -R 777 vscode-tunnel + +RUN cp vscode-tunnel/scripts/start.sh /usr/bin/start-tunnel \ + && cp vscode-tunnel/scripts/stop.sh /usr/bin/stop-tunnel \ + && cp vscode-tunnel/scripts/refresh.sh /usr/bin/restart-tunnel \ + && cp vscode-tunnel/scripts/status.sh /usr/bin/tunnel-status \ + && chmod +x /usr/bin/start-tunnel /usr/bin/stop-tunnel /usr/bin/restart-tunnel /usr/bin/tunnel-status + +COPY TUNNELING.md /vscode-tunnel/docs/README.md + +RUN git config --global credential.helper store \ + && git config --global core.filemode false + +# Install Projector & PyCharm Community +COPY projector-proxy projector-proxy +RUN pip install --no-cache-dir projector-installer==1.7.0 --upgrade \ + && projector --accept-license --config-directory /usr/local/share/projector \ + ide autoinstall --config-name pycharm --ide-name "PyCharm Community Edition 2021.3.1" \ + && chmod -R a=rwx /usr/local/share/projector \ + && pip install --no-cache-dir ./projector-proxy + +# force shell to bash as Peak Platform isn't respecting jupyterhub option +RUN ln -sf /bin/bash /bin/sh + +EXPOSE 8000 +CMD ["jupyterhub", \ + "--Spawner.default_url=/lab", \ + "--Spawner.http_timeout=90", \ + "--NotebookApp.terminado_settings={'shell_command':'[/bin/bash]'}" \ + ] diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.9.18/README.md b/workspace/python/ds-pack/2.1.0-base-python-3.9.18/README.md new file mode 100644 index 00000000..c654d1b2 --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.9.18/README.md @@ -0,0 +1,79 @@ +# workspace-python-ds-pack-2.1.0-base-python-3.9.18 +This is the Python Data Science pack image made for Workspaces. +The image runs a [Jupyterhub](https://jupyter.org/hub) server that has [PyCharm](https://lp.jetbrains.com/projector/) and [VSCode](https://github.com/coder/code-server) pre-installed with all of the essential and most used packages. +The images also comes pre installed with [peak-sdk](https://docs.peak.ai/sdk/). + +## Image details +### Base image +This image uses [python:3.9.18-slim-bookworm](https://hub.docker.com/layers/library/python/3.9.18-slim-bookworm/images/sha256-3b4e3a36cce74c444b333a26958d65d08b0ded00869f1557faffe8d131a0bdc6?context=explore) as its base which is maintained by [the Docker Community](https://github.com/docker-library/python). + +### OS and other details +``` +Debian GNU/Linux 12 (bookworm) +Linux Kernel 5.10.186-179.751.amzn2.x86_64 +Python 3.9.18 +``` + +### Important Linux packages installed +``` +aws-cli 2.7.4 +curl 7.88.1 +git 2.39.2 +jq 1.6 +nano 7.2 +vim 9.0 +fish 3.6.0 +zsh 5.9 +R 4.3.1 +node 18.17.1 +docker 24.0.6 +htop 3.2.0 +pandoc 2.17.1.1 +less 590 +latex 3.141592653-2.6-1.40.24 +vscode-cli 1.82.2 +``` + +### Python libraries installed +``` +ipywidgets 8.1.0 +jupyter-server-proxy 4.0.0 +jupyterhub 4.0.2 +jupyterlab-git 0.42.0 +jupyterlab-lsp 4.2.0 +jupyterlab 3.6.5 +jupyterlab_widgets 3.0.8 +jupytext 1.15.1 +lckr-jupyterlab-variableinspector 3.0.9 +mypy-ls 0.5.1 +nbconvert 7.8.0 +notebook 6.5.5 +pyls-black 0.4.7 +pyls-flake 80.4.0 +pyls-isort 0.2.2 +pyls-mypy 0.1.8 +python-lsp-black 1.3.0 +python-lsp-server[all] 1.4.1 +virtualenv 20.24.5 +peak-sdk 1.0.0 +jupysql 0.10.1 +``` + +### VS Code extensions installed +``` +cweijan.vscode-database-client2 6.6.3 +``` + +### VS Code Tunneling + +The image supports creating [remote tunnels](https://code.visualstudio.com/docs/remote/tunnels). For simplicity the image comes in handy with some node/bash scripts which can be used to `initialise`, `start`, `stop`, and `restart` the remote tunnels. More info can be found [here](./TUNNELING.md). + +## Building the image +To build the image locally, run the docker build command and pass in the required build arguments: +``` +docker build . -t workflow-python-ds-pack-2.1.0-base-python-3.9.18 +``` + +## Using the image +To use the image, select it when configuring the Workspace. +If you need to install additional dependencies or add some use case specific environment variables, it can be easily extended. diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.9.18/TUNNELING.md b/workspace/python/ds-pack/2.1.0-base-python-3.9.18/TUNNELING.md new file mode 100644 index 00000000..32f42936 --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.9.18/TUNNELING.md @@ -0,0 +1,65 @@ +# Tunneling with VS Code + +This README provides instructions for setting up and managing remote tunnels using this image, making it easier for you to access your workspace remotely within VS Code. + +## What is VS Code Tunneling ? + +VS Code Tunneling is a feature that allows users to connect to a remote machine, such as a desktop PC or virtual machine, via a secure tunnel. This feature securely transmits data from one network to another, allowing users to develop against any machine of their choosing from a VS Code desktop or web client without the need for SSH or HTTPS setup. For more information, please refer (this)[https://code.visualstudio.com/docs/remote/tunnels] link. + +## Initializing a Remote Tunnel + +To create a new remote tunnel for your workspace, you can use the provided bash script: + +```bash +start-tunnel +``` + +Running this script initializes and starts a new tunnel in the background, which can be accessed remotely. To ensure its functionality, a brief authentication step using GitHub is required. + +After executing the above command, you will see the following message in the console: + +``` +To grant access to the server, please log into https://github.com/login/device and use code +``` + +Follow the link provided and enter the `` displayed in the console. + +That's it! Your new, secure tunnel should now be active and accessible at the following URL: `https://vscode.dev/tunnel/workspace--0` + +## Managing Tunnels + +### Restarting a Tunnel + +If you need to restart your tunnel for smooth operation, you can use the following command: + +```bash +restart-tunnel +``` + +### Stopping a Tunnel + +To stop an active tunnel, use the following command: + +```bash +stop-tunnel +``` + +### Starting a Tunnel + +To start a previously stopped tunnel, use this command: + +```bash +start-tunnel +``` + +### Verifying the status of tunnel + +To check the status of the tunnel, use this command: + +```bash +tunnel-status +``` + +## Additional Notes + +To view the logs of any running or closed tunnel, you can refer to the `output.log` file located at `/vscode-tunnel/logs/output.log`. Similarly, any errors related to running or closed tunnels can be found in the `/vscode-tunnel/logs/error.log` file. diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.9.18/codeserver-proxy/jupyter_codeserver_proxy/__init__.py b/workspace/python/ds-pack/2.1.0-base-python-3.9.18/codeserver-proxy/jupyter_codeserver_proxy/__init__.py new file mode 100644 index 00000000..9739bd3a --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.9.18/codeserver-proxy/jupyter_codeserver_proxy/__init__.py @@ -0,0 +1,28 @@ +import os +import subprocess + +bash_command = "code-server --install-extension cweijan.vscode-database-client2@6.6.3" + +def setup_codeserver(): + _install_vscode_extension() + return { + 'command': ['code-server', + '--auth', + 'none', + '--disable-telemetry', + '--port={port}'], + 'launcher_entry': { + 'title': 'VS Code', + 'icon_path': os.path.join(os.path.dirname(os.path.abspath(__file__)), 'icons', 'vs_code_icon.svg') + } + } + +def _install_vscode_extension(): + # Run the Bash command and capture the output + try: + result = subprocess.check_output(bash_command, shell=True, text=True) + print("Bash command output:") + print(result) + except subprocess.CalledProcessError as e: + print("Error running the Bash command:") + print(e) diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.9.18/codeserver-proxy/jupyter_codeserver_proxy/icons/vs_code_icon.svg b/workspace/python/ds-pack/2.1.0-base-python-3.9.18/codeserver-proxy/jupyter_codeserver_proxy/icons/vs_code_icon.svg new file mode 100644 index 00000000..37cf0a52 --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.9.18/codeserver-proxy/jupyter_codeserver_proxy/icons/vs_code_icon.svg @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.9.18/codeserver-proxy/setup.py b/workspace/python/ds-pack/2.1.0-base-python-3.9.18/codeserver-proxy/setup.py new file mode 100644 index 00000000..644aa275 --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.9.18/codeserver-proxy/setup.py @@ -0,0 +1,18 @@ +import setuptools + +setuptools.setup( + name="jupyter-codeserver-server", + version="1.0", + packages=setuptools.find_packages(), + keywords=['Jupyter'], + classifiers=['Framework :: Jupyter'], + install_requires=['jupyter-server-proxy'], + entry_points={ + 'jupyter_serverproxy_servers': [ + 'codeserver = jupyter_codeserver_proxy:setup_codeserver', + ] + }, + package_data={ + 'jupyter_codeserver_proxy': ['icons/*'] + } +) diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.9.18/projector-proxy/jupyter_projector_proxy/__init__.py b/workspace/python/ds-pack/2.1.0-base-python-3.9.18/projector-proxy/jupyter_projector_proxy/__init__.py new file mode 100644 index 00000000..da8ef670 --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.9.18/projector-proxy/jupyter_projector_proxy/__init__.py @@ -0,0 +1,41 @@ +import os +import shutil + +SYS_PROJECTOR_CONFIG_DIR = "/usr/local/share/projector" +USER_PROJECTOR_CONFIG_DIR = os.path.join(os.environ["HOME"], ".projector") + + +def setup_projector(): + _copy_projector_config() + return { + "command": ["projector", "run", "pycharm"], + "launcher_entry": { + "title": "PyCharm", + "icon_path": os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "icons", + "pycharm_icon.svg", + ), + }, + # default projector config runs on port 9999 + "port": 9999, + # set timeout for projector to become responsive to 60s, as it can sometimes + # take a while + "timeout": 60, + } + + +def _copy_projector_config(): + """ + Copy the projector config folder (containing a pre-downloaded copy of PyCharm + Community) from the system location in /usr/local/share to the user's home + directory. + + Project by default looks for config files in ~/.projector. Copying config folder + from /usr/local/share/ means that each user does not individually have to install + Projector and PyCharm, but can also add their own configurations if they want to + install other Jetbrains IDEs. + """ + + if not os.path.exists(USER_PROJECTOR_CONFIG_DIR): + shutil.copytree(SYS_PROJECTOR_CONFIG_DIR, USER_PROJECTOR_CONFIG_DIR) diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.9.18/projector-proxy/jupyter_projector_proxy/icons/pycharm_icon.svg b/workspace/python/ds-pack/2.1.0-base-python-3.9.18/projector-proxy/jupyter_projector_proxy/icons/pycharm_icon.svg new file mode 100644 index 00000000..694bce26 --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.9.18/projector-proxy/jupyter_projector_proxy/icons/pycharm_icon.svg @@ -0,0 +1,67 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + PyCharm_comm_64 + + + + + + + + + + + + + + + + diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.9.18/projector-proxy/setup.py b/workspace/python/ds-pack/2.1.0-base-python-3.9.18/projector-proxy/setup.py new file mode 100644 index 00000000..fa667057 --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.9.18/projector-proxy/setup.py @@ -0,0 +1,16 @@ +import setuptools + +setuptools.setup( + name="jupyter-projector-server", + version="1.0", + packages=setuptools.find_packages(), + keywords=["Jupyter"], + classifiers=["Framework :: Jupyter"], + install_requires=["jupyter-server-proxy"], + entry_points={ + "jupyter_serverproxy_servers": [ + "projector = jupyter_projector_proxy:setup_projector", + ] + }, + package_data={"jupyter_projector_proxy": ["icons/*"]}, +) diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.9.18/requirements.txt b/workspace/python/ds-pack/2.1.0-base-python-3.9.18/requirements.txt new file mode 100644 index 00000000..e758edd4 --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.9.18/requirements.txt @@ -0,0 +1,21 @@ +ipywidgets==8.1.0 +jupyter-server-proxy==4.0.0 +jupyterhub==4.0.2 +jupyterlab-git==0.42.0 +jupyterlab-lsp==4.2.0 +jupyterlab==3.6.5 +jupyterlab_widgets==3.0.8 +jupytext==1.15.1 +lckr-jupyterlab-variableinspector==3.0.9 +mypy-ls==0.5.1 +nbconvert==7.8.0 +notebook==6.5.5 +pyls-black==0.4.7 +pyls-flake8==0.4.0 +pyls-isort==0.2.2 +pyls-mypy==0.1.8 +python-lsp-black==1.3.0 +python-lsp-server[all]==1.4.1 +virtualenv==20.23.0 +peak-sdk==1.0.0 +jupysql==0.10.1 \ No newline at end of file diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.9.18/spark/config.ini b/workspace/python/ds-pack/2.1.0-base-python-3.9.18/spark/config.ini new file mode 100644 index 00000000..2eb378fd --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.9.18/spark/config.ini @@ -0,0 +1,3 @@ +[spark.credentials] +WORKSPACEID= +TENANTCLUSTER= diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.9.18/vscode-tunnel/scripts/init-or-start.js b/workspace/python/ds-pack/2.1.0-base-python-3.9.18/vscode-tunnel/scripts/init-or-start.js new file mode 100644 index 00000000..d214581c --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.9.18/vscode-tunnel/scripts/init-or-start.js @@ -0,0 +1,111 @@ +/** + * The script aims to setup a vscode tunnel for the workspace. It does by starting a tunnel in the background. + * Usage - node /vscode-tunnel/scripts setup.js + */ +const { spawn } = require('child_process'); +const fs = require('fs'); + +const outputLogFilePath = '/vscode-tunnel/logs/output.log'; +const errorLogFilePath = '/vscode-tunnel/logs/error.log'; +const scriptPath = '/vscode-tunnel/scripts/tunnel.sh'; + +function sleep(milliseconds) { + const start = new Date().getTime(); + while (new Date().getTime() - start < milliseconds); +} + +function waitForDeviceVerification() { + while (true) { + console.log('Waiting for device verification...'); + try { + const data = fs.readFileSync(outputLogFilePath, 'utf8'); + const index = data.lastIndexOf('Open this link in your browser'); + if (index >= 0) { + console.log(data.substring(index)); + break; + } + // check for errors if any + const errors = fs.readFileSync(errorLogFilePath, 'utf8'); + if (errors.length) { + console.error('There was an error creating the tunnel.'); + console.error(error); + break; + } + sleep(4000); + } catch (error) { + if (error.code === 'ENOENT') { + console.log(`File '${fileName}' not found.`); + } else { + console.error(`An error occurred: ${err}`); + } + break; + } + } +} + +function printOutput() { + // Read the file and print its contents + try { + const data = fs.readFileSync(outputLogFilePath, 'utf8'); + const index = data.lastIndexOf('Visual Studio Code Server'); + if (index >= 0) { + console.log(data.substring(index)); + } else { + console.log(data); + } + } catch (error) { + if (error.code === 'ENOENT') { + console.log(`File '${fileName}' not found.`); + } else { + console.error(`An error occurred: ${err}`); + } + } + + try { + const data = fs.readFileSync(errorLogFilePath, 'utf8'); + if (data.length) { + console.error('There was an error creating the tunnel.'); + console.error(data); + return false; + } + } catch (error) { + if (error.code === 'ENOENT') { + console.log(`File '${fileName}' not found.`); + } else { + console.error(`An error occurred: ${err}`); + } + } + + return true; +} + +function main() { + + // Open a file to store the child process's output + const outputLogFile = fs.openSync(outputLogFilePath, 'a'); + const errorLogFile = fs.openSync(errorLogFilePath, 'a'); + + // Spawn a child process to run the shell script with stdin, stdout, and stderr redirected + const childProcess = spawn('bash', [scriptPath], { + detached: true, + stdio: ['pipe', outputLogFile, errorLogFile], // Redirect output to log files + }); + + // Close the file descriptors to release them + fs.closeSync(outputLogFile); + fs.closeSync(errorLogFile); + + console.log('Starting tunnel...'); + // initial wait + sleep(3000); + const result = printOutput(); + + if (result) { + waitForDeviceVerification(); + } + + // Unref the child process to allow the Node.js process to exit + childProcess.unref(); +} + +main(); diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.9.18/vscode-tunnel/scripts/refresh.sh b/workspace/python/ds-pack/2.1.0-base-python-3.9.18/vscode-tunnel/scripts/refresh.sh new file mode 100644 index 00000000..dd1f76cd --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.9.18/vscode-tunnel/scripts/refresh.sh @@ -0,0 +1 @@ +code tunnel restart diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.9.18/vscode-tunnel/scripts/start.sh b/workspace/python/ds-pack/2.1.0-base-python-3.9.18/vscode-tunnel/scripts/start.sh new file mode 100644 index 00000000..233a2e6e --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.9.18/vscode-tunnel/scripts/start.sh @@ -0,0 +1 @@ +node /vscode-tunnel/scripts/init-or-start.js diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.9.18/vscode-tunnel/scripts/status.sh b/workspace/python/ds-pack/2.1.0-base-python-3.9.18/vscode-tunnel/scripts/status.sh new file mode 100644 index 00000000..4d8b473b --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.9.18/vscode-tunnel/scripts/status.sh @@ -0,0 +1 @@ +code tunnel status diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.9.18/vscode-tunnel/scripts/stop.sh b/workspace/python/ds-pack/2.1.0-base-python-3.9.18/vscode-tunnel/scripts/stop.sh new file mode 100644 index 00000000..35bca6ab --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.9.18/vscode-tunnel/scripts/stop.sh @@ -0,0 +1 @@ +code tunnel kill diff --git a/workspace/python/ds-pack/2.1.0-base-python-3.9.18/vscode-tunnel/scripts/tunnel.sh b/workspace/python/ds-pack/2.1.0-base-python-3.9.18/vscode-tunnel/scripts/tunnel.sh new file mode 100644 index 00000000..996e7239 --- /dev/null +++ b/workspace/python/ds-pack/2.1.0-base-python-3.9.18/vscode-tunnel/scripts/tunnel.sh @@ -0,0 +1 @@ +code tunnel diff --git a/workspace/r/ds-pack/v1/Dockerfile b/workspace/r/ds-pack/1.0.0-base-r-4.2.0/Dockerfile similarity index 100% rename from workspace/r/ds-pack/v1/Dockerfile rename to workspace/r/ds-pack/1.0.0-base-r-4.2.0/Dockerfile diff --git a/workspace/r/ds-pack/v1/README.md b/workspace/r/ds-pack/1.0.0-base-r-4.2.0/README.md similarity index 95% rename from workspace/r/ds-pack/v1/README.md rename to workspace/r/ds-pack/1.0.0-base-r-4.2.0/README.md index ffc5a88f..4fdeb477 100644 --- a/workspace/r/ds-pack/v1/README.md +++ b/workspace/r/ds-pack/1.0.0-base-r-4.2.0/README.md @@ -1,4 +1,4 @@ -# workspace-r-ds-pack-v1 +# workspace-r-ds-pack-1.0.0-base-r-4.2.0 This is the R Data Science pack image made for Workspaces. The image runs a [RStudio](https://jupyter.org/hub) server and comes pre-installed pre-installed with all of the essential and most used packages. @@ -48,7 +48,7 @@ You can find more details about setting up ODBC for `redshift` [here](https://do ## Building the image To build the image locally, run the docker build command and pass in the required build arguments: ``` -docker build . -t workflow-python-ds-pack-v1 +docker build . -t workflow-r-ds-pack-1.0.0-base-r-4.2.0 ``` ## Using the image diff --git a/workspace/r/ds-pack/v1/amazon.redshiftodbc.ini b/workspace/r/ds-pack/1.0.0-base-r-4.2.0/amazon.redshiftodbc.ini similarity index 100% rename from workspace/r/ds-pack/v1/amazon.redshiftodbc.ini rename to workspace/r/ds-pack/1.0.0-base-r-4.2.0/amazon.redshiftodbc.ini diff --git a/workspace/r/ds-pack/v1/install_packages.sh b/workspace/r/ds-pack/1.0.0-base-r-4.2.0/install_packages.sh similarity index 100% rename from workspace/r/ds-pack/v1/install_packages.sh rename to workspace/r/ds-pack/1.0.0-base-r-4.2.0/install_packages.sh diff --git a/workspace/r/ds-pack/v1/odbcinst.ini b/workspace/r/ds-pack/1.0.0-base-r-4.2.0/odbcinst.ini similarity index 100% rename from workspace/r/ds-pack/v1/odbcinst.ini rename to workspace/r/ds-pack/1.0.0-base-r-4.2.0/odbcinst.ini diff --git a/workspace/r/ds-pack/1.1.0-base-r-4.3.1/Dockerfile b/workspace/r/ds-pack/1.1.0-base-r-4.3.1/Dockerfile new file mode 100644 index 00000000..3f3c505f --- /dev/null +++ b/workspace/r/ds-pack/1.1.0-base-r-4.3.1/Dockerfile @@ -0,0 +1,93 @@ +FROM rocker/verse:4.3.1 + +# Adding dependencies for top 100 CRAN Packages +RUN apt-get update \ + # installation requirements + && apt-get install -y --no-install-recommends \ + apt-transport-https \ + curl \ + gnupg2 \ + software-properties-common \ + lsb-release \ + # docker repo + && curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg \ + && echo \ + "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu \ + $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + build-essential \ + docker-ce \ + gdal-bin \ + git \ + libcairo2-dev \ + libcurl4-openssl-dev \ + libfontconfig1-dev \ + libfreetype6-dev \ + libfribidi-dev \ + libgdal-dev \ + libgeos-dev \ + libharfbuzz-dev \ + libicu-dev \ + libjpeg-dev \ + libpng-dev \ + libproj-dev \ + libssh2-1-dev \ + libssl-dev \ + libtiff-dev \ + libudunits2-dev \ + libxml2-dev \ + make \ + pandoc \ + zlib1g-dev \ + # Platform Essential Packages + curl \ + jq \ + wget \ + # Add extras for helpfulness + fish \ + htop \ + nano \ + nano \ + vim \ + zsh \ + # For AWS CLI help + groff \ + # ODBC Driver Manager (for snowflake) + unixodbc \ + unixodbc-dev \ + && rm -rf /var/lib/apt/lists/* + +# Install Python + all required packages +COPY install_packages.sh /tmp/install_packages.sh +RUN bash /tmp/install_packages.sh + +# install AWS CLI v2 +RUN curl -fsSL "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.7.4.zip" -o "awscliv2.zip" \ + && unzip -q awscliv2.zip \ + && ./aws/install \ + && rm -rf aws awscliv2.zip +# don't use pager AWS CLI as will stop scripts wrapping AWSCLI +ENV AWS_PAGER="" + +# https://docs.aws.amazon.com/redshift/latest/mgmt/configure-odbc-connection.html#install-odbc-driver-linux +# Install Redshift ODBC Driver +RUN wget https://s3.amazonaws.com/redshift-downloads/drivers/odbc/1.4.52.1000/AmazonRedshiftODBC-64-bit-1.4.52.1000-1.x86_64.deb -O AmazonRedshiftODBC.deb \ + && apt install ./AmazonRedshiftODBC.deb \ + && rm ./AmazonRedshiftODBC.deb + +# Install Snowflake ODBC Driver +RUN wget https://sfc-repo.snowflakecomputing.com/odbc/linux/2.24.4/snowflake-odbc-2.24.4.x86_64.deb -O SnowflakeODBC.deb \ + && apt install -y ./SnowflakeODBC.deb \ + && rm ./SnowflakeODBC.deb + +COPY odbcinst.ini /etc/odbcinst.ini +COPY amazon.redshiftodbc.ini /opt/amazon/redshiftodbc/lib/64/amazon.redshiftodbc.ini + +ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib +ENV ODBCSYSINI=/etc +ENV AMAZONREDSHIFTODBCINI=/opt/amazon/redshiftodbc/lib/64/amazon.redshiftodbc.ini + +# Make git not care about file permission changes +RUN git config --global credential.helper store \ + && git config --global core.filemode false diff --git a/workspace/r/ds-pack/1.1.0-base-r-4.3.1/README.md b/workspace/r/ds-pack/1.1.0-base-r-4.3.1/README.md new file mode 100644 index 00000000..b91a474a --- /dev/null +++ b/workspace/r/ds-pack/1.1.0-base-r-4.3.1/README.md @@ -0,0 +1,62 @@ +# workspace-r-ds-pack-1.0.0-base-4.3.1 +This is the R Data Science pack image made for Workspaces. +The image runs a [RStudio](https://jupyter.org/hub) server and comes pre-installed pre-installed with all of the essential and most used packages. +The images also comes pre installed with [peak-sdk](https://docs.peak.ai/sdk/). + +## Image Details +### Base image +This image uses [rocker/verse:4.3.0](https://hub.docker.com/layers/rocker/verse/4.3.1/images/sha256-9fa09814225979a80c398b5778ecd61d853e0ca788f806c2800d9b336d4634ca?context=explore) as its base which is maintained by [the Rocker Community](https://github.com/rocker-org/rocker). + +### OS and other details +``` +Ubuntu 22.04.3 LTS (Jammy Jellyfish) +Linux Kernel 5.10.186-179.751.amzn2.x86_64 +R 4.3.1 +``` + +### Linux packages installed +``` +aws-cli 2.7.4 +curl 7.81.0 +git 2.34.1 +jq 1.6 +nano 6.2 +vim 8.2 +fish 3.3.1 +zsh 5.8.1 +docker 24.0.6 +htop 3.0.5 +pandoc 3.1.1 +less 590 +latex 3.141592653-2.6-1.40.25 +redshift-odbc-driver 1.4.27.1000 +snowflake-odbc-driver 2.24.4 +``` + +### Python packages installed +``` +peak-sdk 1.0.0 +``` + +### R libraries installed +None + +### Environment variables +- `AMAZONREDSHIFTODBCINI`: This represents the location of `amazon.redshiftodbc.ini` file and is set to `/opt/amazon/redshiftodbc/lib/64/amazon.redshiftodbc.ini` as we have added the file there. +- `ODBCSYSINI`: This represents the location of `odbcinst.ini` file and is set to `/etc` by default. + +### ODBC Setup +- The `odbcinst.ini` file exists in the `/etc` directory, and the `ODBCSYSINI` env has been set accordingly. + +You can find more details about setting up ODBC for `redshift` [here](https://docs.aws.amazon.com/redshift/latest/mgmt/configure-odbc-connection.html). For `snowflake` more details can be found [here](https://docs.snowflake.com/en/user-guide/odbc-linux.html) + + +## Building the image +To build the image locally, run the docker build command and pass in the required build arguments: +``` +docker build . -t workflow-r-ds-pack-1.0.0-base-4.3.1 +``` + +## Using the image +To use the image, select it when configuring the Workspace. +If you need to install additional dependencies or add some use case specific environment variables, it can be easily extended. diff --git a/workspace/r/ds-pack/1.1.0-base-r-4.3.1/amazon.redshiftodbc.ini b/workspace/r/ds-pack/1.1.0-base-r-4.3.1/amazon.redshiftodbc.ini new file mode 100644 index 00000000..39f2d427 --- /dev/null +++ b/workspace/r/ds-pack/1.1.0-base-r-4.3.1/amazon.redshiftodbc.ini @@ -0,0 +1,5 @@ +[Driver] +DriverManagerEncoding=UTF-16 +ErrorMessagesPath=/opt/amazon/redshiftodbc/ErrorMessages +LogPath=[LogPath] +SwapFilePath=/tmp diff --git a/workspace/r/ds-pack/1.1.0-base-r-4.3.1/install_packages.sh b/workspace/r/ds-pack/1.1.0-base-r-4.3.1/install_packages.sh new file mode 100644 index 00000000..20935be5 --- /dev/null +++ b/workspace/r/ds-pack/1.1.0-base-r-4.3.1/install_packages.sh @@ -0,0 +1,56 @@ +#!/bin/bash +set -e + +WORKON_HOME=${WORKON_HOME:-/opt/venv} +PYTHON_VENV_PATH=${PYTHON_VENV_PATH:-${WORKON_HOME}/reticulate} +RETICULATE_MINICONDA_ENABLED=${RETICULATE_MINICONDA_ENABLED:-FALSE} + +apt-get update && apt-get install -y --no-install-recommends \ + libpython3-dev \ + python3-dev \ + python3-pip \ + python3-virtualenv \ + python3-venv \ + python-is-python3 \ + && rm -rf /var/lib/apt/lists/* + +# Some TF tools expect a "python" binary +if [ ! -e /usr/local/bin/python ]; then + ln -s $(which python3) /usr/local/bin/python +fi + +python -m pip --no-cache-dir install --upgrade \ + pip \ + setuptools \ + virtualenv \ + wheel + +mkdir -p ${WORKON_HOME} +python -m venv ${PYTHON_VENV_PATH} + +## Ensure RStudio inherits this env var +echo "" >> ${R_HOME}/etc/Renviron +echo "WORKON_HOME=${WORKON_HOME}" >> ${R_HOME}/etc/Renviron +echo "RETICULATE_MINICONDA_ENABLED=${RETICULATE_MINICONDA_ENABLED}" >> ${R_HOME}/etc/Renviron + +## symlink these so that these are available when switching to a new venv +## -f check for file, -L for link, -e for either +if [ ! -e /usr/local/bin/python ]; then + ln -s $(which python3) /usr/local/bin/python +fi + +if [ ! -e /usr/local/bin/pip ]; then + ln -s ${PYTHON_VENV_PATH}/bin/pip /usr/local/bin/pip +fi + +if [ ! -e /usr/local/bin/virtualenv ]; then + ln -s ${PYTHON_VENV_PATH}/bin/virtualenv /usr/local/bin/virtualenv +fi + +# Install peak-sdk +pip install peak-sdk==1.0.0 + +## Allow staff-level users to modify the shared environment +chown -R :staff ${WORKON_HOME} +chmod g+wx ${WORKON_HOME} +chown :staff ${PYTHON_VENV_PATH} diff --git a/workspace/r/ds-pack/1.1.0-base-r-4.3.1/odbcinst.ini b/workspace/r/ds-pack/1.1.0-base-r-4.3.1/odbcinst.ini new file mode 100644 index 00000000..8531d3e0 --- /dev/null +++ b/workspace/r/ds-pack/1.1.0-base-r-4.3.1/odbcinst.ini @@ -0,0 +1,15 @@ +[ODBC Drivers] +Amazon Redshift (x64)=Installed +SnowflakeDSIIDriver=Installed + +[Amazon Redshift (x64)] +Description=Amazon Redshift ODBC Driver (64-bit) +Driver=/opt/amazon/redshiftodbc/lib/64/libamazonredshiftodbc64.so + +[SnowflakeDSIIDriver] +APILevel=1 +ConnectFunctions=YYY +Description=Snowflake DSII +Driver=/usr/lib/snowflake/odbc/lib/libSnowflake.so +DriverODBCVer=03.52 +SQLLevel=1