From b412340ad05d2cead198aeb2c62d6ebc06eb5727 Mon Sep 17 00:00:00 2001 From: Samir B Amin Date: Sun, 28 Aug 2022 08:53:14 -0400 Subject: [PATCH 1/2] upgrade mkdocs material * support tags * support github issues based comments --- serve_docs.sh | 5 ++- web/mkdocs.yml | 12 ++++-- web/overrides/main.html | 4 +- web/overrides/partials/content.html | 64 ++++------------------------- 4 files changed, 22 insertions(+), 63 deletions(-) diff --git a/serve_docs.sh b/serve_docs.sh index b7d5d0d..e729857 100755 --- a/serve_docs.sh +++ b/serve_docs.sh @@ -18,6 +18,9 @@ conda activate ruby set -euo pipefail cd "${DOCDIR}"/web && echo -e "\nWorkdir is $(pwd)\n" -mkdocs serve +## en0 is wifi address - double check that +# hostip="$(ipconfig getifaddr en0)" +# mkdocs serve --livereload -a "${hostip}":8000 +mkdocs serve ## END ## diff --git a/web/mkdocs.yml b/web/mkdocs.yml index c13809b..48e354f 100644 --- a/web/mkdocs.yml +++ b/web/mkdocs.yml @@ -3,7 +3,7 @@ site_name: Programming 101 site_url: https://code.sbamin.com site_author: "Samir B. Amin" site_description: >- - Notes on programming, mostly using R, bash, and python. + Notes on programming for computational biology: Using R, bash, and python on HPC. # Repository repo_name: sbamin/code101 @@ -37,6 +37,9 @@ theme: - navigation.indexes - navigation.sections - content.code.annotate + - search.suggest + - search.highlight + - search.share palette: # - scheme: default primary: blue @@ -64,10 +67,12 @@ plugins: - macros - git-revision-date-localized: type: iso_date - timezone: US + timezone: 'America/New_York' fallback_to_build_date: true - minify: minify_html: true + - tags: + tags_file: tags.md ## Customization extra_css: @@ -102,7 +107,7 @@ extra: analytics: provider: google property: UA-307297-1 - site_keywords: "linux,getting started,computing,hpc, conda,bash,jupyter,programming,biology,bioinformatics" + site_keywords: "linux,getting started,computing,hpc, conda,bash,jupyter,programming,biology,bioinformatics,gpu" version: provider: mike default: latest @@ -168,5 +173,6 @@ nav: - Part 3: hpc/cpu/sumner_3.md - GPU Computing: - GPU setup: hpc/gpu/winter_1.md + - Tags: tags.md #_end_ diff --git a/web/overrides/main.html b/web/overrides/main.html index 61911e5..5696a15 100644 --- a/web/overrides/main.html +++ b/web/overrides/main.html @@ -2,8 +2,8 @@ {% block extrahead %} - {% if page and page.meta and page.meta.keywords %} - {% set keywords = page.meta.keywords %} + {% if page and page.meta and page.meta.tags %} + {% set keywords = page.meta.tags %} {% elif config.extra.site_keywords %} {% set keywords = config.extra.site_keywords %} diff --git a/web/overrides/partials/content.html b/web/overrides/partials/content.html index cd2283f..eb9eed6 100644 --- a/web/overrides/partials/content.html +++ b/web/overrides/partials/content.html @@ -1,71 +1,21 @@ - - - +{#- + This file was automatically generated - do not edit +-#} {% if page.edit_url %} - + {% include ".icons/material/pencil.svg" %} {% endif %} - - +{% if "tags" in config.plugins %} + {% include "partials/tags.html" %} +{% endif %} {% if not "\x3ch1" in page.content %}

{{ page.title | d(config.site_name, true)}}

{% endif %} - - {{ page.content }} - - {% if page and page.meta and ( page.meta.git_revision_date_localized or page.meta.revision_date ) %} {% include "partials/source-file.html" %} {% endif %} - - - - -{% set disqus = config.extra.disqus %} -{% if page and page.meta and page.meta.disqus is string %} - {% set disqus = page.meta.disqus %} -{% endif %} - - -{% if not page.is_homepage and disqus %} -

{{ lang.t("meta.comments") }}

-
- -{% endif %} From e16918c5ce2f9f08392831840f047048c02af09d Mon Sep 17 00:00:00 2001 From: Samir B Amin Date: Sun, 28 Aug 2022 08:54:44 -0400 Subject: [PATCH 2/2] SSL related configutation * start jupyter server using self-signed SSL * enable tags --- web/docs/hpc/cpu/sumner_1.md | 10 ++- web/docs/hpc/cpu/sumner_2.md | 124 +++++++++++++++++++++++++++-------- web/docs/hpc/cpu/sumner_3.md | 14 +++- web/docs/hpc/gpu/winter_1.md | 11 +++- web/docs/tags.md | 5 ++ 5 files changed, 131 insertions(+), 33 deletions(-) create mode 100644 web/docs/tags.md diff --git a/web/docs/hpc/cpu/sumner_1.md b/web/docs/hpc/cpu/sumner_1.md index 33d4f08..bb51125 100644 --- a/web/docs/hpc/cpu/sumner_1.md +++ b/web/docs/hpc/cpu/sumner_1.md @@ -1,8 +1,14 @@ --- title: "Setting up CPU env - Part 1" -description: "Sumner HPC Setup 2021" -keywords: "sumner,hpc,conda,bash,jupyter,programming" +description: "Getting started with HPC Setup. Part 1: Setup conda from scratch or replace an older environment." +keywords: "hpc,setup,conda,bash,programming,jax,sumner" comments: true +tags: + - hpc + - setup + - conda + - bash + - programming --- ## Set up for HPC Sumner diff --git a/web/docs/hpc/cpu/sumner_2.md b/web/docs/hpc/cpu/sumner_2.md index aaf504f..35ddc6f 100644 --- a/web/docs/hpc/cpu/sumner_2.md +++ b/web/docs/hpc/cpu/sumner_2.md @@ -1,8 +1,15 @@ --- title: "Setting up CPU env - Part 2" -description: "Sumner HPC Setup 2021: Part 2" -keywords: "sumner,hpc,conda,bash,jupyter,programming" +description: "Getting started with HPC Setup. Part 2: Configure conda enviornment, setup R and python, configure Jupyter kernels and start Jupyter Lab." +keywords: "hpc,setup,conda,jupyter,kernels,lab,programming,jax,sumner" comments: true +tags: + - hpc + - setup + - conda + - kernels + - jupyter + - programming --- Following up from [Part 1: Initial HPC setup](../sumner_1/), we now start installing essential softwares or (in conda dictionary) packages, e.g., R, Jupyter, etc. @@ -476,9 +483,20 @@ q(save = "no") ### bash kernel -Stay in _yoda_ env. +We will now install [bash kernel](https://github.com/takluyver/bash_kernel) in _yoda_ env. Unlike other (R and python) kernels, bash kernel do not need to be installed in all of conda env because we can always switch between conda env specific bash env using `mamba activate anakin` or `mamba activate rey`, etc. + +Here, I will install bash kernel in _yoda_ and not _base_ env as I intend to use _yoda_ as my primary go-to env when I login to HPC. + + +!!! tip "Keep use of _base_ env to bare minimum" + Please remember that we keep use of _base_ to bare minimum for maintaining core of codna packages, and should avoid installing (and populating dependencies) packages in _base_ env. You can always delete secondary conda env and restart but you cannot do so with _base_ env! ```sh +## in yoda env +mamba list | grep -E "bash_kernel" +## if this does not show bash_kernel installed, redo install +# mamba install -c conda-forge ipykernel r-irkernel bash_kernel +## and the install kernel into jupyter env python -m bash_kernel.install ``` @@ -662,7 +680,7 @@ Once we have installed env specific kernels as in _yoda_ (and other envs, if a n conda deactivate ``` -!!! warning "You should be in conda _base_ env" +!!! warning "You should now be in conda _base_ env" If you were jumping across more than one conda envs, then each instance of `conda deactivate` command will bring you back to previously active env. So, make sure to return to _base_ env which you can confirm using `echo $CONDA_PREFIX` output. That should point to base path of conda (mambaforge in my case) installation: _/projects/verhaak-lab/amins/hpcenv/mambaforge/_. Also, notice change in bash prompt to `(base) userid@sumner50`. * Once in the _base_ env, generate skeleton for default jupyter configuration. @@ -678,8 +696,61 @@ jupyter server --generate-config !!! danger "Secure Jupyter Server" It is critical that you harden security of jupyterlab server. Default configuration is not good enough (in my view) for launching notebook server over HPC, especially without SSL (or _https_) support. Setting up individual security steps is beyond scope of this documentation. However, I strongly recommend reading official documentation on [running a public Jupyter Server](https://jupyter-server.readthedocs.io/en/latest/operators/public-server.html) and [security in the jupyter server](https://jupyter-server.readthedocs.io/en/latest/operators/security.html). + Checkout [this example guide](https://medium.com/@nyghtowl/setup-jupyter-notebook-access-on-google-compute-engine-with-https-ad69297f438b) on creating self-signed SSL certificates in case you do not have SSL certificates from Research IT department. + +#### self-signed SSL certificates + +Ideally you should have SSL certificates signed by a verified certificate authority (CA) else most of modern web browsers will issue a warning or an error regarding SSL secutity. CA-signed SSLs are typically paid unless issued via [let's encrypt](https://letsencrypt.org/) or your DNS providers. + +Securing a website (Jupyter Env in our case) is beyond scope of this guide but I will suggest to inquire to your research IT department to see if they can help secure your Jupyter env. It's better to have CA-signed SSL over self-signed SSL, and always better to have SSL (https) over insecure (http) connection to your Jupyter env. This is true even when you are working within a secure firewall of your work network. + +Here, I am creating a self-signed SSL which most of you can generate and at least have self-signed SSL. [See this post for details](https://medium.com/@nyghtowl/setup-jupyter-notebook-access-on-google-compute-engine-with-https-ad69297f438b). + +```sh +## switch to base conda env +mamba activate base + +## create dir to save SSL certificates +mkdir ~/ssl_cert && \ +chmod 700 ~/ssl_cert && \ +cd ~/ssl_cert + +## generate a private key +openssl genrsa -out hpc_cert.key 2048 + +## secure your private key +chmod 600 hpc_cert.key + +## create a signed certificate using a key we generated above +openssl req -new -key hpc_cert.key -out hpc_cert.csr + +## finally self-sign this certicate +openssl x509 -req -days 365 -in hpc_cert.csr -signkey hpc_cert.key -out hpc_cert.pem +``` + +>While creating a signed certificate, *hpc_cert.csr*, you will be prompted to enter issuer's country, city, etc., including Common Name (CN). You should put some info related to use of this certificate into respective fields, e.g., Common Name can be `hpcjupyter`. + +!!! warning "SSL certificate - unable to verify SSL connection" + Regardless of what you put in CN, say `hpcjupyter.mywebsite.com`, most of modern web browsers so either SSL warning or error of not verifying your self-signed SSL. This is because you have self-signed this SSL and not using an approved certificate authority (CA) provider for signing your SSL. [Read this post for details](https://support.dnsimple.com/articles/what-is-common-name/). In essense, your self-signed SSL is acting as it is an authentic SSL for a CN you provided above, `hpcjupyter.mywebsite.com`. Since that's a clearly a security risk, most modern browsers will throw a warning before you are allowed to visit a site (Jupyter server in this case) or even may not allow at all to load that website! Hence, prefer asking your Research IT to make a signed SSL available for a specific intranet subdomain, e.g., `jupyter.company.com` for users to use. + +#### cookie file + +* After creating SSL certificates, also create a cookie file for jupyter server. + +```sh +openssl rand -hex 32 > /home/foo/ssl_cert/jp_cookie +chmod 600 /home/foo/ssl_cert/jp_cookie +``` + +#### config file + * Example config for _/home/userid/.jupyter/jupyter_server_config.py_. **Please do not copy and paste these options** without knowing [underlying details](https://jupyter-notebook.readthedocs.io/en/stable/config.html). +!!! warning "Jupyter token - provide a strong token string" + While editing jupyter config file below, please read inline comments carefully, especially for `c.ServerApp.token`. **Do not use the default token as that token is used for login to your running jupyter server**. Provide a secure (a longer string, 32 characters or more) string generated using `uuidgen` and after removing dashes. + + Do not worry of remembering this token. Jupyter does allow you to have an alternate way of login using a custom, user-generated password (see further below). + ```py ## leave commented out portion of default config as it is. ## then you can add your custom config @@ -688,17 +759,18 @@ jupyter server --generate-config #### NOTEBOOK CONFIGS #### ## SSL settings ## read documentation for details -c.ServerApp.certfile = u'/home/foo/xyz/jp.crt' -c.ServerApp.keyfile = u'/home/foo/xyz/jp.pem' +c.ServerApp.certfile = u'/home/foo/ssl_cert/hpc_cert.csr' +c.ServerApp.keyfile = u'/home/foo/ssl_cert/hpc_cert.key' -## openssl rand -hex 32 > /home/foo/xyz/dummy_file -c.JupyterHub.cookie_secret_file = '/home/foo/xyz/dummy_file' +c.JupyterHub.cookie_secret_file = '/home/foo/ssl_cert/jp_cookie' c.ServerApp.open_browser = False ## token used to programmatically login to jupyter, -## e.g., via Atom Hydrogen package -## alphanumeric secret string - longer the better. -c.ServerApp.token = 'dummy_login_token_replace_with_a_secret_token' +## e.g., via VS Code Jupyter extension. +## This is essentially a password to login to jupyter +## Provide an alphanumeric secret string - longer the better. +## you can create one using uuidgen command: Remove dashes. +c.ServerApp.token = 'PLEASE_REPLACE_THIS_TOKEN_c8syb2lnd89g2fosyyfskdfy02h' c.ServerApp.allow_password_change = False ## should be set to False @@ -713,26 +785,25 @@ c.ServerApp.terminado_settings={'shell_command':['bash', '-l']} ## END ## ``` -* Once you customize _/home/userid/.jupyter/jupyter_notebook_config.py_ file, **make sure to generate a secret and strong password using** `jupyter server password` command. Your password then will be written in encrypted format in _/home/userid/.jupyter/jupyter_server_config.json_ file. +#### jupyter password + +* Once you customize config file above, **make sure to generate a secret and strong password using** `jupyter server password` command. Your password then will be written in an encrypted format in _/home/userid/.jupyter/jupyter_server_config.json_ file. + * Make both files read/write-only by you. ```sh ## for directory, we use permission 700 chmod 700 ~/.jupyter -## location where cookie secret is stored -## prefer a secure and stable path -mkdir -p ~/xyz -chmod 700 ~/xyz # For files, we use permission 600 chmod 600 ~/.jupyter/jupyter_server_config.py chmod 600 ~/.jupyter/jupyter_server_config.json -## location of cookie secret file -chmod 600 ~/xyz/dummy_file ``` ### Customizing user interface +If you need to add custom themes, fonts, shortcuts, etc. for Jupyter, you may follow this section, else safe to skip to [Start Jupyter](#start-jupyterlab) section. + Before installing themes or customizing jupyterlab further, I will install [node js](https://nodejs.org/en/) package to _base_ env. >Ideally, _base_ env should not be cluttered with packages except bare mininmum that comes with original conda installation (mambaforge in my case). However, node js is required to setup and manage jupyterlab extensions and how jupyterhub can interact with a few kernels, e.g., `jupyterlab-sql` extension to interact with sql databases that I will end up installing in the future. @@ -746,7 +817,7 @@ npm --version #### Themes -Optional: Themes provide custom user interface and is optional for setup. See example themes at https://github.com/dunovank/jupyter-themes +Themes provide custom user interface and is optional for setup. See example themes at https://github.com/dunovank/jupyter-themes ```sh mamba install -c conda-forge jupyterthemes @@ -801,7 +872,7 @@ If you are familiar with RStudio shortcuts for R pipe `%>%` and assignment `<-` #### gpg signatures -Optional: Import gpg keys, if any for [code signing](https://docs.github.com/en/authentication/managing-commit-signature-verification/signing-commits). More at https://unix.stackexchange.com/a/392355/28675 +Import gpg keys, if any for [code signing](https://docs.github.com/en/authentication/managing-commit-signature-verification/signing-commits). More at https://unix.stackexchange.com/a/392355/28675 Earlier I installed required gpg packages, _gpg_ and _python-gnupg_ but they ended up conflicting with `gpg-agent` that is running by the system gpg at `/usr/bin/gpg`. So, I have to remove both conda packages in order to use system gpg at `/usr/bin/`. @@ -848,7 +919,7 @@ gpg --list-secret-keys #### rmate -Optional: I user `rmate` command to open remote files on HPC in the text editor like Atom or SublimeText on my macbook. +I user `rmate` command to open remote files on HPC in the text editor like Atom or SublimeText on my macbook. * Prefer installing [standalone binary](https://github.com/textmate/rmate) over ruby-based (`gem install rmate`) command. If you prefer ruby based installation, better to add ruby installation in a separate conda env, e.g., in _luke_ or other backend env. @@ -907,8 +978,7 @@ echo $? # jupyter server extension enable --py jupyter_http_over_ws ``` -* Test jupyterlab run: Please [read documentation](https://jupyter-notebook.readthedocs.io/en/stable/public_server.html) carefully on using SSL option and defining port and IP. - +* Test jupyterlab run. !!! warning "☠️ Use SSL and password protection ☠️" Avoid running notebook server without SSL and proper password and token configuration as [detailed above](#configure-jupyterlab)) else you may encounter a significant data security risk. @@ -921,10 +991,12 @@ mkdir -p ~/tmp/jupyter/sumner REMOTEIP="$(hostname -I | head -n1 | xargs)" ## test run from a login or compute node -jupyter lab --no-browser --certfile="${MYPEM}" --keyfile "${MYKEY}" --ip="${REMOTEIP}" --port="$MYPORT" >> ~/tmp/jupyter/sumner/runtime.log 2>&1 +## SSL related settings will be inherited from jupyter config file that +## we already have created as above. +jupyter lab --no-browser --ip="${REMOTEIP}" |& tee -a ~/tmp/jupyter/sumner/runtime.log ``` -* Once a jupyter session begins and assuming you are on a secure local area network, you can open URL: `https://:/lab` to launch jupyter lab. +* Once a jupyter session begins and assuming you are on a secure local area network, you can open URL: `https://:/lab` to launch jupyter lab. Here, `` is randomly assigned when you start a server and URL will be displayed on the terminal or in a log file at *~/tmp/jupyter/sumner/runtime.log*. !!! warning "Run jupyterlab from a compute and not login node" **Avoid running JupyterLab server on a login node.** It will most likely be killed by HPC admins. For longer running and compute-intensive jupyterlab sessions, it is preferable to run jupyterlab from a compute and not a login node. This requires series of secure port forwarding which is beyond the scope of current documentation. However, your HPC may already have support for running JupyterLab on a compute node, e.g, similar to this one at [Univ. of Bern](https://hpc-unibe-ch.github.io/software/JupyterLab.html) or [Princeton Univ.](https://researchcomputing.princeton.edu/support/knowledge-base/jupyter). Talk to your HPC staff for policies on running JupyterLab server. @@ -938,4 +1010,4 @@ exit # from sumner ssh sumner ``` -[In Part 3](../sumner_3/), I will finalize setting up Sumner (or CPU-based) HPC and also install a dedicated conda env for Winter (GPU-based) HPC. +[In Part 3](../sumner_3/), I will finalize setting up Sumner (or CPU-based) HPC and also install a dedicated conda env for Winter (GPU-based) HPC. If you like to stop here, you may except I prefer that you follow [bash startup]({{ config.site_url }}/hpc/cpu/sumner_3/#bash-startup) section in Part 3, so that conda and program-specific (R and python) environment variables are consistently loaded across HPC bash user env. diff --git a/web/docs/hpc/cpu/sumner_3.md b/web/docs/hpc/cpu/sumner_3.md index 7d7938a..43cf21a 100644 --- a/web/docs/hpc/cpu/sumner_3.md +++ b/web/docs/hpc/cpu/sumner_3.md @@ -1,8 +1,16 @@ --- title: "Setting up CPU env - Part 3" -description: "Sumner HPC Setup 2021: Part 3" -keywords: "sumner,hpc,conda,bash,jupyter,programming" +description: "Getting started with HPC Setup. Part 3: Configure conda environments for julia, python2, perl, and postgresql. Add Modules and containers, finally setup bash startup sequence." +keywords: "hpc,setup,conda,jupyter,julia,database,programming,jax,sumner" comments: true +tags: + - hpc + - setup + - conda + - startup + - database + - jupyter + - programming --- Continuing the setup from the [Part 2](../sumner_2/), now we will finalize setup for Sumner or CPU-optimized HPC with following key configurations: @@ -849,7 +857,7 @@ Over years of fiddling with bash startup, my startup setup has become overly com [![](https://mermaid.ink/img/eyJjb2RlIjoiICAgIGdyYXBoIFRCXG4gICAgICBBW1N5c3RlbSA8YnI-L2V0Yy9wcm9maWxlXSAtLT4gQltVc2VyPGJyPi5iYXNoX3Byb2ZpbGVdXG4gICAgICBCIC0tPiBDe1VzZXIgLnByb2ZpbGUuZC8gZGlyZWN0b3J5fTtcbiAgICAgIEMgLS0-fFllc3wgRFtzb3VyY2UgLnByb2ZpbGUuZC9BMDEuc2hdO1xuICAgICAgRCAtLi0-IEVbc291cmNlIC5wcm9maWxlLmQvQTAyLnNoXTtcbiAgICAgIEUgLS4tPiBGW3NvdXJjZSAucHJvZmlsZS5kL1o5OS5zaF07XG4gICAgICBGIC0tPiBHO1xuICAgICAgQyAtLT4gfE5vfCBHe1VzZXIgLmJhc2hyY307XG4gICAgICBHIC0tPiB8SW50ZXJhY3RpdmUgc2hlbGw8YnI-UFMxIHZhciBpcyBzZXR8IEhbc291cmNlIC9ldGMvYmFzaHJjXVxuICAgICAgSCAtLT4gSVtVc2VyIC5iYXNoX2FsaWFzZXNdXG4gICAgICBJIC0tPiBKW2Jhc2ggdGVybWluYWwgY29uZmlnLCA8YnI-IGUuZy4sIGNvbG9ycywgaGlzdG9yeSwgZXRjLl1cbiAgICAgIEogLS0-IEtbSW5pdGlhbGl6ZTxicj5Db25kYSBFbnZpcm9ubWVudF1cbiAgICAgIEcgLS0-IHxOb24taW50ZXJhY3RpdmUgc2hlbGw8YnI-UFMxIHZhciBpcyB1bnNldHwgS1xuICAgICAgTFtVc2VyIC5wcm9maWxlLmQvdm9pZF0gLS0-IE1bc291cmNlIDxicj4gLnByb2ZpbGUuZC92b2lkL1ZBMDEuc2hdO1xuICAgICAgTSAtLi0-IE5bc291cmNlIDxicj4gLnByb2ZpbGUuZC92b2lkL1ZaOTkuc2hdO1xuICAgICAgSyAtLT4gTDtcbiAgICAgIE4gLS0-IE9bU2V0IFBBVEhdXG4gICAgICBPIC0tPiBQW1NldCBQUzFdIiwibWVybWFpZCI6eyJ0aGVtZSI6ImRlZmF1bHQifSwidXBkYXRlRWRpdG9yIjpmYWxzZSwiYXV0b1N5bmMiOnRydWUsInVwZGF0ZURpYWdyYW0iOmZhbHNlfQ)](https://mermaid-js.github.io/mermaid-live-editor/edit#eyJjb2RlIjoiICAgIGdyYXBoIFRCXG4gICAgICBBW1N5c3RlbSA8YnI-L2V0Yy9wcm9maWxlXSAtLT4gQltVc2VyPGJyPi5iYXNoX3Byb2ZpbGVdXG4gICAgICBCIC0tPiBDe1VzZXIgLnByb2ZpbGUuZC8gZGlyZWN0b3J5fTtcbiAgICAgIEMgLS0-fFllc3wgRFtzb3VyY2UgLnByb2ZpbGUuZC9BMDEuc2hdO1xuICAgICAgRCAtLi0-IEVbc291cmNlIC5wcm9maWxlLmQvQTAyLnNoXTtcbiAgICAgIEUgLS4tPiBGW3NvdXJjZSAucHJvZmlsZS5kL1o5OS5zaF07XG4gICAgICBGIC0tPiBHO1xuICAgICAgQyAtLT4gfE5vfCBHe1VzZXIgLmJhc2hyY307XG4gICAgICBHIC0tPiB8SW50ZXJhY3RpdmUgc2hlbGw8YnI-UFMxIHZhciBpcyBzZXR8IEhbc291cmNlIC9ldGMvYmFzaHJjXVxuICAgICAgSCAtLT4gSVtVc2VyIC5iYXNoX2FsaWFzZXNdXG4gICAgICBJIC0tPiBKW2Jhc2ggdGVybWluYWwgY29uZmlnLCA8YnI-IGUuZy4sIGNvbG9ycywgaGlzdG9yeSwgZXRjLl1cbiAgICAgIEogLS0-IEtbSW5pdGlhbGl6ZTxicj5Db25kYSBFbnZpcm9ubWVudF1cbiAgICAgIEcgLS0-IHxOb24taW50ZXJhY3RpdmUgc2hlbGw8YnI-UFMxIHZhciBpcyB1bnNldHwgS1xuICAgICAgTFtVc2VyIC5wcm9maWxlLmQvdm9pZF0gLS0-IE1bc291cmNlIDxicj4gLnByb2ZpbGUuZC92b2lkL1ZBMDEuc2hdO1xuICAgICAgTSAtLi0-IE5bc291cmNlIDxicj4gLnByb2ZpbGUuZC92b2lkL1ZaOTkuc2hdO1xuICAgICAgSyAtLT4gTDtcbiAgICAgIE4gLS0-IE9bU2V0IFBBVEhdXG4gICAgICBPIC0tPiBQW1NldCBQUzFdIiwibWVybWFpZCI6IntcbiAgXCJ0aGVtZVwiOiBcImRlZmF1bHRcIlxufSIsInVwZGF0ZUVkaXRvciI6ZmFsc2UsImF1dG9TeW5jIjp0cnVlLCJ1cGRhdGVEaWFncmFtIjpmYWxzZX0) !!! info "Example bash startup files" - Fix link - You can :octicons-file-code-16: [download my bash startup files]({{ repo.url }}{{ repo.tree }}/confs/hpc/user_env/). It will not work by cloning into your linux env. However, each file has inline comments that should help customizing your bash startup. + Fix link - You can :octicons-file-code-16: [download my bash startup files]({{ repo.url }}{{ repo.blob }}/confs/hpc/user_env/). It will not work by cloning into your linux env. However, each file has inline comments that should help customizing your bash startup. And that's all! :checkered_flag: diff --git a/web/docs/hpc/gpu/winter_1.md b/web/docs/hpc/gpu/winter_1.md index b0cbb04..99d2b38 100644 --- a/web/docs/hpc/gpu/winter_1.md +++ b/web/docs/hpc/gpu/winter_1.md @@ -1,8 +1,15 @@ --- title: "Setting up GPU env" -description: "Winter HPC Setup 2021" -keywords: "winter,hpc,gpu,tensorflow,keras,pytorch,machine_learning,conda,jupyter,segmentation" +description: "Getting started with HPC GPU Setup. Configure conda environment specific for GPU based HPC and for deep learning usage. Setup Tensorflow, Keras, PyTorch and imaging-based tools like CellPose, StarDist, and CellProfiler. Configure bash startup specific to GPU based HPC." +keywords: "hpc,gpu,tensorflow,keras,pytorch,machine_learning,deep_learning,conda,jupyter,segmentation,winter,jax" comments: true +tags: + - hpc + - setup + - gpu + - "deep learning" + - imaging + - programming --- Winter HPC at JAX is a GPU-based computing cluster and it is powered by [NVIDIA(r) V100 series](https://www.nvidia.com/en-us/data-center/v100/) GPU cards. If you are working on GPU-based HPC or linux env, following page should guide you on setting up commonly used GPU libraries, e.g., [Tensorflow 2](https://www.tensorflow.org/), [Keras](https://keras.io/about/), and [PyTorch](https://pytorch.org/). GPU setup involves several technical jargon related to hardware compliant libraries, e.g., CUDA toolkit if using NVIDIA marketed GPU cards. I will not go into details of each step here and instead link to installation guide for further details. Knowing such details should be useful while working with deep learning tools and debugging runtime errors. diff --git a/web/docs/tags.md b/web/docs/tags.md new file mode 100644 index 0000000..fef9bb4 --- /dev/null +++ b/web/docs/tags.md @@ -0,0 +1,5 @@ +# Tags + +Following is a list of relevant tags: + +[TAGS]