adding smoke test, updating gha versions, and adding a readme

.github/workflows/nodejs.yml

@@ -13,7 +13,7 @@ jobs:
         node_version: [8, 10, 12]
 
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v4
     - name: Use Node.js ${{ matrix.node-version }}
       uses: actions/setup-node@v1
       with:

.github/workflows/publish.yml

@@ -26,7 +26,7 @@ jobs:
     runs-on: self-hosted
     steps:
       - name: Checkout current code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: ${{ github.event.inputs.branch }}
 
@@ -36,7 +36,7 @@ jobs:
           python-version: '3.10'
 
       - id: auth
-        uses: google-github-actions/auth@v1
+        uses: google-github-actions/auth@v2
         with:
           credentials_json: ${{ secrets.TD_GCP_SA_KEY }}
           create_credentials_file: true

.github/workflows/test-terra-base-jupyter.yml

@@ -0,0 +1,89 @@
+name: Test terra-base-jupyter
+# Perform smoke tests on the terra-base-jupyter Docker image to have some amount of confidence that
+# Python package versions are compatible.
+#
+# To configure the minimal auth needed for these tests to be able to read public data from Google Cloud Platform:
+# Step 1: Create a service account per these instructions:
+#         https://github.com/google-github-actions/setup-gcloud/blob/master/setup-gcloud/README.md
+# Step 2: Give the service account the following permissions within the project: BigQuery User
+# Step 3: Store its key and project id as GitHub repository secrets TD_GCP_SA_KEY and GCP_PROJECT_ID.
+#         https://docs.github.com/en/free-pro-team@latest/actions/reference/encrypted-secrets#creating-encrypted-secrets-for-a-repository
+
+on:
+  pull_request:
+    branches: [ master ]
+    paths:
+    - 'terra-base-jupyter/**'
+    - '.github/workflows/test-terra-base-jupyter.yml'
+
+  push:
+    # Note: GitHub secrets are not passed to pull requests from forks. For community contributions from
+    # regular contributors, its a good idea for the contributor to configure the GitHub actions to run correctly
+    # in their fork as described above.
+    #
+    # For occasional contributors, the dev team will merge the PR fork branch to a branch in upstream named
+    # test-community-contribution-<PR#> to run all the GitHub Action smoke tests.
+    branches: [ 'test-community-contribution*' ]
+    paths:
+    - 'terra-base-jupyter/**'
+    - '.github/workflows/test-terra-base-jupyter.yml'
+
+  workflow_dispatch:
+    # Allows manually triggering of workflow on a selected branch via the GitHub Actions tab.
+    # GitHub blog demo: https://github.blog/changelog/2020-07-06-github-actions-manual-triggers-with-workflow_dispatch/.
+
+env:
+  GOOGLE_PROJECT: ${{ secrets.GCP_PROJECT_ID }}
+
+jobs:
+
+  test_docker_image:
+    runs-on: self-hosted
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.10'
+
+    - name: Free up some disk space
+      run: sudo rm -rf /usr/share/dotnet
+
+    - id: auth
+      uses: google-github-actions/auth@v2
+      with:
+        credentials_json: ${{ secrets.TD_GCP_SA_KEY }}
+        create_credentials_file: true
+
+    - name: Set up Cloud SDK
+      uses: google-github-actions/setup-gcloud@v0.3.0
+      with:
+        project_id: ${{ secrets.GCP_PROJECT_ID }}
+
+    - name: Build Docker image and base images too, if needed
+      run: |
+        gcloud auth configure-docker
+        ./build_smoke_test_image.sh terra-base-jupyter
+
+    - name: Upload workflow artifacts
+      uses: actions/upload-artifact@v2
+      with:
+        name: notebook-execution-results
+        path: terra-base-jupyter/tests/*.html
+        retention-days: 30
+
+    - name: Test Python code with pytest
+      run: |
+        chmod a+r "${{ steps.auth.outputs.credentials_file_path }}"
+        docker run \
+          --env GOOGLE_PROJECT \
+          --volume "${{ steps.auth.outputs.credentials_file_path }}":/tmp/credentials.json:ro \
+          --env GOOGLE_APPLICATION_CREDENTIALS="/tmp/credentials.json" \
+          --volume $GITHUB_WORKSPACE/terra-base-jupyter/tests:/tests \
+          --workdir=/tests \
+          --entrypoint="" \
+          terra-base-jupyter:smoke-test \
+          /bin/sh -c "pip3 install pytest; pytest"

.github/workflows/test-terra-jupyter-aou.yml

@@ -42,7 +42,7 @@ jobs:
 
     steps:
     - name: Checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Set up Python
       uses: actions/setup-python@v4
@@ -53,7 +53,7 @@ jobs:
       run: sudo rm -rf /usr/share/dotnet
 
     - id: auth
-      uses: google-github-actions/auth@v1
+      uses: google-github-actions/auth@v2
       with:
         credentials_json: ${{ secrets.TD_GCP_SA_KEY }}
         create_credentials_file: true

.github/workflows/test-terra-jupyter-base.yml

@@ -42,7 +42,7 @@ jobs:
 
     steps:
     - name: Checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Set up Python
       uses: actions/setup-python@v4
@@ -53,7 +53,7 @@ jobs:
       run: sudo rm -rf /usr/share/dotnet
 
     - id: auth
-      uses: google-github-actions/auth@v1
+      uses: google-github-actions/auth@v2
       with:
         credentials_json: ${{ secrets.TD_GCP_SA_KEY }}
         create_credentials_file: true

.github/workflows/test-terra-jupyter-bioconductor.yml

@@ -42,7 +42,7 @@ jobs:
 
     steps:
     - name: Checkout
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
 
     - name: Set up Python
       uses: actions/setup-python@v2
@@ -53,7 +53,7 @@ jobs:
       run: sudo rm -rf /usr/share/dotnet
 
     - id: auth
-      uses: google-github-actions/auth@v1
+      uses: google-github-actions/auth@v2
       with:
         credentials_json: ${{ secrets.TD_GCP_SA_KEY }}
         create_credentials_file: true

.github/workflows/test-terra-jupyter-gatk.yml

@@ -42,7 +42,7 @@ jobs:
 
     steps:
     - name: Checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Set up Python
       uses: actions/setup-python@v4
@@ -53,7 +53,7 @@ jobs:
       run: sudo rm -rf /usr/share/dotnet
 
     - id: auth
-      uses: google-github-actions/auth@v1
+      uses: google-github-actions/auth@v2
       with:
         credentials_json: ${{ secrets.TD_GCP_SA_KEY }}
         create_credentials_file: true

.github/workflows/test-terra-jupyter-hail.yml

@@ -42,7 +42,7 @@ jobs:
 
     steps:
     - name: Checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Set up Python
       uses: actions/setup-python@v4
@@ -53,7 +53,7 @@ jobs:
       run: sudo rm -rf /usr/share/dotnet
 
     - id: auth
-      uses: google-github-actions/auth@v1
+      uses: google-github-actions/auth@v2
       with:
         credentials_json: ${{ secrets.TD_GCP_SA_KEY }}
         create_credentials_file: true

.github/workflows/test-terra-jupyter-python.yml

@@ -44,7 +44,7 @@ jobs:
 
     steps:
     - name: Checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Set up Python
       uses: actions/setup-python@v4
@@ -55,7 +55,7 @@ jobs:
       run: sudo rm -rf /usr/share/dotnet
 
     - id: auth
-      uses: google-github-actions/auth@v1
+      uses: google-github-actions/auth@v2
       with:
         credentials_json: ${{ secrets.TD_GCP_SA_KEY }}
         create_credentials_file: true

terra-base-jupyter/README.md

@@ -1,24 +1,23 @@
-# terra-base image
+# terra-base-jupyter image
 
-This repo contains the terra-base image that is compatible with the
-notebook service in [Terra]("https://app.terra.bio/") called Leonardo.
-The goal is to provide a lean, GPU-enabled base image that can be use for both Terra-suported, and custome image builds.
-[See design doc]("https://broadworkbench.atlassian.net/wiki/spaces/IA/pages/2842460165/2023-08-28+Terra+Docker+Refactoring+POC#Provide-a-leaner-set-of-terra-docker-images")
+This repo contains the terra-base-jupyter image that is compatible with the
+jupyter notebook service in [Terra]("https://app.terra.bio/") called Leonardo.
+The goal is to provide a lean, GPU-enabled base image that can be use for both Terra-suported, and custom image builds.
+
+The goal is to provide the user with a base conda environment where they can install any
+package they need, while keeping the jupyter server itself in an isolated virtual environment
+that DSP devs control using poetry as the dependency manager
 
 ## Image contents
 
-`terra-base` extends an cuda 12.2.0 and Ubuntu 22.04 base image with the minimum 
+`terra-base-jupyter` extends an cuda 12.2.0 and Ubuntu 22.04 base image with the minimum 
 requirements necessary to set up Jupyter and provide compatibility with Leonardo and GPU machines.
 
 - OS prerequisites
-- Python 3.10
-- R 4.1.2
-- Anaconda
-- Jupyter
-- Leonardo customizations/extensions
-- Terra python client library ([FISS](https://github.com/broadinstitute/fiss))
-- Terra notebook utils
-- Full list of python packages is available [here](pyproject.toml)
+- Miniconda with python 3.10
+- Two python virtual environments based on miniconda python
+-- The user base Conda environment, full list of python packages is available [here](conda-environment.yml)
+-- The jupyter virtual environment managed separately by poetry, full list of python packages is available [here](pyproject.toml) that contains the terra-specific configurations (e.g. WelderContentManager)
 
 To see the complete contents of this image please see the [Dockerfile](./Dockerfile).
 
@@ -67,7 +66,7 @@ $> export LDFLAGS="-L/opt/homebrew/opt/libomp/lib -L/opt/homebrew/opt/llvm/lib"
 $> export CPPFLAGS="-I/opt/homebrew/opt/libomp/include -I/opt/homebrew/opt/llvmp/include"
 ```
 
-From inside the `terra-base` directory, run the following command to install
+From inside the `terra-base-jupyter` directory, run the following command to install
 the python 3.10 environment with the relevant packages:
 
 ```bash

terra-base-jupyter/tests/smoke_test.py

@@ -0,0 +1,19 @@
+"""Brief smoke tests for terra-base-jupyter.
+
+Minimal tests simply import packages and run 'hello world' code to determine whether any exceptions
+are thrown.
+
+TODO(all): Add additional tests and/or tests with particular assertions, as we encounter Python package version
+incompatibilities not currently detected by these tests.
+"""
+
+import os
+import pytest
+import subprocess
+
+def test_firecloud():
+  import firecloud
+
+def test_terra_notebook_utils():
+  os.system("gcloud auth application-default login")
+  import terra_notebook_utils