Merge branch 'master' into IA-4808-welder-uid

.github/workflows/test-terra-jupyter-aou.yml

@@ -42,7 +42,7 @@ jobs:
 
     steps:
     - name: Checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Set up Python
       uses: actions/setup-python@v4
@@ -53,7 +53,7 @@ jobs:
       run: sudo rm -rf /usr/share/dotnet
 
     - id: auth
-      uses: google-github-actions/auth@v1
+      uses: google-github-actions/auth@v2
       with:
         credentials_json: ${{ secrets.TD_GCP_SA_KEY }}
         create_credentials_file: true

config/community_images.json

@@ -10,21 +10,21 @@
 },
 {
     "id": "RStudio",
-    "label": "RStudio (R 4.3.1, Bioconductor 3.17, Python 3.10.12)",
-    "version": "3.17.1",
-    "updated": "2023-09-13",
-    "packages": "https://raw.githubusercontent.com/anvilproject/anvil-docker/main/anvil-rstudio-bioconductor/info/anvil-rstudio-bioconductor-3.17.1-versions.json",
-    "image": "us.gcr.io/broad-dsp-gcr-public/anvil-rstudio-bioconductor:3.17.1",
+    "label": "RStudio (R 4.3.2, Bioconductor 3.18, Python 3.10.12)",
+    "version": "3.18.0",
+    "updated": "2023-11-07",
+    "packages": "https://raw.githubusercontent.com/anvilproject/anvil-docker/main/anvil-rstudio-bioconductor/info/anvil-rstudio-bioconductor-3.18.0-versions.json",
+    "image": "us.gcr.io/broad-dsp-gcr-public/anvil-rstudio-bioconductor:3.18.0",
     "requiresSpark": false,
     "isRStudio": true
 },
 {
     "id": "LegacyRStudio",
-    "label": "Legacy RStudio (R 4.2.3, Bioconductor 3.16, Python 3.10.6)",
-    "version": "3.16.1",
-    "updated": "2023-05-12",
-    "packages": "https://raw.githubusercontent.com/anvilproject/anvil-docker/main/anvil-rstudio-bioconductor/info/anvil-rstudio-bioconductor-3.16.1-versions.json",
-    "image": "us.gcr.io/broad-dsp-gcr-public/anvil-rstudio-bioconductor:3.16.1",
+    "label": "RStudio (R 4.3.1, Bioconductor 3.17, Python 3.10.12)",
+    "version": "3.17.1",
+    "updated": "2023-09-13",
+    "packages": "https://raw.githubusercontent.com/anvilproject/anvil-docker/main/anvil-rstudio-bioconductor/info/anvil-rstudio-bioconductor-3.17.1-versions.json",
+    "image": "us.gcr.io/broad-dsp-gcr-public/anvil-rstudio-bioconductor:3.17.1",
     "requiresSpark": false,
     "isRStudio": true
 }]

config/conf.json

@@ -26,7 +26,7 @@
                     "tidyverse"
                 ]
             },
-            "version" : "2.2.3",
+            "version" : "2.2.4",
             "automated_flags" : {
                 "generate_docs" : true,
                 "include_in_ui" : true,
@@ -47,7 +47,7 @@
                     "hail"
                 ]
             },
-            "version" : "1.1.7",
+            "version" : "1.1.10",
             "automated_flags" : {
                 "generate_docs" : true,
                 "include_in_ui" : true,
@@ -68,7 +68,7 @@
                     "scikit-learn"
                 ]
             },
-            "version" : "1.1.4",
+            "version" : "1.1.5",
             "automated_flags" : {
                 "generate_docs" : true,
                 "include_in_ui" : true,
@@ -100,7 +100,7 @@
                 "r"
             ],
             "packages" : {},
-            "version" : "2.2.3",
+            "version" : "2.2.4",
             "automated_flags" : {
                 "include_in_ui" : false,
                 "generate_docs" : true,
@@ -118,7 +118,7 @@
                 "r"
             ],
             "packages" : {},
-            "version" : "2.3.4",
+            "version" : "2.3.6",
             "automated_flags" : {
                 "include_in_ui" : true,
                 "generate_docs" : true,
@@ -135,7 +135,7 @@
                 "r"
             ],
             "packages" : {},
-            "version" : "2.2.7",
+            "version" : "2.2.11",
             "automated_flags" : {
                 "include_in_ui" : false,
                 "generate_docs" : false,

terra-jupyter-aou/CHANGELOG.md

@@ -1,10 +1,36 @@
-## 2.2.8 - 2022-04-23
-- Update `terra-jupyter-gatk` to `2.3.5`
+## 2.2.12 - 2022-04-23
+- Update `terra-jupyter-gatk` to `2.3.7`
   - Update `terra-jupyter-r` to `2.2.4`
-  - Update `terra-jupyter-python` to `1.1.5`
+  - Update `terra-jupyter-python` to `1.1.6`
     - Update `terra-jupyter-base` to `1.1.4`
       - Change the welder user id
 
+Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.2.12`
+
+## 2.2.11 - 2024-04-10
+- Update `hail` to `0.2.130`
+  - See https://hail.is/docs/0.2/change_log.html#version-0-2-130) for details
+
+Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.2.11`
+
+## 2.2.10 - 2024-04-05
+- Update `hail` to `0.2.129`
+  - See https://hail.is/docs/0.2/change_log.html#version-0-2-129) for details
+
+Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.2.10`
+
+## 2.2.9 - 2024-01-12
+- Update `hail` to `0.2.128`
+  - See https://hail.is/docs/0.2/change_log.html#version-0-2-128) for details
+
+Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.2.9`
+
+## 2.2.8 - 2023-11-14T15:56:44.507726350Z
+
+- Update `terra-jupyter-gatk` to `2.3.6`
+  - Bioconductor 3.18 release
+  - Samtools 1.18
+
 Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.2.8`
 
 ## 2.2.7 - 2023-11-01

terra-jupyter-aou/Dockerfile

@@ -1,4 +1,4 @@
-FROM us.gcr.io/broad-dsp-gcr-public/terra-jupyter-gatk:2.3.5
+FROM us.gcr.io/broad-dsp-gcr-public/terra-jupyter-gatk:2.3.7
 
 USER root
 
@@ -23,7 +23,6 @@ RUN apt-get update && apt-get install -yq --no-install-recommends \
   && apt-get clean \
   && rm -rf /var/lib/apt/lists/*
 
-
 ENV PIP_USER=false
 
 RUN pip3 install --upgrade \
@@ -48,7 +47,7 @@ ENV PYTHONPATH $PYTHONPATH:/usr/lib/spark/python
 
 ENV PYSPARK_PYTHON=python3
 
-ENV HAIL_VERSION=0.2.126
+ENV HAIL_VERSION=0.2.130
 
 RUN find $JUPYTER_HOME/scripts -name '*.sh' -type f | xargs chmod +x \
     && $JUPYTER_HOME/scripts/kernel/kernelspec.sh $JUPYTER_HOME/scripts/kernel /opt/conda/share/jupyter/kernels \
@@ -96,7 +95,7 @@ RUN mkdir -p /tmp/plink && \
 
 RUN mkdir -p /tmp/plink2 && \
   cd /tmp/plink2 && \
-  curl -fsSL -o plink2.zip "https://s3.amazonaws.com/plink2-assets/alpha2/plink2_linux_x86_64.zip" && \
+  curl -fsSL -o plink2.zip "https://s3.amazonaws.com/plink2-assets/plink2_linux_x86_64_latest.zip" && \
   unzip plink2.zip && \
   mv plink2 /bin/plink2 && \
   cd $HOME && \

terra-jupyter-bioconductor/CHANGELOG.md

@@ -1,8 +1,15 @@
-## 2.2.4 - 2024-04-23
+## 2.2.5 - 2024-04-23
 - Update `terra-jupyter-r` to `2.2.4`
   - Update `terra-jupyter-base` to `1.1.4`
     - Change welder user id
 
+Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-bioconductor:2.2.5`
+
+## 2.2.4 - 2023-11-14T15:56:44.470211534Z
+
+- Update `terra-jupyter-r` to `2.2.4`
+  - Bioconductor 3.18 release
+
 Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-bioconductor:2.2.4`
 
 ## 2.2.3 - 2023-10-03

terra-jupyter-gatk/CHANGELOG.md

@@ -1,9 +1,25 @@
-## 2.3.5 - 2024-04-23
+## 2.3.7 - 2024-04-23
 - Update `terra-jupyter-r` to `2.2.4`
-- Update `terra-jupyter-python` to `1.1.5`
+- Update `terra-jupyter-python` to `1.1.6`
   - Update `terra-jupyter-base` to `1.1.4`
     - Change the welder user id
 
+Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-gatk:2.3.7`
+
+## 2.3.6 - 2023-11-27
+
+- Update Samtools to `1.18`
+- Update `terra-jupyter-python` to `2.2.5`
+
+Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-gatk:2.3.6`
+
+## 2.3.5 - 2023-11-14T15:56:44.486168932Z
+
+- Update `terra-jupyter-r` to `2.2.4`
+  - Bioconductor 3.18 release
+
+Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-gatk:2.3.5`
+
 ## 2.3.4 - 2023-10-06
 - Update `terra-jupyter-python` to `1.1.4`
   - Downgrade pytz to `2023.3`

terra-jupyter-gatk/Dockerfile

@@ -16,7 +16,6 @@ RUN apt-get update && apt-get install -yq --no-install-recommends \
   liblzo2-dev \
   zlib1g-dev \
   libz-dev \
-  samtools \
   git-lfs \
   && apt-get clean \
   && rm -rf /var/lib/apt/lists/*
@@ -49,13 +48,25 @@ RUN mkdir -p /tmp/nextflow && \
   chown -R $USER:users $HOME/.nextflow && \
   rm -rf /tmp/nextflow
 
+# Install samtools
+RUN mkdir -p /tmp/samtools && \
+    cd /tmp/samtools && \
+    curl -L -o samtools.tar.bz2 https://github.com/samtools/samtools/releases/download/1.18/samtools-1.18.tar.bz2 && \
+    bzip2 -d samtools.tar.bz2 && \
+    tar xvf samtools.tar && \
+    cd samtools-1.18 && \
+    make && \
+    make install && \
+    cd $HOME && \
+    rm -rf /tmp/samtools
+
 # Install bcftools
 RUN mkdir -p /tmp/bcftools && \
     cd /tmp/bcftools && \
-    curl -L -o bcftools.tar.bz2 https://github.com/samtools/bcftools/releases/download/1.17/bcftools-1.17.tar.bz2 && \
+    curl -L -o bcftools.tar.bz2 https://github.com/samtools/bcftools/releases/download/1.18/bcftools-1.18.tar.bz2 && \
     bzip2 -d bcftools.tar.bz2 && \
     tar xvf bcftools.tar && \
-    cd bcftools-1.17 && \
+    cd bcftools-1.18 && \
     make && \
     make install && \
     cd $HOME && \

terra-jupyter-hail/CHANGELOG.md

@@ -1,8 +1,25 @@
-## 1.1.8 - 2024-04-23
-- Update `terra-jupyter-python` to `1.1.5`
+## 1.1.11 - 2024-04-23
+- Update `terra-jupyter-python` to `1.1.6`
   - Update `terra-jupyter-base` to `1.1.4`
     - Change the welder user id
 
+Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-hail:1.1.11`
+
+## 1.1.10 - 2024-04-10
+- Update `hail` to `0.2.130`
+  - See https://hail.is/docs/0.2/change_log.html#version-0-2-130) for details
+
+Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-hail:1.1.10`
+
+## 1.1.9 - 2024-01-12
+- Update `hail` to `0.2.128`
+  - See https://hail.is/docs/0.2/change_log.html#version-0-2-128) for details
+
+Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-hail:1.1.9`
+
+## 1.1.8 - 2023-12-01
+- Update `terra-jupyter-python` to `1.1.5`
+
 Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-hail:1.1.8`
 
 ## 1.1.7 - 2023-11-01

terra-jupyter-hail/Dockerfile

@@ -7,7 +7,7 @@ USER root
 ENV PIP_USER=false
 ENV PYTHONPATH $PYTHONPATH:/usr/lib/spark/python
 ENV PYSPARK_PYTHON=python3
-ENV HAIL_VERSION=0.2.126
+ENV HAIL_VERSION=0.2.130
 
 RUN find $JUPYTER_HOME/scripts -name '*.sh' -type f | xargs chmod +x \
     && $JUPYTER_HOME/scripts/kernel/kernelspec.sh $JUPYTER_HOME/scripts/kernel /opt/conda/share/jupyter/kernels \

terra-jupyter-python/CHANGELOG.md

@@ -1,7 +1,12 @@
-## 1.1.5 - 2024-04-23
+## 1.1.6 - 2024-04-23
 - Update `terra-jupyter-base` to `1.1.4`
   - Change the welder user id
 
+Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-python:1.1.6`
+
+## 1.1.5 - 2023-11-28
+- Pin bleach to `6.0.0`
+
 Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-python:1.1.5`
 
 ## 1.1.4 - 2023-10-06

terra-jupyter-python/Dockerfile

@@ -51,8 +51,8 @@ ENV TF_ENABLE_ONEDNN_OPTS=1
 
 
 
-# Note: this entrypoint is provided for running Jupyter independently of Leonardo.
-# When Leonardo deploys this image onto a cluster, the entrypoint is overwritten to enable
-# additional setup inside the container before execution.  Jupyter execution occurs when the
-# init-actions.sh script uses 'docker exec' to call run-jupyter.sh.
+# # Note: this entrypoint is provided for running Jupyter independently of Leonardo.
+# # When Leonardo deploys this image onto a cluster, the entrypoint is overwritten to enable
+# # additional setup inside the container before execution.  Jupyter execution occurs when the
+# # init-actions.sh script uses 'docker exec' to call run-jupyter.sh.
 ENTRYPOINT ["/opt/conda/bin/jupyter", "notebook"]

terra-jupyter-python/requirements.txt

@@ -16,7 +16,7 @@ biopython
 bx-python
 fastinterval
 matplotlib-venn
-bleach
+bleach==6.0.0
 cycler
 h5py
 html5lib
@@ -42,3 +42,5 @@ google-resumable-media
 #adding intel optimized xgboost and intel extension for scikit-learn
 scikit-learn-intelex
 xgboost
+#older version of nbconvert fail to convert notebooks to html
+nbconvert>=7.7.3

terra-jupyter-r/CHANGELOG.md

@@ -1,7 +1,13 @@
-## 2.2.4 - 2024-04-23
+## 2.2.5 - 2024-04-23
 - Update `terra-jupyter-base` to `1.1.4`
   - Change the welder user id
 
+Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-r:2.2.5`
+
+## 2.2.4 - 2023-11-14T15:56:44.443656567Z
+
+- Bioconductor 3.18 release
+
 Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-r:2.2.4`
 
 ## 2.2.3 - 2023-10-03

terra-jupyter-r/Dockerfile

@@ -5,8 +5,8 @@ USER root
 COPY scripts $JUPYTER_HOME/scripts
 
 # Add env vars to identify binary package installation
-ENV TERRA_R_PLATFORM="terra-jupyter-r-1.1.3"
-ENV TERRA_R_PLATFORM_BINARY_VERSION=3.17
+ENV TERRA_R_PLATFORM="terra-jupyter-r-2.2.4"
+ENV TERRA_R_PLATFORM_BINARY_VERSION=3.18
 
 # Install protobuf 3.20.3. Note this version comes from base deep learning image. Use `conda list` to see what's installed
 RUN cd /tmp \
@@ -103,6 +103,10 @@ RUN apt-get update \
         libeigen3-dev \
         ## for rawrr package
         mono-runtime \
+        ## for nloptr package
+	cmake \
+ 	## for archive package
+ 	libarchive-dev \
     && ln -s /usr/lib/gcc/x86_64-linux-gnu/7/libgfortran.so /usr/lib/x86_64-linux-gnu/libgfortran.so \
     && ln -s /usr/lib/gcc/x86_64-linux-gnu/7/libstdc++.so /usr/lib/x86_64-linux-gnu/libstdc++.so \
     && apt-get clean \
@@ -128,7 +132,7 @@ RUN pip3 -V \
 
 RUN R -e 'install.packages("BiocManager")' \
     ## check version
-    && R -e 'BiocManager::install(version="3.17", ask=FALSE)' \
+    && R -e 'BiocManager::install(version="3.18", ask=FALSE)' \
     && R -e 'BiocManager::install(c( \
     "boot", \
     "class", \