diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index cb569f9f..a407985c 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -35,6 +35,9 @@ jobs: with: python-version: '3.10' + - name: Free up some disk space + run: sudo rm -rf /usr/share/dotnet + - id: auth uses: google-github-actions/auth@v1 with: diff --git a/.github/workflows/test-terra-jupyter-aou.yml b/.github/workflows/test-terra-jupyter-aou.yml index 836644ad..f64e24f1 100644 --- a/.github/workflows/test-terra-jupyter-aou.yml +++ b/.github/workflows/test-terra-jupyter-aou.yml @@ -87,7 +87,7 @@ jobs: /bin/bash -c 'for nb in {terra-jupyter-python/tests,terra-jupyter-aou/tests}/*ipynb ; do jupyter nbconvert --to html_embed --ExecutePreprocessor.allow_errors=True --execute "${nb}" ; done' - name: Upload workflow artifacts - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: notebook-execution-results path: | diff --git a/.github/workflows/test-terra-jupyter-base.yml b/.github/workflows/test-terra-jupyter-base.yml index 710c8cc6..295afc15 100644 --- a/.github/workflows/test-terra-jupyter-base.yml +++ b/.github/workflows/test-terra-jupyter-base.yml @@ -69,7 +69,7 @@ jobs: ./build_smoke_test_image.sh terra-jupyter-base - name: Upload workflow artifacts - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: notebook-execution-results path: terra-jupyter-base/tests/*.html diff --git a/.github/workflows/test-terra-jupyter-bioconductor.yml b/.github/workflows/test-terra-jupyter-bioconductor.yml index 64bcadd5..669e6df1 100644 --- a/.github/workflows/test-terra-jupyter-bioconductor.yml +++ b/.github/workflows/test-terra-jupyter-bioconductor.yml @@ -87,7 +87,7 @@ jobs: /bin/sh -c 'for nb in *ipynb ; do jupyter nbconvert --to html_embed --ExecutePreprocessor.timeout=300 --execute "${nb}" ; done' - name: Upload workflow artifacts - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: notebook-execution-results path: terra-jupyter-bioconductor/tests/*.html diff --git a/.github/workflows/test-terra-jupyter-gatk.yml b/.github/workflows/test-terra-jupyter-gatk.yml index cdbcf044..ebb10bf8 100644 --- a/.github/workflows/test-terra-jupyter-gatk.yml +++ b/.github/workflows/test-terra-jupyter-gatk.yml @@ -87,7 +87,7 @@ jobs: /bin/bash -c 'for nb in {terra-jupyter-python/tests,terra-jupyter-gatk/tests}/*ipynb ; do jupyter nbconvert --to html_embed --ExecutePreprocessor.allow_errors=True --execute "${nb}" ; done' - name: Upload workflow artifacts - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: notebook-execution-results path: | diff --git a/.github/workflows/test-terra-jupyter-hail.yml b/.github/workflows/test-terra-jupyter-hail.yml index 968179bf..18c172ff 100644 --- a/.github/workflows/test-terra-jupyter-hail.yml +++ b/.github/workflows/test-terra-jupyter-hail.yml @@ -87,7 +87,7 @@ jobs: /bin/bash -c 'for nb in {terra-jupyter-python/tests,terra-jupyter-hail/tests}/*ipynb ; do jupyter nbconvert --to html_embed --ExecutePreprocessor.allow_errors=True --execute "${nb}" ; done' - name: Upload workflow artifacts - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: notebook-execution-results path: | diff --git a/.github/workflows/test-terra-jupyter-python.yml b/.github/workflows/test-terra-jupyter-python.yml index 3d6f3116..63fd0f14 100644 --- a/.github/workflows/test-terra-jupyter-python.yml +++ b/.github/workflows/test-terra-jupyter-python.yml @@ -92,7 +92,7 @@ jobs: /bin/sh -c 'for nb in *ipynb ; do jupyter nbconvert --to html_embed --ExecutePreprocessor.allow_errors=True --execute "${nb}" ; done' - name: Upload workflow artifacts - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: notebook-execution-results path: terra-jupyter-python/tests/*.html diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 65f4b446..13369a07 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,7 +6,7 @@ We do our work on a feature branch based on master. When you open a pull request, add the JIRA issue number to the PR title. This will make a reference from JIRA to the GitHub issue. Briefly describe your changes in the description section. -This is also a good opportunity to verify that all the appropriate image versions were bumped. It is important to keep in mind which images extend from where ([Image Dependecies](https://github.com/DataBiosphere/terra-docker#image-dependencies)). Make sure to bump the version of the image you are changing and update its reference in all images that extend from it. +This is also a good opportunity to verify that all the appropriate image versions were bumped. It is important to keep in mind which images extend from where ([Image Dependecies](https://github.com/DataBiosphere/terra-docker#image-dependencies)). Make sure to bump the version of the image you are changing in `config/conf.json` and update its reference in all images that extend from it. You can also run `amm ./updateVersions.sc ` to upgrade derived images. If you're updating `terra-jupyter-aou` image, please make a pull request similar to [this PR](https://github.com/DataBiosphere/leonardo/pull/1612) (less updating `terra_jupyter_aou_old`). diff --git a/config/conf.json b/config/conf.json index f71b554c..16340e4e 100644 --- a/config/conf.json +++ b/config/conf.json @@ -26,7 +26,7 @@ "tidyverse" ] }, - "version" : "2.2.5", + "version" : "2.2.6", "automated_flags" : { "generate_docs" : true, "include_in_ui" : true, @@ -47,7 +47,7 @@ "hail" ] }, - "version" : "1.1.12", + "version" : "1.1.13", "automated_flags" : { "generate_docs" : true, "include_in_ui" : true, @@ -68,7 +68,7 @@ "scikit-learn" ] }, - "version" : "1.1.5", + "version" : "1.1.6", "automated_flags" : { "generate_docs" : true, "include_in_ui" : true, @@ -84,9 +84,9 @@ "python" ], "packages" : { - + }, - "version" : "1.1.3", + "version" : "1.1.4", "automated_flags" : { "generate_docs" : true, "include_in_ui" : false, @@ -102,9 +102,9 @@ "r" ], "packages" : { - + }, - "version" : "2.2.5", + "version" : "2.2.6", "automated_flags" : { "include_in_ui" : false, "generate_docs" : true, @@ -122,9 +122,9 @@ "r" ], "packages" : { - + }, - "version" : "2.3.7", + "version" : "2.3.8", "automated_flags" : { "include_in_ui" : true, "generate_docs" : true, @@ -141,9 +141,9 @@ "r" ], "packages" : { - + }, - "version" : "2.2.13", + "version" : "2.2.14", "automated_flags" : { "include_in_ui" : false, "generate_docs" : false, @@ -159,7 +159,7 @@ "r" ], "packages" : { - + }, "version" : "0.1.1", "automated_flags" : { @@ -176,7 +176,7 @@ "tools" : [ ], "packages" : { - + }, "version" : "0.0.1", "automated_flags" : { diff --git a/terra-jupyter-aou/CHANGELOG.md b/terra-jupyter-aou/CHANGELOG.md index 8daaab99..ef10e31d 100644 --- a/terra-jupyter-aou/CHANGELOG.md +++ b/terra-jupyter-aou/CHANGELOG.md @@ -1,3 +1,11 @@ +## 2.2.14 - 2024-09-04 +- Update `terra-jupyter-gatk` to `2.3.8` + - Update `terra-jupyter-python` to `1.1.6` + - Update `terra-jupyter-base` to `1.1.4` + - Update terra-notebook-utils to `0.14.0` + +Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.2.14` + ## 2.2.13 - 2024-08-12 - Update `hail` to `0.2.130.post1` - See https://github.com/hail-is/hail/releases/tag/0.2.130.post1 for details diff --git a/terra-jupyter-aou/Dockerfile b/terra-jupyter-aou/Dockerfile index 64a79974..aa5f471d 100644 --- a/terra-jupyter-aou/Dockerfile +++ b/terra-jupyter-aou/Dockerfile @@ -1,4 +1,4 @@ -FROM us.gcr.io/broad-dsp-gcr-public/terra-jupyter-gatk:2.3.7 +FROM us.gcr.io/broad-dsp-gcr-public/terra-jupyter-gatk:2.3.8 USER root @@ -195,10 +195,10 @@ RUN apt-get update && \ && rm -rf /var/lib/apt/lists/* RUN pip install --upgrade pip && pip install --verbose h5py -# RW-8725 INSTALL QIMEE2 -RUN curl -fsSL -o qiime2-2022.8.yml https://data.qiime2.org/distro/core/qiime2-2022.8-py38-linux-conda.yml && \ - conda env create -n qiime2-2022.8 --file qiime2-2022.8.yml && \ - rm qiime2-2022.8.yml +# QIIME 2 2024.5 Amplicon Distribution +RUN curl -fsSL -o qiime2-2024.5.yml https://data.qiime2.org/distro/amplicon/qiime2-amplicon-2024.5-py39-linux-conda.yml && \ + conda env create -n qiime2-2024.5 --file qiime2-2024.5.yml && \ + rm qiime2-2024.5.yml # Install jellyfish RUN mkdir -p /tmp/jellyfish && \ diff --git a/terra-jupyter-base/CHANGELOG.md b/terra-jupyter-base/CHANGELOG.md index 6a0e5572..66ab6082 100644 --- a/terra-jupyter-base/CHANGELOG.md +++ b/terra-jupyter-base/CHANGELOG.md @@ -1,3 +1,8 @@ +## 1.1.4 - 2024-09-04 +- Update terra-notebook-utils to `0.14.0` + +Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-base:1.1.4` + ## 1.1.3 - 2023-10-03 - Downgrade notebook to `6.5.4` and jupyterlab-server to `2.23.0` diff --git a/terra-jupyter-base/Dockerfile b/terra-jupyter-base/Dockerfile index 3feb7517..d3f72be4 100644 --- a/terra-jupyter-base/Dockerfile +++ b/terra-jupyter-base/Dockerfile @@ -108,10 +108,13 @@ RUN pip3 -V \ # for jupyter_delocalize.py and jupyter_notebook_config.py # DO NOT CHANGE OR UNPIN NOTEBOOK OR JUPYTERLAB-SERVER! When trying with 6.5.5 and 2.24.0 notebook commands could not find the notebook config file && pip3 install notebook==6.5.4 \ + && pip3 install jupyter-server==1.24.0 \ && pip3 install jupyterlab-server==2.23.0 \ + # See: https://stackoverflow.com/questions/77549493/modulenotfounderror-no-module-named-jupyter-server-contents/77549554#77549554 + && pip3 install traitlets==5.9.0 \ && pip3 install requests \ && pip3 install firecloud \ - && pip3 install terra-notebook-utils==0.13.0 \ + && pip3 install terra-notebook-utils==0.14.0 \ && pip3 install crcmod \ # For gcloud alpha storage support. && pip3 install google-crc32c --target /usr/lib/google-cloud-sdk/lib/third_party \ diff --git a/terra-jupyter-bioconductor/CHANGELOG.md b/terra-jupyter-bioconductor/CHANGELOG.md index dce91612..86d31fd2 100644 --- a/terra-jupyter-bioconductor/CHANGELOG.md +++ b/terra-jupyter-bioconductor/CHANGELOG.md @@ -1,3 +1,11 @@ +## 2.2.6 - 2024-09-04 + +- Update `terra-jupyter-r` to `2.2.6` + - Update `terra-jupyter-base` to `1.1.4` + - Update terra-notebook-utils to `0.14.0` + +Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-bioconductor:2.2.6` + ## 2.2.5 - 2024-05-15T15:36:28.576892790Z - Update `terra-jupyter-r` to `2.2.5` diff --git a/terra-jupyter-bioconductor/Dockerfile b/terra-jupyter-bioconductor/Dockerfile index 916163fd..43f74e12 100644 --- a/terra-jupyter-bioconductor/Dockerfile +++ b/terra-jupyter-bioconductor/Dockerfile @@ -1 +1 @@ -FROM us.gcr.io/broad-dsp-gcr-public/terra-jupyter-r:2.2.5 +FROM us.gcr.io/broad-dsp-gcr-public/terra-jupyter-r:2.2.6 diff --git a/terra-jupyter-gatk/CHANGELOG.md b/terra-jupyter-gatk/CHANGELOG.md index 0069b6df..329ea343 100644 --- a/terra-jupyter-gatk/CHANGELOG.md +++ b/terra-jupyter-gatk/CHANGELOG.md @@ -1,3 +1,11 @@ +## 2.3.8 - 2024-09-04 +- Update `terra-jupyter-python` to `1.1.6` +- Update `terra-jupyter-r` to `2.2.6` + - Update `terra-jupyter-base` to `1.1.4` + - Update terra-notebook-utils to `0.14.0` + +Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-gatk:2.3.8` + ## 2.3.7 - 2024-05-15T15:36:28.602743427Z - Update `terra-jupyter-r` to `2.2.5` diff --git a/terra-jupyter-gatk/Dockerfile b/terra-jupyter-gatk/Dockerfile index 4e182a70..742a545c 100644 --- a/terra-jupyter-gatk/Dockerfile +++ b/terra-jupyter-gatk/Dockerfile @@ -1,6 +1,6 @@ -FROM us.gcr.io/broad-dsp-gcr-public/terra-jupyter-python:1.1.5 AS python +FROM us.gcr.io/broad-dsp-gcr-public/terra-jupyter-python:1.1.6 AS python -FROM us.gcr.io/broad-dsp-gcr-public/terra-jupyter-r:2.2.5 +FROM us.gcr.io/broad-dsp-gcr-public/terra-jupyter-r:2.2.6 USER root @@ -27,7 +27,7 @@ ENV PIP_USER=false # uninstalling packages: https://github.com/conda/conda/issues/10357 COPY --from=python /etc/terra-docker/requirements.txt /etc/terra-docker/base_requirements.txt -RUN pip install --upgrade -r /etc/terra-docker/base_requirements.txt +RUN pip install -r /etc/terra-docker/base_requirements.txt # Install GATK ENV GATK_VERSION 4.3.0.0 diff --git a/terra-jupyter-hail/CHANGELOG.md b/terra-jupyter-hail/CHANGELOG.md index 61fd1b86..328fa845 100644 --- a/terra-jupyter-hail/CHANGELOG.md +++ b/terra-jupyter-hail/CHANGELOG.md @@ -1,3 +1,10 @@ +## 1.1.13 - 2024-09-04 +- Update `terra-jupyter-python` to `1.1.6` + - Update `terra-jupyter-base` to `1.1.4` + - Update terra-notebook-utils to `0.14.0` + +Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-hail:1.1.13` + ## 1.1.12 - 2024-08-12 - Update `hail` to `0.2.130.post1` - See https://github.com/hail-is/hail/releases/tag/0.2.130.post1 for details diff --git a/terra-jupyter-hail/Dockerfile b/terra-jupyter-hail/Dockerfile index 4abe843d..8a802ef3 100644 --- a/terra-jupyter-hail/Dockerfile +++ b/terra-jupyter-hail/Dockerfile @@ -1,4 +1,4 @@ -FROM us.gcr.io/broad-dsp-gcr-public/terra-jupyter-python:1.1.5 +FROM us.gcr.io/broad-dsp-gcr-public/terra-jupyter-python:1.1.6 USER root diff --git a/terra-jupyter-python/CHANGELOG.md b/terra-jupyter-python/CHANGELOG.md index 1ee01976..0f4c2464 100644 --- a/terra-jupyter-python/CHANGELOG.md +++ b/terra-jupyter-python/CHANGELOG.md @@ -1,3 +1,9 @@ +## 1.1.6 - 2024-09-04 +- Update `terra-jupyter-base` to `1.1.4` + - Update terra-notebook-utils to `0.14.0` + +Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-python:1.1.6` + ## 1.1.5 - 2023-11-28 - Pin bleach to `6.0.0` diff --git a/terra-jupyter-python/Dockerfile b/terra-jupyter-python/Dockerfile index b98b5220..a80f6c50 100644 --- a/terra-jupyter-python/Dockerfile +++ b/terra-jupyter-python/Dockerfile @@ -1,4 +1,4 @@ -FROM us.gcr.io/broad-dsp-gcr-public/terra-jupyter-base:1.1.3 +FROM us.gcr.io/broad-dsp-gcr-public/terra-jupyter-base:1.1.4 USER root # This makes it so pip runs as root, not the user. diff --git a/terra-jupyter-python/requirements.txt b/terra-jupyter-python/requirements.txt index 6ee08d33..8a24cc44 100644 --- a/terra-jupyter-python/requirements.txt +++ b/terra-jupyter-python/requirements.txt @@ -44,3 +44,9 @@ scikit-learn-intelex xgboost #older version of nbconvert fail to convert notebooks to html nbconvert>=7.7.3 +# See: https://github.com/GoogleCloudPlatform/generative-ai/issues/849 +shapely==1.8.5.post1 +pygeos==0.12.0 +geopandas==0.10.2 +# See: https://github.com/pypa/setuptools/issues/4483 +setuptools<71.0.0 diff --git a/terra-jupyter-python/tests/smoke_test.ipynb b/terra-jupyter-python/tests/smoke_test.ipynb index abaa3328..862eeb55 100644 --- a/terra-jupyter-python/tests/smoke_test.ipynb +++ b/terra-jupyter-python/tests/smoke_test.ipynb @@ -85,9 +85,9 @@ "metadata": {}, "outputs": [], "source": [ - "output = !pip3 show pendulum\n", + "output = !pip3 show emoji\n", "print(output) # Should show not yet installed.\n", - "assert(0 == output.count('Name: pendulum'))" + "assert(0 == output.count('Name: emoji'))" ] }, { @@ -96,7 +96,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install pendulum==2.1.2 --user --ignore-installed" + "%pip install emoji --user --ignore-installed" ] }, { @@ -105,9 +105,9 @@ "metadata": {}, "outputs": [], "source": [ - "output = !pip3 show pendulum\n", + "output = !pip3 show emoji\n", "print(output) # Should show that it is now installed!\n", - "assert(1 == output.count('Name: pendulum'))" + "assert(1 == output.count('Name: emoji'))" ] }, { @@ -248,7 +248,7 @@ "source": [ "import numpy as np\n", "import pandas as pd\n", - "from pandas_profiling import ProfileReport\n", + "from ydata_profiling import ProfileReport\n", "\n", "df = pd.DataFrame(\n", " np.random.rand(100, 5),\n", diff --git a/terra-jupyter-r/CHANGELOG.md b/terra-jupyter-r/CHANGELOG.md index 6a369f7a..6dff7527 100644 --- a/terra-jupyter-r/CHANGELOG.md +++ b/terra-jupyter-r/CHANGELOG.md @@ -1,3 +1,9 @@ +## 2.2.6 - 2024-09-04 +- Update `terra-jupyter-base` to `1.1.4` + - Update terra-notebook-utils to `0.14.0` + +Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-r:2.2.6` + ## 2.2.5 - 2024-05-15T15:36:28.555690376Z - Update for R 4.4.0 and Bioc 3.19 diff --git a/terra-jupyter-r/Dockerfile b/terra-jupyter-r/Dockerfile index 985b154b..a79d41bb 100644 --- a/terra-jupyter-r/Dockerfile +++ b/terra-jupyter-r/Dockerfile @@ -1,11 +1,11 @@ -FROM us.gcr.io/broad-dsp-gcr-public/terra-jupyter-base:1.1.3 +FROM us.gcr.io/broad-dsp-gcr-public/terra-jupyter-base:1.1.4 USER root COPY scripts $JUPYTER_HOME/scripts # Add env vars to identify binary package installation -ENV TERRA_R_PLATFORM="terra-jupyter-r-2.2.5" +ENV TERRA_R_PLATFORM="terra-jupyter-r-2.2.6" ENV TERRA_R_PLATFORM_BINARY_VERSION=3.19 # Install protobuf 3.20.3. Note this version comes from base deep learning image. Use `conda list` to see what's installed @@ -117,8 +117,10 @@ RUN apt-get update \ # Add libsbml CFLAGS ENV LIBSBML_CFLAGS="-I/usr/include" ENV LIBSBML_LIBS="-lsbml" +ENV XML_CONFIG="/usr/bin/xml2-config" RUN echo 'export LIBSBML_CFLAGS="-I/usr/include"' >> /etc/profile \ - && echo 'export LIBSBML_LIBS="-lsbml"' >> /etc/profile + && echo 'export LIBSBML_LIBS="-lsbml"' >> /etc/profile \ + && echo 'export XML_CONFIG="/usr/bin/xml2-config"' >> /etc/profile ## set pip3 to run as root, not as jupyter user ENV PIP_USER=false @@ -187,4 +189,4 @@ ENV PIP_USER=true RUN R -e 'IRkernel::installspec(user=FALSE)' \ && chown -R $USER:users /usr/local/lib/R/site-library /home/jupyter -USER $USER +USER $USER \ No newline at end of file