From c455c8e9aa049d7936ac913b1b03cadd481497dc Mon Sep 17 00:00:00 2001 From: Saman Ehsan Date: Tue, 3 Sep 2024 16:43:46 -0400 Subject: [PATCH 01/21] Update TNU version in Dockerfile Use latest version 0.14 --- terra-jupyter-base/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terra-jupyter-base/Dockerfile b/terra-jupyter-base/Dockerfile index 3feb7517..4689b13e 100644 --- a/terra-jupyter-base/Dockerfile +++ b/terra-jupyter-base/Dockerfile @@ -111,7 +111,7 @@ RUN pip3 -V \ && pip3 install jupyterlab-server==2.23.0 \ && pip3 install requests \ && pip3 install firecloud \ - && pip3 install terra-notebook-utils==0.13.0 \ + && pip3 install terra-notebook-utils==0.14.0 \ && pip3 install crcmod \ # For gcloud alpha storage support. && pip3 install google-crc32c --target /usr/lib/google-cloud-sdk/lib/third_party \ From 9fa5420b2b09b17073218719e9490fd52e87d2a6 Mon Sep 17 00:00:00 2001 From: Saman Ehsan Date: Wed, 4 Sep 2024 14:35:25 -0400 Subject: [PATCH 02/21] Update image versions --- CONTRIBUTING.md | 2 +- config/conf.json | 26 ++++++++++++------------- terra-jupyter-aou/CHANGELOG.md | 8 ++++++++ terra-jupyter-aou/Dockerfile | 2 +- terra-jupyter-base/CHANGELOG.md | 5 +++++ terra-jupyter-bioconductor/CHANGELOG.md | 8 ++++++++ terra-jupyter-bioconductor/Dockerfile | 2 +- terra-jupyter-gatk/CHANGELOG.md | 7 +++++++ terra-jupyter-gatk/Dockerfile | 2 +- terra-jupyter-hail/CHANGELOG.md | 7 +++++++ terra-jupyter-hail/Dockerfile | 2 +- terra-jupyter-python/CHANGELOG.md | 6 ++++++ terra-jupyter-python/Dockerfile | 2 +- terra-jupyter-r/CHANGELOG.md | 6 ++++++ terra-jupyter-r/Dockerfile | 2 +- 15 files changed, 67 insertions(+), 20 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 65f4b446..13369a07 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,7 +6,7 @@ We do our work on a feature branch based on master. When you open a pull request, add the JIRA issue number to the PR title. This will make a reference from JIRA to the GitHub issue. Briefly describe your changes in the description section. -This is also a good opportunity to verify that all the appropriate image versions were bumped. It is important to keep in mind which images extend from where ([Image Dependecies](https://github.com/DataBiosphere/terra-docker#image-dependencies)). Make sure to bump the version of the image you are changing and update its reference in all images that extend from it. +This is also a good opportunity to verify that all the appropriate image versions were bumped. It is important to keep in mind which images extend from where ([Image Dependecies](https://github.com/DataBiosphere/terra-docker#image-dependencies)). Make sure to bump the version of the image you are changing in `config/conf.json` and update its reference in all images that extend from it. You can also run `amm ./updateVersions.sc ` to upgrade derived images. If you're updating `terra-jupyter-aou` image, please make a pull request similar to [this PR](https://github.com/DataBiosphere/leonardo/pull/1612) (less updating `terra_jupyter_aou_old`). diff --git a/config/conf.json b/config/conf.json index f71b554c..16340e4e 100644 --- a/config/conf.json +++ b/config/conf.json @@ -26,7 +26,7 @@ "tidyverse" ] }, - "version" : "2.2.5", + "version" : "2.2.6", "automated_flags" : { "generate_docs" : true, "include_in_ui" : true, @@ -47,7 +47,7 @@ "hail" ] }, - "version" : "1.1.12", + "version" : "1.1.13", "automated_flags" : { "generate_docs" : true, "include_in_ui" : true, @@ -68,7 +68,7 @@ "scikit-learn" ] }, - "version" : "1.1.5", + "version" : "1.1.6", "automated_flags" : { "generate_docs" : true, "include_in_ui" : true, @@ -84,9 +84,9 @@ "python" ], "packages" : { - + }, - "version" : "1.1.3", + "version" : "1.1.4", "automated_flags" : { "generate_docs" : true, "include_in_ui" : false, @@ -102,9 +102,9 @@ "r" ], "packages" : { - + }, - "version" : "2.2.5", + "version" : "2.2.6", "automated_flags" : { "include_in_ui" : false, "generate_docs" : true, @@ -122,9 +122,9 @@ "r" ], "packages" : { - + }, - "version" : "2.3.7", + "version" : "2.3.8", "automated_flags" : { "include_in_ui" : true, "generate_docs" : true, @@ -141,9 +141,9 @@ "r" ], "packages" : { - + }, - "version" : "2.2.13", + "version" : "2.2.14", "automated_flags" : { "include_in_ui" : false, "generate_docs" : false, @@ -159,7 +159,7 @@ "r" ], "packages" : { - + }, "version" : "0.1.1", "automated_flags" : { @@ -176,7 +176,7 @@ "tools" : [ ], "packages" : { - + }, "version" : "0.0.1", "automated_flags" : { diff --git a/terra-jupyter-aou/CHANGELOG.md b/terra-jupyter-aou/CHANGELOG.md index 8daaab99..ef10e31d 100644 --- a/terra-jupyter-aou/CHANGELOG.md +++ b/terra-jupyter-aou/CHANGELOG.md @@ -1,3 +1,11 @@ +## 2.2.14 - 2024-09-04 +- Update `terra-jupyter-gatk` to `2.3.8` + - Update `terra-jupyter-python` to `1.1.6` + - Update `terra-jupyter-base` to `1.1.4` + - Update terra-notebook-utils to `0.14.0` + +Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.2.14` + ## 2.2.13 - 2024-08-12 - Update `hail` to `0.2.130.post1` - See https://github.com/hail-is/hail/releases/tag/0.2.130.post1 for details diff --git a/terra-jupyter-aou/Dockerfile b/terra-jupyter-aou/Dockerfile index 64a79974..fa99ff16 100644 --- a/terra-jupyter-aou/Dockerfile +++ b/terra-jupyter-aou/Dockerfile @@ -1,4 +1,4 @@ -FROM us.gcr.io/broad-dsp-gcr-public/terra-jupyter-gatk:2.3.7 +FROM us.gcr.io/broad-dsp-gcr-public/terra-jupyter-gatk:2.3.8 USER root diff --git a/terra-jupyter-base/CHANGELOG.md b/terra-jupyter-base/CHANGELOG.md index 6a0e5572..66ab6082 100644 --- a/terra-jupyter-base/CHANGELOG.md +++ b/terra-jupyter-base/CHANGELOG.md @@ -1,3 +1,8 @@ +## 1.1.4 - 2024-09-04 +- Update terra-notebook-utils to `0.14.0` + +Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-base:1.1.4` + ## 1.1.3 - 2023-10-03 - Downgrade notebook to `6.5.4` and jupyterlab-server to `2.23.0` diff --git a/terra-jupyter-bioconductor/CHANGELOG.md b/terra-jupyter-bioconductor/CHANGELOG.md index dce91612..86d31fd2 100644 --- a/terra-jupyter-bioconductor/CHANGELOG.md +++ b/terra-jupyter-bioconductor/CHANGELOG.md @@ -1,3 +1,11 @@ +## 2.2.6 - 2024-09-04 + +- Update `terra-jupyter-r` to `2.2.6` + - Update `terra-jupyter-base` to `1.1.4` + - Update terra-notebook-utils to `0.14.0` + +Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-bioconductor:2.2.6` + ## 2.2.5 - 2024-05-15T15:36:28.576892790Z - Update `terra-jupyter-r` to `2.2.5` diff --git a/terra-jupyter-bioconductor/Dockerfile b/terra-jupyter-bioconductor/Dockerfile index 916163fd..43f74e12 100644 --- a/terra-jupyter-bioconductor/Dockerfile +++ b/terra-jupyter-bioconductor/Dockerfile @@ -1 +1 @@ -FROM us.gcr.io/broad-dsp-gcr-public/terra-jupyter-r:2.2.5 +FROM us.gcr.io/broad-dsp-gcr-public/terra-jupyter-r:2.2.6 diff --git a/terra-jupyter-gatk/CHANGELOG.md b/terra-jupyter-gatk/CHANGELOG.md index 0069b6df..e2b5371c 100644 --- a/terra-jupyter-gatk/CHANGELOG.md +++ b/terra-jupyter-gatk/CHANGELOG.md @@ -1,3 +1,10 @@ +## 2.3.8 - 2024-09-04 +- Update `terra-jupyter-python` to `1.1.6` + - Update `terra-jupyter-base` to `1.1.4` + - Update terra-notebook-utils to `0.14.0` + +Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-gatk:2.3.8` + ## 2.3.7 - 2024-05-15T15:36:28.602743427Z - Update `terra-jupyter-r` to `2.2.5` diff --git a/terra-jupyter-gatk/Dockerfile b/terra-jupyter-gatk/Dockerfile index 4e182a70..b0753430 100644 --- a/terra-jupyter-gatk/Dockerfile +++ b/terra-jupyter-gatk/Dockerfile @@ -1,4 +1,4 @@ -FROM us.gcr.io/broad-dsp-gcr-public/terra-jupyter-python:1.1.5 AS python +FROM us.gcr.io/broad-dsp-gcr-public/terra-jupyter-python:1.1.6 AS python FROM us.gcr.io/broad-dsp-gcr-public/terra-jupyter-r:2.2.5 diff --git a/terra-jupyter-hail/CHANGELOG.md b/terra-jupyter-hail/CHANGELOG.md index 61fd1b86..328fa845 100644 --- a/terra-jupyter-hail/CHANGELOG.md +++ b/terra-jupyter-hail/CHANGELOG.md @@ -1,3 +1,10 @@ +## 1.1.13 - 2024-09-04 +- Update `terra-jupyter-python` to `1.1.6` + - Update `terra-jupyter-base` to `1.1.4` + - Update terra-notebook-utils to `0.14.0` + +Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-hail:1.1.13` + ## 1.1.12 - 2024-08-12 - Update `hail` to `0.2.130.post1` - See https://github.com/hail-is/hail/releases/tag/0.2.130.post1 for details diff --git a/terra-jupyter-hail/Dockerfile b/terra-jupyter-hail/Dockerfile index 4abe843d..8a802ef3 100644 --- a/terra-jupyter-hail/Dockerfile +++ b/terra-jupyter-hail/Dockerfile @@ -1,4 +1,4 @@ -FROM us.gcr.io/broad-dsp-gcr-public/terra-jupyter-python:1.1.5 +FROM us.gcr.io/broad-dsp-gcr-public/terra-jupyter-python:1.1.6 USER root diff --git a/terra-jupyter-python/CHANGELOG.md b/terra-jupyter-python/CHANGELOG.md index 1ee01976..0f4c2464 100644 --- a/terra-jupyter-python/CHANGELOG.md +++ b/terra-jupyter-python/CHANGELOG.md @@ -1,3 +1,9 @@ +## 1.1.6 - 2024-09-04 +- Update `terra-jupyter-base` to `1.1.4` + - Update terra-notebook-utils to `0.14.0` + +Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-python:1.1.6` + ## 1.1.5 - 2023-11-28 - Pin bleach to `6.0.0` diff --git a/terra-jupyter-python/Dockerfile b/terra-jupyter-python/Dockerfile index b98b5220..a80f6c50 100644 --- a/terra-jupyter-python/Dockerfile +++ b/terra-jupyter-python/Dockerfile @@ -1,4 +1,4 @@ -FROM us.gcr.io/broad-dsp-gcr-public/terra-jupyter-base:1.1.3 +FROM us.gcr.io/broad-dsp-gcr-public/terra-jupyter-base:1.1.4 USER root # This makes it so pip runs as root, not the user. diff --git a/terra-jupyter-r/CHANGELOG.md b/terra-jupyter-r/CHANGELOG.md index 6a369f7a..6dff7527 100644 --- a/terra-jupyter-r/CHANGELOG.md +++ b/terra-jupyter-r/CHANGELOG.md @@ -1,3 +1,9 @@ +## 2.2.6 - 2024-09-04 +- Update `terra-jupyter-base` to `1.1.4` + - Update terra-notebook-utils to `0.14.0` + +Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-r:2.2.6` + ## 2.2.5 - 2024-05-15T15:36:28.555690376Z - Update for R 4.4.0 and Bioc 3.19 diff --git a/terra-jupyter-r/Dockerfile b/terra-jupyter-r/Dockerfile index 985b154b..3d8599b3 100644 --- a/terra-jupyter-r/Dockerfile +++ b/terra-jupyter-r/Dockerfile @@ -1,4 +1,4 @@ -FROM us.gcr.io/broad-dsp-gcr-public/terra-jupyter-base:1.1.3 +FROM us.gcr.io/broad-dsp-gcr-public/terra-jupyter-base:1.1.4 USER root From 453d7681b19f377df46012a96ca6f737bf8791c7 Mon Sep 17 00:00:00 2001 From: LizBaldo Date: Thu, 5 Sep 2024 15:32:09 -0400 Subject: [PATCH 03/21] update terra jupyter python smoke test --- terra-jupyter-python/tests/smoke_test.ipynb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/terra-jupyter-python/tests/smoke_test.ipynb b/terra-jupyter-python/tests/smoke_test.ipynb index abaa3328..55511ae8 100644 --- a/terra-jupyter-python/tests/smoke_test.ipynb +++ b/terra-jupyter-python/tests/smoke_test.ipynb @@ -85,9 +85,9 @@ "metadata": {}, "outputs": [], "source": [ - "output = !pip3 show pendulum\n", + "output = !pip3 show emoji\n", "print(output) # Should show not yet installed.\n", - "assert(0 == output.count('Name: pendulum'))" + "assert(0 == output.count('Name: emoji'))" ] }, { @@ -96,7 +96,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install pendulum==2.1.2 --user --ignore-installed" + "%pip install emoji --user --ignore-installed" ] }, { @@ -105,9 +105,9 @@ "metadata": {}, "outputs": [], "source": [ - "output = !pip3 show pendulum\n", + "output = !pip3 show emoji\n", "print(output) # Should show that it is now installed!\n", - "assert(1 == output.count('Name: pendulum'))" + "assert(1 == output.count('Name: emoji'))" ] }, { From 7ba9c7e154649fff2d39aa2cb3cf6ba7652896a9 Mon Sep 17 00:00:00 2001 From: LizBaldo Date: Thu, 5 Sep 2024 17:36:27 -0400 Subject: [PATCH 04/21] upper bound to shapely versions compatible with big query --- terra-jupyter-python/requirements.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/terra-jupyter-python/requirements.txt b/terra-jupyter-python/requirements.txt index 6ee08d33..370977cc 100644 --- a/terra-jupyter-python/requirements.txt +++ b/terra-jupyter-python/requirements.txt @@ -44,3 +44,6 @@ scikit-learn-intelex xgboost #older version of nbconvert fail to convert notebooks to html nbconvert>=7.7.3 +# shapely > 2.X is not compatible with bigquery yet, see https://github.com/googleapis/python-aiplatform/issues/1852 +shapely<2.0.0 + From 7129cbd751cbea5a57f9b149e06916bd1f19f960 Mon Sep 17 00:00:00 2001 From: LizBaldo Date: Thu, 5 Sep 2024 19:12:55 -0400 Subject: [PATCH 05/21] more pinning --- terra-jupyter-python/requirements.txt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/terra-jupyter-python/requirements.txt b/terra-jupyter-python/requirements.txt index 370977cc..22b31f2c 100644 --- a/terra-jupyter-python/requirements.txt +++ b/terra-jupyter-python/requirements.txt @@ -44,6 +44,8 @@ scikit-learn-intelex xgboost #older version of nbconvert fail to convert notebooks to html nbconvert>=7.7.3 -# shapely > 2.X is not compatible with bigquery yet, see https://github.com/googleapis/python-aiplatform/issues/1852 -shapely<2.0.0 +# See: https://github.com/GoogleCloudPlatform/generative-ai/issues/849 +shapely==1.8.5.post1 +pygeos==0.12.0 +geopandas==0.10.2 From ad03f844122caf7cd3d3a78076e881b9940af28f Mon Sep 17 00:00:00 2001 From: LizBaldo Date: Thu, 5 Sep 2024 22:49:13 -0400 Subject: [PATCH 06/21] updating deprecasted pandas commands --- terra-jupyter-python/tests/smoke_test.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/terra-jupyter-python/tests/smoke_test.ipynb b/terra-jupyter-python/tests/smoke_test.ipynb index 55511ae8..71027ef4 100644 --- a/terra-jupyter-python/tests/smoke_test.ipynb +++ b/terra-jupyter-python/tests/smoke_test.ipynb @@ -248,14 +248,14 @@ "source": [ "import numpy as np\n", "import pandas as pd\n", - "from pandas_profiling import ProfileReport\n", + "from pandas_profiling import ydata_profiling\n", "\n", "df = pd.DataFrame(\n", " np.random.rand(100, 5),\n", " columns=['a', 'b', 'c', 'd', 'e']\n", ")\n", "\n", - "profile = ProfileReport(df, title='Pandas Profiling Report')\n", + "profile = ydata_profiling(df, title='Pandas Profiling Report')\n", "profile" ] }, From bca0857034bcf0b1a4804107de39713d0c92bdc7 Mon Sep 17 00:00:00 2001 From: LizBaldo Date: Fri, 6 Sep 2024 08:28:50 -0400 Subject: [PATCH 07/21] see if ydata profiling is already installed --- terra-jupyter-python/tests/smoke_test.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terra-jupyter-python/tests/smoke_test.ipynb b/terra-jupyter-python/tests/smoke_test.ipynb index 71027ef4..a3f33b3d 100644 --- a/terra-jupyter-python/tests/smoke_test.ipynb +++ b/terra-jupyter-python/tests/smoke_test.ipynb @@ -248,7 +248,7 @@ "source": [ "import numpy as np\n", "import pandas as pd\n", - "from pandas_profiling import ydata_profiling\n", + "import ydata_profiling\n", "\n", "df = pd.DataFrame(\n", " np.random.rand(100, 5),\n", From d79f6367a7978834e73c0cbb1d9813f613132d30 Mon Sep 17 00:00:00 2001 From: LizBaldo Date: Fri, 6 Sep 2024 11:05:47 -0400 Subject: [PATCH 08/21] correct use of data profiling --- terra-jupyter-python/tests/smoke_test.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/terra-jupyter-python/tests/smoke_test.ipynb b/terra-jupyter-python/tests/smoke_test.ipynb index a3f33b3d..862eeb55 100644 --- a/terra-jupyter-python/tests/smoke_test.ipynb +++ b/terra-jupyter-python/tests/smoke_test.ipynb @@ -248,14 +248,14 @@ "source": [ "import numpy as np\n", "import pandas as pd\n", - "import ydata_profiling\n", + "from ydata_profiling import ProfileReport\n", "\n", "df = pd.DataFrame(\n", " np.random.rand(100, 5),\n", " columns=['a', 'b', 'c', 'd', 'e']\n", ")\n", "\n", - "profile = ydata_profiling(df, title='Pandas Profiling Report')\n", + "profile = ProfileReport(df, title='Pandas Profiling Report')\n", "profile" ] }, From 8dd8ed99d2eb8eb4198c7b67c6549477a8e28cb7 Mon Sep 17 00:00:00 2001 From: LizBaldo Date: Fri, 6 Sep 2024 12:52:51 -0400 Subject: [PATCH 09/21] fix tera jupyter r version --- terra-jupyter-r/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terra-jupyter-r/Dockerfile b/terra-jupyter-r/Dockerfile index 3d8599b3..1f051905 100644 --- a/terra-jupyter-r/Dockerfile +++ b/terra-jupyter-r/Dockerfile @@ -5,7 +5,7 @@ USER root COPY scripts $JUPYTER_HOME/scripts # Add env vars to identify binary package installation -ENV TERRA_R_PLATFORM="terra-jupyter-r-2.2.5" +ENV TERRA_R_PLATFORM="terra-jupyter-r-2.2.6" ENV TERRA_R_PLATFORM_BINARY_VERSION=3.19 # Install protobuf 3.20.3. Note this version comes from base deep learning image. Use `conda list` to see what's installed From 0266baf0dcc0067d854d466bf81ef60f0256ebd7 Mon Sep 17 00:00:00 2001 From: LizBaldo Date: Fri, 6 Sep 2024 15:46:28 -0400 Subject: [PATCH 10/21] also need to update the base R image for GATK --- terra-jupyter-gatk/CHANGELOG.md | 1 + terra-jupyter-gatk/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/terra-jupyter-gatk/CHANGELOG.md b/terra-jupyter-gatk/CHANGELOG.md index e2b5371c..329ea343 100644 --- a/terra-jupyter-gatk/CHANGELOG.md +++ b/terra-jupyter-gatk/CHANGELOG.md @@ -1,5 +1,6 @@ ## 2.3.8 - 2024-09-04 - Update `terra-jupyter-python` to `1.1.6` +- Update `terra-jupyter-r` to `2.2.6` - Update `terra-jupyter-base` to `1.1.4` - Update terra-notebook-utils to `0.14.0` diff --git a/terra-jupyter-gatk/Dockerfile b/terra-jupyter-gatk/Dockerfile index b0753430..d12c9fa9 100644 --- a/terra-jupyter-gatk/Dockerfile +++ b/terra-jupyter-gatk/Dockerfile @@ -1,6 +1,6 @@ FROM us.gcr.io/broad-dsp-gcr-public/terra-jupyter-python:1.1.6 AS python -FROM us.gcr.io/broad-dsp-gcr-public/terra-jupyter-r:2.2.5 +FROM us.gcr.io/broad-dsp-gcr-public/terra-jupyter-r:2.2.6 USER root From a8f01121e7017547fdc9d2262ca0429068ecefaa Mon Sep 17 00:00:00 2001 From: LizBaldo Date: Fri, 6 Sep 2024 16:40:54 -0400 Subject: [PATCH 11/21] need to also pin traitlets to a version compatible with old jupyter notebook --- terra-jupyter-base/Dockerfile | 2 ++ terra-jupyter-python/requirements.txt | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/terra-jupyter-base/Dockerfile b/terra-jupyter-base/Dockerfile index 4689b13e..0efd17ba 100644 --- a/terra-jupyter-base/Dockerfile +++ b/terra-jupyter-base/Dockerfile @@ -109,6 +109,8 @@ RUN pip3 -V \ # DO NOT CHANGE OR UNPIN NOTEBOOK OR JUPYTERLAB-SERVER! When trying with 6.5.5 and 2.24.0 notebook commands could not find the notebook config file && pip3 install notebook==6.5.4 \ && pip3 install jupyterlab-server==2.23.0 \ + # See: https://stackoverflow.com/questions/77549493/modulenotfounderror-no-module-named-jupyter-server-contents/77549554#77549554 + && pip3 install traitlets==5.9.0 \ && pip3 install requests \ && pip3 install firecloud \ && pip3 install terra-notebook-utils==0.14.0 \ diff --git a/terra-jupyter-python/requirements.txt b/terra-jupyter-python/requirements.txt index 22b31f2c..3d6a14ce 100644 --- a/terra-jupyter-python/requirements.txt +++ b/terra-jupyter-python/requirements.txt @@ -48,4 +48,5 @@ nbconvert>=7.7.3 shapely==1.8.5.post1 pygeos==0.12.0 geopandas==0.10.2 - +# See: https://stackoverflow.com/questions/77549493/modulenotfounderror-no-module-named-jupyter-server-contents/77549554#77549554 +traitlets==5.9.0 From 8e5c192dc9038e745393adb5d41d1fb46791c917 Mon Sep 17 00:00:00 2001 From: LizBaldo Date: Mon, 9 Sep 2024 16:44:19 -0400 Subject: [PATCH 12/21] pining jupyter server as well --- terra-jupyter-base/Dockerfile | 1 + terra-jupyter-python/requirements.txt | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/terra-jupyter-base/Dockerfile b/terra-jupyter-base/Dockerfile index 0efd17ba..d3f72be4 100644 --- a/terra-jupyter-base/Dockerfile +++ b/terra-jupyter-base/Dockerfile @@ -108,6 +108,7 @@ RUN pip3 -V \ # for jupyter_delocalize.py and jupyter_notebook_config.py # DO NOT CHANGE OR UNPIN NOTEBOOK OR JUPYTERLAB-SERVER! When trying with 6.5.5 and 2.24.0 notebook commands could not find the notebook config file && pip3 install notebook==6.5.4 \ + && pip3 install jupyter-server==1.24.0 \ && pip3 install jupyterlab-server==2.23.0 \ # See: https://stackoverflow.com/questions/77549493/modulenotfounderror-no-module-named-jupyter-server-contents/77549554#77549554 && pip3 install traitlets==5.9.0 \ diff --git a/terra-jupyter-python/requirements.txt b/terra-jupyter-python/requirements.txt index 3d6a14ce..414cd082 100644 --- a/terra-jupyter-python/requirements.txt +++ b/terra-jupyter-python/requirements.txt @@ -48,5 +48,3 @@ nbconvert>=7.7.3 shapely==1.8.5.post1 pygeos==0.12.0 geopandas==0.10.2 -# See: https://stackoverflow.com/questions/77549493/modulenotfounderror-no-module-named-jupyter-server-contents/77549554#77549554 -traitlets==5.9.0 From 3e4649a85e11498004e7e242e94f03c389e469a7 Mon Sep 17 00:00:00 2001 From: LizBaldo Date: Tue, 10 Sep 2024 20:20:04 -0400 Subject: [PATCH 13/21] try to install XML as a prerequisite --- terra-jupyter-r/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/terra-jupyter-r/Dockerfile b/terra-jupyter-r/Dockerfile index 1f051905..ec5b20dd 100644 --- a/terra-jupyter-r/Dockerfile +++ b/terra-jupyter-r/Dockerfile @@ -168,6 +168,7 @@ RUN R -e 'install.packages("BiocManager")' \ # Install Bioconductor packages found at: # https://raw.githubusercontent.com/anvilproject/anvil-docker/master/anvil-rstudio-bioconductor/install.R RUN R -e 'BiocManager::install(c( \ + "XML", \ "AnVIL", \ "SingleCellExperiment", \ "GenomicFeatures", \ @@ -187,4 +188,4 @@ ENV PIP_USER=true RUN R -e 'IRkernel::installspec(user=FALSE)' \ && chown -R $USER:users /usr/local/lib/R/site-library /home/jupyter -USER $USER +USER $USER \ No newline at end of file From 74a4594266c819e94bd3db17dc39d0b03b6cfb92 Mon Sep 17 00:00:00 2001 From: LizBaldo Date: Tue, 10 Sep 2024 21:49:51 -0400 Subject: [PATCH 14/21] avoid bad version of setuptools --- terra-jupyter-python/requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/terra-jupyter-python/requirements.txt b/terra-jupyter-python/requirements.txt index 414cd082..8a24cc44 100644 --- a/terra-jupyter-python/requirements.txt +++ b/terra-jupyter-python/requirements.txt @@ -48,3 +48,5 @@ nbconvert>=7.7.3 shapely==1.8.5.post1 pygeos==0.12.0 geopandas==0.10.2 +# See: https://github.com/pypa/setuptools/issues/4483 +setuptools<71.0.0 From 1d61fdde8bd8bad5cf4fef8984a85ead490b48e9 Mon Sep 17 00:00:00 2001 From: LizBaldo Date: Wed, 11 Sep 2024 14:05:59 -0400 Subject: [PATCH 15/21] do not upgrade python packages for GATK image --- terra-jupyter-gatk/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terra-jupyter-gatk/Dockerfile b/terra-jupyter-gatk/Dockerfile index d12c9fa9..742a545c 100644 --- a/terra-jupyter-gatk/Dockerfile +++ b/terra-jupyter-gatk/Dockerfile @@ -27,7 +27,7 @@ ENV PIP_USER=false # uninstalling packages: https://github.com/conda/conda/issues/10357 COPY --from=python /etc/terra-docker/requirements.txt /etc/terra-docker/base_requirements.txt -RUN pip install --upgrade -r /etc/terra-docker/base_requirements.txt +RUN pip install -r /etc/terra-docker/base_requirements.txt # Install GATK ENV GATK_VERSION 4.3.0.0 From 011afc4b29d734b8135f1a1de20d8fc9b010c7b1 Mon Sep 17 00:00:00 2001 From: LizBaldo Date: Wed, 11 Sep 2024 15:07:54 -0400 Subject: [PATCH 16/21] explicitly export XML config as env --- terra-jupyter-r/Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/terra-jupyter-r/Dockerfile b/terra-jupyter-r/Dockerfile index ec5b20dd..bf59e586 100644 --- a/terra-jupyter-r/Dockerfile +++ b/terra-jupyter-r/Dockerfile @@ -117,8 +117,10 @@ RUN apt-get update \ # Add libsbml CFLAGS ENV LIBSBML_CFLAGS="-I/usr/include" ENV LIBSBML_LIBS="-lsbml" +ENV XML_CONFIG="/opt/conda/bin/xml2-config" RUN echo 'export LIBSBML_CFLAGS="-I/usr/include"' >> /etc/profile \ - && echo 'export LIBSBML_LIBS="-lsbml"' >> /etc/profile + && echo 'export LIBSBML_LIBS="-lsbml"' >> /etc/profile \ + && echo 'export XML_CONFIG="/opt/conda/bin/xml2-config"' >> /etc/profile ## set pip3 to run as root, not as jupyter user ENV PIP_USER=false @@ -168,7 +170,6 @@ RUN R -e 'install.packages("BiocManager")' \ # Install Bioconductor packages found at: # https://raw.githubusercontent.com/anvilproject/anvil-docker/master/anvil-rstudio-bioconductor/install.R RUN R -e 'BiocManager::install(c( \ - "XML", \ "AnVIL", \ "SingleCellExperiment", \ "GenomicFeatures", \ From 02b644c5e4c7feaf99bf7e224b806b312ed15924 Mon Sep 17 00:00:00 2001 From: LizBaldo Date: Wed, 11 Sep 2024 21:29:15 -0400 Subject: [PATCH 17/21] next issue seems to be with rtracklayer --- terra-jupyter-r/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/terra-jupyter-r/Dockerfile b/terra-jupyter-r/Dockerfile index bf59e586..1471ff33 100644 --- a/terra-jupyter-r/Dockerfile +++ b/terra-jupyter-r/Dockerfile @@ -158,6 +158,8 @@ RUN R -e 'install.packages("BiocManager")' \ # GCP essentials "bigrquery", \ "googleCloudStorageR", \ + # GenomicFeatures dependency + "rtracklayer", \ # User oriented packages "reticulate", \ "remotes", \ From 1a547f89886e31d698d2006a96e20b5300535b0c Mon Sep 17 00:00:00 2001 From: LizBaldo Date: Thu, 12 Sep 2024 10:19:20 -0400 Subject: [PATCH 18/21] do not use libxml2 installed by conda --- terra-jupyter-r/Dockerfile | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/terra-jupyter-r/Dockerfile b/terra-jupyter-r/Dockerfile index 1471ff33..a79d41bb 100644 --- a/terra-jupyter-r/Dockerfile +++ b/terra-jupyter-r/Dockerfile @@ -117,10 +117,10 @@ RUN apt-get update \ # Add libsbml CFLAGS ENV LIBSBML_CFLAGS="-I/usr/include" ENV LIBSBML_LIBS="-lsbml" -ENV XML_CONFIG="/opt/conda/bin/xml2-config" +ENV XML_CONFIG="/usr/bin/xml2-config" RUN echo 'export LIBSBML_CFLAGS="-I/usr/include"' >> /etc/profile \ && echo 'export LIBSBML_LIBS="-lsbml"' >> /etc/profile \ - && echo 'export XML_CONFIG="/opt/conda/bin/xml2-config"' >> /etc/profile + && echo 'export XML_CONFIG="/usr/bin/xml2-config"' >> /etc/profile ## set pip3 to run as root, not as jupyter user ENV PIP_USER=false @@ -158,8 +158,6 @@ RUN R -e 'install.packages("BiocManager")' \ # GCP essentials "bigrquery", \ "googleCloudStorageR", \ - # GenomicFeatures dependency - "rtracklayer", \ # User oriented packages "reticulate", \ "remotes", \ From 5b8a5f72c6bd342fbceedb589c43697f3fbb5c5a Mon Sep 17 00:00:00 2001 From: LizBaldo Date: Thu, 12 Sep 2024 10:26:15 -0400 Subject: [PATCH 19/21] update upload artifact github action --- .github/workflows/test-terra-jupyter-aou.yml | 2 +- .github/workflows/test-terra-jupyter-base.yml | 2 +- .github/workflows/test-terra-jupyter-bioconductor.yml | 2 +- .github/workflows/test-terra-jupyter-gatk.yml | 2 +- .github/workflows/test-terra-jupyter-hail.yml | 2 +- .github/workflows/test-terra-jupyter-python.yml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test-terra-jupyter-aou.yml b/.github/workflows/test-terra-jupyter-aou.yml index 836644ad..f64e24f1 100644 --- a/.github/workflows/test-terra-jupyter-aou.yml +++ b/.github/workflows/test-terra-jupyter-aou.yml @@ -87,7 +87,7 @@ jobs: /bin/bash -c 'for nb in {terra-jupyter-python/tests,terra-jupyter-aou/tests}/*ipynb ; do jupyter nbconvert --to html_embed --ExecutePreprocessor.allow_errors=True --execute "${nb}" ; done' - name: Upload workflow artifacts - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: notebook-execution-results path: | diff --git a/.github/workflows/test-terra-jupyter-base.yml b/.github/workflows/test-terra-jupyter-base.yml index 710c8cc6..295afc15 100644 --- a/.github/workflows/test-terra-jupyter-base.yml +++ b/.github/workflows/test-terra-jupyter-base.yml @@ -69,7 +69,7 @@ jobs: ./build_smoke_test_image.sh terra-jupyter-base - name: Upload workflow artifacts - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: notebook-execution-results path: terra-jupyter-base/tests/*.html diff --git a/.github/workflows/test-terra-jupyter-bioconductor.yml b/.github/workflows/test-terra-jupyter-bioconductor.yml index 64bcadd5..669e6df1 100644 --- a/.github/workflows/test-terra-jupyter-bioconductor.yml +++ b/.github/workflows/test-terra-jupyter-bioconductor.yml @@ -87,7 +87,7 @@ jobs: /bin/sh -c 'for nb in *ipynb ; do jupyter nbconvert --to html_embed --ExecutePreprocessor.timeout=300 --execute "${nb}" ; done' - name: Upload workflow artifacts - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: notebook-execution-results path: terra-jupyter-bioconductor/tests/*.html diff --git a/.github/workflows/test-terra-jupyter-gatk.yml b/.github/workflows/test-terra-jupyter-gatk.yml index cdbcf044..ebb10bf8 100644 --- a/.github/workflows/test-terra-jupyter-gatk.yml +++ b/.github/workflows/test-terra-jupyter-gatk.yml @@ -87,7 +87,7 @@ jobs: /bin/bash -c 'for nb in {terra-jupyter-python/tests,terra-jupyter-gatk/tests}/*ipynb ; do jupyter nbconvert --to html_embed --ExecutePreprocessor.allow_errors=True --execute "${nb}" ; done' - name: Upload workflow artifacts - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: notebook-execution-results path: | diff --git a/.github/workflows/test-terra-jupyter-hail.yml b/.github/workflows/test-terra-jupyter-hail.yml index 968179bf..18c172ff 100644 --- a/.github/workflows/test-terra-jupyter-hail.yml +++ b/.github/workflows/test-terra-jupyter-hail.yml @@ -87,7 +87,7 @@ jobs: /bin/bash -c 'for nb in {terra-jupyter-python/tests,terra-jupyter-hail/tests}/*ipynb ; do jupyter nbconvert --to html_embed --ExecutePreprocessor.allow_errors=True --execute "${nb}" ; done' - name: Upload workflow artifacts - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: notebook-execution-results path: | diff --git a/.github/workflows/test-terra-jupyter-python.yml b/.github/workflows/test-terra-jupyter-python.yml index 3d6f3116..63fd0f14 100644 --- a/.github/workflows/test-terra-jupyter-python.yml +++ b/.github/workflows/test-terra-jupyter-python.yml @@ -92,7 +92,7 @@ jobs: /bin/sh -c 'for nb in *ipynb ; do jupyter nbconvert --to html_embed --ExecutePreprocessor.allow_errors=True --execute "${nb}" ; done' - name: Upload workflow artifacts - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: notebook-execution-results path: terra-jupyter-python/tests/*.html From 3320c3ae283ec84fe91f03d5c95600dc6f62d36b Mon Sep 17 00:00:00 2001 From: LizBaldo Date: Fri, 13 Sep 2024 10:29:05 -0400 Subject: [PATCH 20/21] try to free up disk space before building container --- .github/workflows/publish.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index cb569f9f..a407985c 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -35,6 +35,9 @@ jobs: with: python-version: '3.10' + - name: Free up some disk space + run: sudo rm -rf /usr/share/dotnet + - id: auth uses: google-github-actions/auth@v1 with: From 06ee889a0edce67df99c7062222706730c40a86d Mon Sep 17 00:00:00 2001 From: LizBaldo Date: Mon, 16 Sep 2024 09:30:49 -0400 Subject: [PATCH 21/21] update qiime 2 package for AOU --- terra-jupyter-aou/Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/terra-jupyter-aou/Dockerfile b/terra-jupyter-aou/Dockerfile index fa99ff16..aa5f471d 100644 --- a/terra-jupyter-aou/Dockerfile +++ b/terra-jupyter-aou/Dockerfile @@ -195,10 +195,10 @@ RUN apt-get update && \ && rm -rf /var/lib/apt/lists/* RUN pip install --upgrade pip && pip install --verbose h5py -# RW-8725 INSTALL QIMEE2 -RUN curl -fsSL -o qiime2-2022.8.yml https://data.qiime2.org/distro/core/qiime2-2022.8-py38-linux-conda.yml && \ - conda env create -n qiime2-2022.8 --file qiime2-2022.8.yml && \ - rm qiime2-2022.8.yml +# QIIME 2 2024.5 Amplicon Distribution +RUN curl -fsSL -o qiime2-2024.5.yml https://data.qiime2.org/distro/amplicon/qiime2-amplicon-2024.5-py39-linux-conda.yml && \ + conda env create -n qiime2-2024.5 --file qiime2-2024.5.yml && \ + rm qiime2-2024.5.yml # Install jellyfish RUN mkdir -p /tmp/jellyfish && \