diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index cb63466d..cb569f9f 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -11,7 +11,6 @@ on: - terra-jupyter-base - terra-jupyter-bioconductor - terra-jupyter-gatk - - terra-jupyter-gatk-ovtf - terra-jupyter-hail - terra-jupyter-python - terra-jupyter-r @@ -34,7 +33,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 with: - python-version: 3.7 + python-version: '3.10' - id: auth uses: google-github-actions/auth@v1 diff --git a/.github/workflows/test-terra-jupyter-aou.yml b/.github/workflows/test-terra-jupyter-aou.yml index 7c3fdc4c..d3412f11 100644 --- a/.github/workflows/test-terra-jupyter-aou.yml +++ b/.github/workflows/test-terra-jupyter-aou.yml @@ -47,7 +47,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 with: - python-version: 3.7 + python-version: '3.10' - name: Free up some disk space run: sudo rm -rf /usr/share/dotnet diff --git a/.github/workflows/test-terra-jupyter-base.yml b/.github/workflows/test-terra-jupyter-base.yml index 7d46a589..2d61c72a 100644 --- a/.github/workflows/test-terra-jupyter-base.yml +++ b/.github/workflows/test-terra-jupyter-base.yml @@ -47,7 +47,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 with: - python-version: 3.7 + python-version: '3.10' - name: Free up some disk space run: sudo rm -rf /usr/share/dotnet diff --git a/.github/workflows/test-terra-jupyter-bioconductor.yml b/.github/workflows/test-terra-jupyter-bioconductor.yml index 7a793e34..e90e61e1 100644 --- a/.github/workflows/test-terra-jupyter-bioconductor.yml +++ b/.github/workflows/test-terra-jupyter-bioconductor.yml @@ -47,7 +47,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 with: - python-version: 3.7 + python-version: '3.10' - name: Free up some disk space run: sudo rm -rf /usr/share/dotnet diff --git a/.github/workflows/test-terra-jupyter-gatk-ovtf.yml b/.github/workflows/test-terra-jupyter-gatk-ovtf.yml deleted file mode 100644 index b8666e69..00000000 --- a/.github/workflows/test-terra-jupyter-gatk-ovtf.yml +++ /dev/null @@ -1,126 +0,0 @@ -name: Test terra-jupyter-gatk-ovtf -# Perform smoke tests on the terra-jupyter-gatk-ovtf Docker image to have some amount of confidence that -# Python package versions are compatible. -# -# To configure the minimal auth needed for these tests to be able to read public data from Google Cloud Platform: -# Step 1: Create a service account per these instructions: -# https://github.com/google-github-actions/setup-gcloud/blob/master/setup-gcloud/README.md -# Step 2: Give the service account the following permissions within the project: BigQuery User -# Step 3: Store its key and project id as GitHub repository secrets TD_GCP_SA_KEY and GCP_PROJECT_ID. -# https://docs.github.com/en/free-pro-team@latest/actions/reference/encrypted-secrets#creating-encrypted-secrets-for-a-repository - -on: - pull_request: - branches: [ master ] - paths: - - 'terra-jupyter-gatk-ovtf/**' - - '.github/workflows/test-terra-jupyter-gatk-ovtf.yml' - - push: - # Note: GitHub secrets are not passed to pull requests from forks. For community contributions from - # regular contributors, its a good idea for the contributor to configure the GitHub actions to run correctly - # in their fork as described above. - # - # For occasional contributors, the dev team will merge the PR fork branch to a branch in upstream named - # test-community-contribution- to run all the GitHub Action smoke tests. - branches: [ 'test-community-contribution*' ] - paths: - - 'terra-jupyter-gatk-ovtf/**' - - '.github/workflows/test-terra-jupyter-gatk-ovtf.yml' - - workflow_dispatch: - # Allows manually triggering of workflow on a selected branch via the GitHub Actions tab. - # GitHub blog demo: https://github.blog/changelog/2020-07-06-github-actions-manual-triggers-with-workflow_dispatch/. - -env: - GOOGLE_PROJECT: ${{ secrets.GCP_PROJECT_ID }} - -jobs: - - test_docker_image: - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: 3.7 - - - name: Free up some disk space - run: sudo rm -rf /usr/share/dotnet - - - id: auth - uses: google-github-actions/auth@v1 - with: - credentials_json: ${{ secrets.TD_GCP_SA_KEY }} - create_credentials_file: true - - - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@v0.3.0 - with: - project_id: ${{ secrets.GCP_PROJECT_ID }} - - - name: Build Docker image and base images too, if needed - run: | - gcloud auth configure-docker - ./build_smoke_test_image.sh terra-jupyter-gatk-ovtf - - - name: Run Python code specific to notebooks with nbconvert - # Run all notebooks from start to finish, regardles of error, so that we can capture the - # result as a workflow artifact. - # See also https://github.com/marketplace/actions/run-notebook if a more complicated - # workflow for notebooks is needed in the future. - run: | - chmod a+w -R $GITHUB_WORKSPACE - chmod a+r "${{ steps.auth.outputs.credentials_file_path }}" - docker run \ - --env GOOGLE_PROJECT \ - --volume "${{ steps.auth.outputs.credentials_file_path }}":/tmp/credentials.json:ro \ - --env GOOGLE_APPLICATION_CREDENTIALS="/tmp/credentials.json" \ - --volume $GITHUB_WORKSPACE:/tests \ - --workdir=/tests \ - --entrypoint="" \ - terra-jupyter-gatk-ovtf:smoke-test \ - /bin/bash -c 'for nb in {terra-jupyter-python/tests,terra-jupyter-gatk-ovtf/tests}/*ipynb ; do jupyter nbconvert --to html --ExecutePreprocessor.allow_errors=True --execute "${nb}" ; done' - - - name: Upload workflow artifacts - uses: actions/upload-artifact@v2 - with: - name: notebook-execution-results - path: | - terra-jupyter-python/tests/*.html - terra-jupyter-gatk-ovtf/tests/*.html - retention-days: 30 - - - name: Test Python code with pytest - run: | - chmod a+r "${{ steps.auth.outputs.credentials_file_path }}" - docker run \ - --env GOOGLE_PROJECT \ - --volume "${{ steps.auth.outputs.credentials_file_path }}":/tmp/credentials.json:ro \ - --env GOOGLE_APPLICATION_CREDENTIALS="/tmp/credentials.json" \ - --volume $GITHUB_WORKSPACE:/tests \ - --workdir=/tests \ - --entrypoint="" \ - terra-jupyter-gatk-ovtf:smoke-test \ - /bin/bash -c 'pip3 install pytest ; pytest terra-jupyter-python/tests/ terra-jupyter-gatk-ovtf/tests/' - - - name: Test Python code specific to notebooks with nbconvert - # Simply 'Cell -> Run All` these notebooks and expect no errors in the case of a successful run of the test suite. - # If the tests throw any exceptions, execution of the notebooks will halt at that point. Look at the workflow - # artifacts to understand if there are more failures than just the one that caused this task to halt. - run: | - chmod a+r "${{ steps.auth.outputs.credentials_file_path }}" - docker run \ - --env GOOGLE_PROJECT \ - --volume "${{ steps.auth.outputs.credentials_file_path }}":/tmp/credentials.json:ro \ - --env GOOGLE_APPLICATION_CREDENTIALS="/tmp/credentials.json" \ - --volume $GITHUB_WORKSPACE:/tests \ - --workdir=/tests \ - --entrypoint="" \ - terra-jupyter-gatk-ovtf:smoke-test \ - /bin/bash -c 'for nb in {terra-jupyter-python/tests,terra-jupyter-gatk-ovtf/tests}/*ipynb ; do jupyter nbconvert --to html --execute "${nb}" ; done' - diff --git a/.github/workflows/test-terra-jupyter-gatk.yml b/.github/workflows/test-terra-jupyter-gatk.yml index be81b133..d537b728 100644 --- a/.github/workflows/test-terra-jupyter-gatk.yml +++ b/.github/workflows/test-terra-jupyter-gatk.yml @@ -47,7 +47,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 with: - python-version: 3.7 + python-version: '3.10' - name: Free up some disk space run: sudo rm -rf /usr/share/dotnet diff --git a/.github/workflows/test-terra-jupyter-hail.yml b/.github/workflows/test-terra-jupyter-hail.yml index 61cafe61..cc6c777d 100644 --- a/.github/workflows/test-terra-jupyter-hail.yml +++ b/.github/workflows/test-terra-jupyter-hail.yml @@ -47,7 +47,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 with: - python-version: 3.7 + python-version: '3.10' - name: Free up some disk space run: sudo rm -rf /usr/share/dotnet diff --git a/.github/workflows/test-terra-jupyter-python.yml b/.github/workflows/test-terra-jupyter-python.yml index b45dd5c6..1c903ca7 100644 --- a/.github/workflows/test-terra-jupyter-python.yml +++ b/.github/workflows/test-terra-jupyter-python.yml @@ -49,7 +49,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 with: - python-version: 3.7 + python-version: '3.10' - name: Free up some disk space run: sudo rm -rf /usr/share/dotnet @@ -112,10 +112,8 @@ jobs: --workdir=/tests \ --entrypoint="" \ terra-jupyter-python:smoke-test \ - /bin/sh -c "pip3 install --no-deps pytest iniconfig pluggy py; pytest" - # Use --no-deps to avoid an unneccessary upgrade of pyparsing which then causes tensorflow tests to fail. - # TODO: whenever we no longer need to pin pyparsing, simplify the above command to 'pip3 install pytest ; pytest' - + /bin/sh -c "pip3 install pytest ; pytest" + - name: Test Python code specific to notebooks with nbconvert # Simply 'Cell -> Run All` these notebooks and expect no errors in the case of a successful run of the test suite. # If the tests throw any exceptions, execution of the notebooks will halt at that point. Look at the workflow diff --git a/.gitignore b/.gitignore index 4f7c29c6..c7b1ac97 100644 --- a/.gitignore +++ b/.gitignore @@ -15,4 +15,6 @@ package-lock.json .bash_history .conda/ .python_history -.keras/ \ No newline at end of file +.keras/ +.ammonite/ +.metals/ \ No newline at end of file diff --git a/README.md b/README.md index 982b49c1..817878ef 100644 --- a/README.md +++ b/README.md @@ -80,13 +80,19 @@ Detailed documentation on how to integrate the terra-docker image with Leonardo - Ensure that the `terra-docker-versions-candidate.json` file (which is what the UI sources the dropdown from) in the `terra-docjker-image-documentation-[env]` bucket correclty references your new docker image - [Update the terra-docker version candidate json](https://broadworkbench.atlassian.net/wiki/spaces/IA/pages/2519564289/Integrating+new+Terra+docker+images+with+Leonardo#6.-Update-terra-docker-versions-candidate.json) +## Testing your image manually +Build the image: +run `docker build [your_dir] -t [name]`. -If you wish to build locally, run `docker build [your_dir] -t [name]`. +`docker build terra-jupyter-base -t terra-jupyter-base` -It is not advised to run build.sh locally, as this will push to the remote docker repo and delete the image locally upon completion. +If you're on an M1 and building an image from a locally built image, replace the current FROM command: + +`FROM --platform=linux/amd64 terra-jupyter-base` + +It is not advised to run build.sh locally, as this will push to the remote docker repo and delete the image locally upon completion. -## Testing your image manually All images can be run locally. For example: ``` docker run --rm -it -p 8000:8000 us.gcr.io/broad-dsp-gcr-public/terra-jupyter-base:0.0.7 @@ -103,7 +109,7 @@ Running locally is conventient for quick development and exploring the image. Ho - there are no environment variables like `GOOGLE_PROJECT`, `WORKSPACE_NAME`, `WORKSPACE_BUCKET`, etc when running locally - there is no workspace-syncing when run locally -To launch an image through Terra, navigate to https://app.terra.bio, select a workspace, enter your image in the "Custom Image" field, and click Create. +To launch an image through Terra, navigate to https://app.terra.bio or your BEE's UI, select a workspace, enter your new image in the "Custom Image" field, and click Create. ## Automation Tests [Here](https://github.com/DataBiosphere/leonardo/tree/develop/automation/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/notebooks) are automation tests for various docker image, please update the image hash for relevant tests. You can run the job build-terra-docker to automatically create a PR with your branch if you manually specify versions. diff --git a/build_smoke_test_image.sh b/build_smoke_test_image.sh index d8e98ebe..aacf44f9 100755 --- a/build_smoke_test_image.sh +++ b/build_smoke_test_image.sh @@ -1,7 +1,7 @@ #!/bin/bash # Perform a local build of an image for smoke-testing purposes. Also build its tree of base images, when missing. # -# Example: ./build_smoke_test_image terra-jupyter-python +# Example: ./build_smoke_test_image.sh terra-jupyter-python # # Current working directory must be the directory in which this script resides (one level above all the Dockerfiles). # It walks down each relevant image directory, making changes to the FROM statements, diff --git a/config/community_images.json b/config/community_images.json index 5863508a..4d36ef18 100644 --- a/config/community_images.json +++ b/config/community_images.json @@ -27,14 +27,4 @@ "image": "us.gcr.io/broad-dsp-gcr-public/anvil-rstudio-bioconductor:3.16.1", "requiresSpark": false, "isRStudio": true -}, -{ - "id": "OpenVINO integration with Tensorflow", - "label": "OpenVINO integration with Tensorflow (openvino-tensorflow 1.1.0, Python 3.7.12, GATK 4.2.4.1)", - "version": "0.2.0", - "updated": "2022-01-31", - "packages": "https://storage.googleapis.com/terra-docker-image-documentation/terra-jupyter-gatk-ovtf-0.1.7-versions.json", - "image": "us.gcr.io/broad-dsp-gcr-public/terra-jupyter-gatk-ovtf:0.1.7", - "requiresSpark": false, - "isCommunity": true }] diff --git a/config/conf.json b/config/conf.json index 9b1371d7..e3e0b1f8 100644 --- a/config/conf.json +++ b/config/conf.json @@ -26,7 +26,7 @@ "tidyverse" ] }, - "version" : "2.1.11", + "version" : "2.2.0", "automated_flags" : { "generate_docs" : true, "include_in_ui" : true, @@ -47,7 +47,7 @@ "hail" ] }, - "version" : "1.0.27", + "version" : "1.1.0", "automated_flags" : { "generate_docs" : true, "include_in_ui" : true, @@ -68,7 +68,7 @@ "scikit-learn" ] }, - "version" : "1.0.15", + "version" : "1.1.0", "automated_flags" : { "generate_docs" : true, "include_in_ui" : true, @@ -86,7 +86,7 @@ "packages" : { }, - "version" : "1.0.14", + "version" : "1.1.0", "automated_flags" : { "generate_docs" : true, "include_in_ui" : false, @@ -104,7 +104,7 @@ "packages" : { }, - "version" : "2.1.10", + "version" : "2.2.0", "automated_flags" : { "include_in_ui" : false, "generate_docs" : true, @@ -124,7 +124,7 @@ "packages" : { }, - "version" : "2.2.14", + "version" : "2.3.0", "automated_flags" : { "include_in_ui" : true, "generate_docs" : true, @@ -143,7 +143,7 @@ "packages" : { }, - "version" : "2.1.22", + "version" : "2.2.0", "automated_flags" : { "include_in_ui" : false, "generate_docs" : false, @@ -170,26 +170,6 @@ "include_in_custom_gce" : false } }, - { - "name" : "terra-jupyter-gatk-ovtf", - "base_label" : "OpenVINO", - "tools" : [ - "gatk", - "python", - "r" - ], - "packages" : { - - }, - "version" : "0.2.10", - "automated_flags" : { - "include_in_ui" : false, - "generate_docs" : true, - "build" : true, - "include_in_custom_dataproc" : false, - "include_in_custom_gce" : false - } - }, { "name" : "wondershaper", "base_label" : "wondershaper", diff --git a/config/legacy_static_images.json b/config/legacy_static_images.json index 5cd1a167..c0bb6c29 100644 --- a/config/legacy_static_images.json +++ b/config/legacy_static_images.json @@ -1,11 +1,20 @@ [ { - "updated": "2023-02-09", - "image": "us.gcr.io/broad-dsp-gcr-public/terra-jupyter-bioconductor:2.1.9", + "updated": "2023-06-23", + "image": "us.gcr.io/broad-dsp-gcr-public/terra-jupyter-bioconductor:2.1.11", "label": "Legacy R / Bioconductor (R 4.2.2, Bioconductor 3.16, Python 3.7.12)", - "version": "2.1.9", + "version": "2.1.11", "requiresSpark": false, - "packages": "https://storage.googleapis.com/terra-docker-image-documentation/terra-jupyter-bioconductor-2.1.9-versions.json", + "packages": "https://storage.googleapis.com/terra-docker-image-documentation/terra-jupyter-bioconductor-2.1.11-versions.json", "id": "terra-jupyter-bioconductor_legacy" + }, + { + "updated": "2023-06-23", + "image": "us.gcr.io/broad-dsp-gcr-public/terra-jupyter-gatk:2.2.14", + "label": "Legacy GATK (GATK 4.2.4.0, Python 3.7.12, R 4.3.0)", + "version": "2.2.14", + "requiresSpark": false, + "packages": "https://storage.googleapis.com/terra-docker-image-documentation/terra-jupyter-gatk:2.2.14-versions.json", + "id": "terra-jupyter-gatk_legacy" } ] diff --git a/jenkins/Jenkinsfile b/jenkins/Jenkinsfile index 1ab3e876..f9b8d216 100644 --- a/jenkins/Jenkinsfile +++ b/jenkins/Jenkinsfile @@ -229,7 +229,7 @@ pipeline { description: "Specify this, along with the images below, to force the build of specific images. Can be used in conjunction with the branch parameter for testing. Note this will fail if an image for the version in the VERSION file exists in google.") string(name: "BRANCH", defaultValue: "master", description: "Specify along with useCustomImageIdentifier to build the image with a specific name") - text(name: 'IMAGES_TO_BUILD', defaultValue: 'terra-jupyter-base\nterra-jupyter-python\nterra-jupyter-r\nterra-jupyter-gatk\nterra-jupyter-bioconductor\nterra-jupyter-hail\nterra-jupyter-aou\nterra-jupyter-gatk-ovtf\nterra-rstudio-aou', + text(name: 'IMAGES_TO_BUILD', defaultValue: 'terra-jupyter-base\nterra-jupyter-python\nterra-jupyter-r\nterra-jupyter-gatk\nterra-jupyter-bioconductor\nterra-jupyter-hail\nterra-jupyter-aou\nterra-rstudio-aou', description: 'The is a text field for newline separate images to build. Each image should be specified via the directory it resides in. The default value is a working example.') } diff --git a/terra-jupyter-aou/CHANGELOG.md b/terra-jupyter-aou/CHANGELOG.md index 7709ec23..d00de538 100644 --- a/terra-jupyter-aou/CHANGELOG.md +++ b/terra-jupyter-aou/CHANGELOG.md @@ -1,3 +1,9 @@ +## 2.2.0 - 2023-06-23 + +- Update python from 3.7 to 3.10 + +Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.2.0` + ## 2.1.22 - 2023-06-01T17:49:47.721962740Z - Update `terra-jupyter-r` to `2.1.10` diff --git a/terra-jupyter-aou/Dockerfile b/terra-jupyter-aou/Dockerfile index a3bec54b..b2b07b4c 100644 --- a/terra-jupyter-aou/Dockerfile +++ b/terra-jupyter-aou/Dockerfile @@ -1,4 +1,4 @@ -FROM us.gcr.io/broad-dsp-gcr-public/terra-jupyter-gatk:2.2.12 +FROM us.gcr.io/broad-dsp-gcr-public/terra-jupyter-gatk:2.3.0 USER root @@ -25,12 +25,9 @@ RUN apt-get update && apt-get install -yq --no-install-recommends \ ENV PIP_USER=false -# pandas-profiling 3.1.0 causes conflicts with pyplot -# pandas-profiling 3.2.0 is incompatible with markupsafe 2.0.1 -# 3.0.0 (lower is untested) avoids both of these issues. RUN pip3 install --upgrade \ - "pandas_profiling<=3.0.0" \ - "markupsafe==2.0.1" + "pandas_profiling==3.6.6" \ + "markupsafe==2.1.2" RUN pip3 install \ nbstripout \ @@ -48,7 +45,7 @@ ENV PYTHONPATH $PYTHONPATH:/usr/lib/spark/python ENV PYSPARK_PYTHON=python3 -ENV HAIL_VERSION=0.2.107 +ENV HAIL_VERSION=0.2.109 RUN find $JUPYTER_HOME/scripts -name '*.sh' -type f | xargs chmod +x \ && $JUPYTER_HOME/scripts/kernel/kernelspec.sh $JUPYTER_HOME/scripts/kernel /opt/conda/share/jupyter/kernels \ @@ -59,7 +56,7 @@ RUN find $JUPYTER_HOME/scripts -name '*.sh' -type f | xargs chmod +x \ && update-alternatives --install /etc/hadoop/conf hadoop-conf /etc/hadoop/conf.empty 100 \ && update-alternatives --install /etc/hive/conf hive-conf /etc/hive/conf.dist 100 \ && apt-get update \ - && apt install -yq --no-install-recommends \ + && apt install -yq --no-install-recommends openjdk-8-jdk \ g++ \ liblz4-dev \ # specify Java 8 @@ -264,8 +261,6 @@ RUN curl -L http://www.repeatmasker.org/RepeatMasker/RepeatMasker-4.1.4.tar.gz \ RUN rm -f /usr/local/bin/workspace_cromwell.py # alias cromshell_beta to cromshell RUN echo "alias cromshell='cromshell-beta'" >> ~/.bash_aliases -# Downgrade jupyter_contrib_nbextensions as the newer version breaks snippet_menu extension -RUN pip install --force-reinstall -v "jupyter_contrib_nbextensions==0.5.1" # Downgrade Jinja2 as the newer version breaks the notebook # See https://jinja.palletsprojects.com/en/3.1.x/changes/#version-3-1-0 diff --git a/terra-jupyter-aou/tests/aou_smoke_test.ipynb b/terra-jupyter-aou/tests/aou_smoke_test.ipynb index 6975fb39..1bb59ad5 100644 --- a/terra-jupyter-aou/tests/aou_smoke_test.ipynb +++ b/terra-jupyter-aou/tests/aou_smoke_test.ipynb @@ -149,7 +149,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.10" + "version": "3.10.9" }, "toc": { "base_numbering": 1, diff --git a/terra-jupyter-base/CHANGELOG.md b/terra-jupyter-base/CHANGELOG.md index e1eedbc5..2d32aa95 100644 --- a/terra-jupyter-base/CHANGELOG.md +++ b/terra-jupyter-base/CHANGELOG.md @@ -1,3 +1,8 @@ +## 1.1.0 - 2023-06-23 + +- Upgrade python 3.7 to 3.10 + +Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-base:1.1.0` ## 1.0.14 - 2023-03-13T17:26:34.128391Z - Include `jupyter lab build` step diff --git a/terra-jupyter-base/Dockerfile b/terra-jupyter-base/Dockerfile index b6940294..ac89733c 100644 --- a/terra-jupyter-base/Dockerfile +++ b/terra-jupyter-base/Dockerfile @@ -1,6 +1,6 @@ # adapted from https://hub.docker.com/r/jupyter/base-notebook/ AKA https://github.com/jupyter/docker-stacks/tree/master/base-notebook -FROM gcr.io/deeplearning-platform-release/tf-gpu.2-7 +FROM gcr.io/deeplearning-platform-release/tf-gpu.2-11.py310 USER root @@ -21,7 +21,6 @@ RUN sudo -i \ # see article: https://developer.nvidia.com/blog/updating-the-cuda-linux-gpg-repository-key/ # first we remove the bad keys from base image (https://github.com/NVIDIA/nvidia-docker/issues/1631#issuecomment-1112828208) RUN rm /etc/apt/sources.list.d/cuda.list \ - && rm /etc/apt/sources.list.d/nvidia-ml.list \ && sudo apt-key del 7fa2af80 \ && curl -L -O https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-keyring_1.0-1_all.deb \ && sudo dpkg -i cuda-keyring_1.0-1_all.deb @@ -48,6 +47,8 @@ RUN apt-get update && apt-get install -yq --no-install-recommends \ libbz2-dev \ libreadline-dev \ libsqlite3-dev \ + libexempi3 \ + libv8-3.14-dev \ llvm \ libncurses5-dev \ libncursesw5-dev \ @@ -55,8 +56,6 @@ RUN apt-get update && apt-get install -yq --no-install-recommends \ libffi-dev \ liblzma-dev \ python-openssl \ - libexempi3 \ - libv8-3.14-dev \ # install script requirements sudo \ locales \ @@ -70,7 +69,7 @@ RUN apt-get update && apt-get install -yq --no-install-recommends \ # Generate locale && locale-gen \ && apt-get clean \ - && rm -rf /var/lib/apt/lists/* + && rm -rf /var/lib/apt/lists/* ENV LC_ALL en_US.UTF-8 @@ -101,7 +100,7 @@ RUN pip3 -V \ && pip3 install python-datauri \ && pip3 install jupyter_contrib_nbextensions \ && pip3 install jupyter_nbextensions_configurator \ - && pip3 install markupsafe==2.0.1 \ + && pip3 install markupsafe==2.1.2 \ # Avoid broken lower versions: https://github.com/jupyter/nbconvert/pull/1624 && pip3 install "nbconvert>=6.4.5" \ # for jupyter_delocalize.py and jupyter_notebook_config.py @@ -109,17 +108,12 @@ RUN pip3 -V \ && pip3 install firecloud \ && pip3 install terra-notebook-utils \ && pip3 install crcmod \ - # For gcloud alpha storage support. - && pip3 install google-crc32c --target /usr/lib/google-cloud-sdk/lib/third_party - -# install Cromshell 2.0 -RUN git clone https://github.com/broadinstitute/cromshell.git \ - && cd cromshell \ - && git checkout cromshell_2.0 \ - && pip install . + # For gcloud alpha storage support. + && pip3 install google-crc32c --target /usr/lib/google-cloud-sdk/lib/third_party \ + && pip3 install cromshell==2.0.0 -# verify cromshell 2.0 is installed -RUN cromshell-beta version +# to allow file-saving in different formats +RUN jupyter nbextension enable toc2/main # tmp hack min-5 # I'm not installing jupyterlab and I can't update init-actions.sh to not access it diff --git a/terra-jupyter-base/README.md b/terra-jupyter-base/README.md index d1cf36ad..446559e1 100644 --- a/terra-jupyter-base/README.md +++ b/terra-jupyter-base/README.md @@ -8,7 +8,7 @@ The terra-jupyter-base extends the [Ubuntu base image]("https://github.com/tiano - OS prerequisites - google-cloud-sdk -- Python 3.7 +- Python 3.10 - pip - conda - Jupyter & JupyterLab diff --git a/terra-jupyter-bioconductor/CHANGELOG.md b/terra-jupyter-bioconductor/CHANGELOG.md index f4a06047..7eb3e677 100644 --- a/terra-jupyter-bioconductor/CHANGELOG.md +++ b/terra-jupyter-bioconductor/CHANGELOG.md @@ -1,3 +1,9 @@ +## 2.2.0 - 2023-06-23 + +- Update python 3.7 to 3.10 + +Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-bioconductor:2.2.0` + ## 2.1.11 - 2023-06-01T17:49:47.622531112Z - Update `terra-jupyter-r` to `2.1.10` diff --git a/terra-jupyter-bioconductor/Dockerfile b/terra-jupyter-bioconductor/Dockerfile index 4bc5e08d..2a3b1601 100644 --- a/terra-jupyter-bioconductor/Dockerfile +++ b/terra-jupyter-bioconductor/Dockerfile @@ -1 +1 @@ -FROM us.gcr.io/broad-dsp-gcr-public/terra-jupyter-r:2.1.10 +FROM us.gcr.io/broad-dsp-gcr-public/terra-jupyter-r:2.2.0 \ No newline at end of file diff --git a/terra-jupyter-gatk-ovtf/CHANGELOG.md b/terra-jupyter-gatk-ovtf/CHANGELOG.md deleted file mode 100644 index bb7394fa..00000000 --- a/terra-jupyter-gatk-ovtf/CHANGELOG.md +++ /dev/null @@ -1,142 +0,0 @@ -## 0.2.10 - 2023-06-01T17:49:47.736670188Z - -- Update `terra-jupyter-r` to `2.1.10` - - Bioconductor 3.17 release with R 4.3 - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-gatk-ovtf:0.2.10` - -## 0.2.9 - 2023-03-13T17:26:34.258214Z - -- Update `terra-jupyter-base` to `1.0.14` - - Include `jupyter lab build` step - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-gatk-ovtf:0.2.9` - -## 0.2.8 - 2023-02-09T14:18:49.219503Z - -- Update `terra-jupyter-base` to `1.0.12` - - Add cloud.google.com gpg key. - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-gatk-ovtf:0.2.8` - -## 0.2.7 - 2022-11-22T20:49:44.239518474Z - -- Update `terra-jupyter-r` to `2.1.7` - - Update Bioconductor for 3.16 release - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-gatk-ovtf:0.2.7` - -## 0.2.6 - 2022-07-27T13:32:20.053219Z - -- Update `terra-jupyter-r` to `2.1.6` - - Installs AnVIL and supporting packages. - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-gatk-ovtf:0.2.6` - -## 0.2.5 - 2022-06-23T10:58:12.961300Z - -- Update `terra-jupyter-base` to `1.0.10` - - Fix leo_url variable in workspace_cromwell.py script for AoU projects - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-gatk-ovtf:0.2.4` - -## 0.2.4 - 2022-06-23T10:58:12.961300Z - -- reverted, do not use - -## 0.2.3 - 2022-05-20T18:06:39.600661Z - -- Update `terra-jupyter-base` to `1.0.9` - - Fix adding workspace_cromwell.py script to manage Cromwell App - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-gatk-ovtf:0.2.3` - -## 0.2.2 - 2022-05-17T17:14:41.382147Z - -- Update `terra-jupyter-base` to `1.0.8` - - Add script that manages Cromwell app - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-gatk-ovtf:0.2.2` - -## 0.2.1 - 2022-05-10T22:08:26.070940Z - -- Update `terra-jupyter-base` to `1.0.7` - - Install Cromshell 2.0 (https://github.com/broadinstitute/cromshell) - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-gatk-ovtf:0.2.1` - -## 0.2.0 - 2022-05-02 - -- Update `terra-jupyter-r` to `2.1.0` - - Update Bioconductor to 3.15.0 and R to 4.2.0 - -## 0.1.10 - 2022-05-02T14:33:24.040339Z - -- Update `terra-jupyter-base` to `1.0.6` - - use new nvidia key - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-gatk-ovtf:0.1.10` - -## 0.1.9 - 2022-04-17 - -- Update `terra-jupyter-base` to `1.0.5` - - Install gcloud alpha storage dependency - -## 0.1.8 - 2022-02-02T20:47:42.409160Z - -- Update `terra-jupyter-base` to `1.0.4` - - update notebook_dir to $HOME; install scikit-learn-intelex, xgboost - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-gatk-ovtf:0.1.8` - -## 0.1.7 - 2022-01-26T17:05:04.719540Z - -- OVTF1.1.0 released, Updated to GATK 4.2.4.1, TF2.7, GPU support enabled - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-gatk-ovtf:0.1.7` - -## 0.1.6 - 2022-01-06T18:21:41.248206Z - -- Update `terra-jupyter-base` to `1.0.3` - - Bumping Google Deeplearning image to 2.7, removing gcc-6 and gcc-8 install due to package problems - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-gatk-ovtf:0.1.6` - -## 0.1.5 - 2021-12-14T19:48:03.867076Z - -- Update `terra-jupyter-r` to `2.0.3` - - Update Terra Jupyter image to use latest Bioconductor 3.14 - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-gatk-ovtf:0.1.5` - -## 0.1.4 - 2021-12-15T18:30:57.568303Z - -- Upgrade gatk image to 4.2.4.0 - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-gatk-ovtf:0.1.4` - -## 0.1.3 - 2021-10-07T14:47:43.454585Z - -- Update `terra-jupyter-base` to `1.0.2` - - Update AsyncMappingKernelManager https://github.com/jupyter/notebook/issues/6164 -- Unpinning cwltool version and updating protobuf version to 3.18 - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-gatk-ovtf:0.1.3` - -## 0.1.2 - 2021-09-22T15:25:34.080267Z - -- Intel OVTF patch for updating to TF2.6.0/2.5.0 - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-gatk-ovtf:0.1.2` - -## 0.1.1 - 2021-09-10T15:10:44.164165Z - -- Update `terra-jupyter-base` to `1.0.1` - - Update base image to gcr.io/deeplearning-platform-release/tf2-gpu.2-6 to support TensorFlow 2.6.0 - - Fix multipart Jupyter uploads - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-gatk-ovtf:0.1.1` - -## 0.1.0 - 08/25/2021 - -- Added OpenVINO integration with TensorFlow (OVTF) diff --git a/terra-jupyter-gatk-ovtf/Dockerfile b/terra-jupyter-gatk-ovtf/Dockerfile deleted file mode 100644 index 6b820ec1..00000000 --- a/terra-jupyter-gatk-ovtf/Dockerfile +++ /dev/null @@ -1,70 +0,0 @@ -FROM us.gcr.io/broad-dsp-gcr-public/terra-jupyter-python:1.0.13 AS python - -FROM us.gcr.io/broad-dsp-gcr-public/terra-jupyter-r:2.1.10 - -# copy everything pip installed from the python image -COPY --from=python /opt/conda/lib/python3.7/site-packages /opt/conda/lib/python3.7/site-packages - -USER root - -# need to apt-get everything for python since we can only copy pip installed packages -RUN apt-get update && apt-get install -yq --no-install-recommends \ - python3.7-dev \ - python-tk \ - openjdk-8-jdk \ - tk-dev \ - libssl-dev \ - xz-utils \ - libhdf5-dev \ - openssl \ - make \ - liblzo2-dev \ - zlib1g-dev \ - libz-dev \ - samtools \ - git-lfs \ - # specify Java 8 - && update-alternatives --set java /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - -# Install GATK -ENV GATK_VERSION 4.2.4.1 -ENV GATK_ZIP_PATH /tmp/gatk-${GATK_VERSION}.zip - -RUN curl -L -o $GATK_ZIP_PATH https://github.com/broadinstitute/gatk/releases/download/$GATK_VERSION/gatk-$GATK_VERSION.zip \ - && unzip -o $GATK_ZIP_PATH -d /etc/ \ - && ln -s /etc/gatk-$GATK_VERSION/gatk /bin/gatk - -# Install OpenVINO™ integration with TensorFlow (OVTF) -ENV TF_VER 2.7.0 -ENV KERAS_VER 2.7.0 -ENV OVTF_VER 1.1.0 -ENV TF_ENABLE_ONEDNN_OPTS 1 - -ENV PIP_USER=false - -# Force reinstalling TF and Keras to satisfy OVTF compatibility. -RUN pip install --force-reinstall tensorflow==$TF_VER keras==$KERAS_VER markupsafe==2.0.1 -RUN conda install six typing_extensions --force-reinstall \ - && pip install openvino_tensorflow==$OVTF_VER \ - && pip install matplotlib scikit-learn \ - && pip install /etc/gatk-$GATK_VERSION/gatkPythonPackageArchive.zip -ENV PIP_USER=true - -COPY gatk-ovtf.patch /etc/gatk-$GATK_VERSION/gatk-ovtf.patch -COPY ovtf-cuda-disabling.patch /etc/gatk-$GATK_VERSION/ovtf-cuda-disabling.patch - -# Apply patch to GATK to fix TFv2 compatibility. -RUN cd /opt/conda/lib/python3.7/site-packages/vqsr_cnn/vqsr_cnn/ \ - && patch -p1 < /etc/gatk-$GATK_VERSION/gatk-ovtf.patch - -# Apply patch to OVTF to disable CUDA devices if OVTF is enabled. -RUN cd /opt/conda/lib/python3.7/site-packages/openvino_tensorflow/ \ - && patch __init__.py < /etc/gatk-$GATK_VERSION/ovtf-cuda-disabling.patch - -COPY --chown=jupyter:users run_gatk.sh /home/jupyter/run_gatk.sh -COPY --chown=jupyter:users GATK-OVTF-Notebook.ipynb /home/jupyter/GATK-OVTF-Notebook.ipynb - -ENV USER jupyter -USER $USER diff --git a/terra-jupyter-gatk-ovtf/GATK-OVTF-Notebook.ipynb b/terra-jupyter-gatk-ovtf/GATK-OVTF-Notebook.ipynb deleted file mode 100644 index 1ccd9485..00000000 --- a/terra-jupyter-gatk-ovtf/GATK-OVTF-Notebook.ipynb +++ /dev/null @@ -1,104 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# OpenVINO integration with TensorFlow on GATK CNNScoreVariants\n", - "\n", - "This notebook demonstrates using OpenVINO integration with TensorFlow on GATK CNNScoreVariants to achieve an improved performance.\n", - "\n", - "1. First, we execute CNNScoreVariants on a sample input data to execute and observe the improved throughput.\n", - "\n", - "2. Then, we execute CNNScoreVariants using the same input data and parameters again. However, this time we disable OpenVINO integration with TensorFlow to compare the throughput with stock TensorFlow. \n", - "\n", - "***Note:*** This notebook is intented to be run with \"terra-docker-gatk-ovtf\" image." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## GATK CNNScoreVariants on OV-TF" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "!gatk CNNScoreVariants \\\n", - " -I gs://gatk-tutorials/workshop_2002/2-germline/CNNScoreVariants/bams/g94982_chr20_1m_10m_bamout.bam \\\n", - " -V gs://gatk-tutorials/workshop_2002/2-germline/CNNScoreVariants/vcfs/g94982_b37_chr20_1m_15871.vcf.gz \\\n", - " -R gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta \\\n", - " -O my_2d_cnn_scored.vcf \\\n", - " --tensor-type read_tensor \\\n", - " --transfer-batch-size 256 \\\n", - " --inference-batch-size 256" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Baseline GATK CNNScoreVariants" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "!OPENVINO_TF_DISABLE=\"1\" \\\n", - " gatk CNNScoreVariants \\\n", - " -I gs://gatk-tutorials/workshop_2002/2-germline/CNNScoreVariants/bams/g94982_chr20_1m_10m_bamout.bam \\\n", - " -V gs://gatk-tutorials/workshop_2002/2-germline/CNNScoreVariants/vcfs/g94982_b37_chr20_1m_15871.vcf.gz \\\n", - " -R gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta \\\n", - " -O my_2d_cnn_scored_ovtf.vcf \\\n", - " --tensor-type read_tensor \\\n", - " --transfer-batch-size 256 \\\n", - " --inference-batch-size 256" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.10" - }, - "toc": { - "base_numbering": 1, - "nav_menu": {}, - "number_sections": true, - "sideBar": true, - "skip_h1_title": false, - "title_cell": "Table of Contents", - "title_sidebar": "Contents", - "toc_cell": false, - "toc_position": {}, - "toc_section_display": true, - "toc_window_display": false - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/terra-jupyter-gatk-ovtf/README.md b/terra-jupyter-gatk-ovtf/README.md deleted file mode 100644 index 90ba9397..00000000 --- a/terra-jupyter-gatk-ovtf/README.md +++ /dev/null @@ -1,99 +0,0 @@ -# terra-jupyter-gatk-ovtf image - -This repo contains the `terra-jupyter-gatk-ovtf` image that is compatible with notebook service in [Terra](https://app.terra.bio/) called Leonardo. - -To use this in [Terra](https://app.terra.bio/), use `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-gatk-ovtf:` as image identifier. See [CHANGELOG.md](CHANGELOG.md) for the most recent tag. - -In Terra, this image is available in the drop-down menu under **COMMUNITY-MAINTAINED JUPYTER ENVIRONMENTS** - - - - -## Image contents - -The `terra-jupyter-gatk-ovtf` docker image extends the [terra-jupyter-python](../terra-jupyter-python/README.md) and [terra-jupyter-r](../terra-jupyter-r/README.md) by including the following: - -- Open JDK -- GATK -- Samtools -- [OpenVINO integration with TensorFlow](https://github.com/openvinotoolkit/openvino_tensorflow) (OVTF) - -To see the complete contents of this image please see the [Dockerfile](./Dockerfile). - -## Sample Notebook - -This repo provides a sample notebook [GATK-OVTF-Notebook.ipynb](./GATK-OVTF-Notebook.ipynb) which showcases the performance benefits obtained by using OpenVINO™ integration with TensorFlow. - -## Note on CUDA-enabled GPU devices Support - -Enabling OpenVINO™ integration with TensorFlow disables TensorFlow GPU support for CUDA-enabled devices in this docker image. -If you prefer to use CUDA-enabled GPU devices, DONOT import openvino_tensorflow - -#### Scenario #1: Use CUDA-enabled GPU devices with native Tensorflow. -After installing CUDA drivers in the host machine, you will need to start the docker with `--gpus all` to enable GPU devices inside docker container. - -Example: `docker run --gpus all --rm -it -p 8000:8000 terra-jupyter-gatk-ovtf`, Then navigate a browser to http://localhost:8000/notebooks to access the Jupyter UI. - -``` -import tensorflow as tf -# CPU and GPU support available. openvino_tensorflow is disabled -tf.config.list_physical_devices() -``` - -#### Scenario #2: Use OpenVINO™ integration with TensorFlow -``` -import tensorflow as tf -import openvino_tensorflow as ovtf -ovtf.set_backend("CPU") - -# By importing openvino_tensorflow, CUDA-enabled GPU devices support is disabled. -# To re-enable GPU support, restart the Jupyter kernel, and remove "import openvino_tensorflow as ovtf" -``` - -#### Scenario #3: Use GATK with CUDA-enabled GPU devices -In this Docker image, GATK is enabled with OpenVINO™ integration with TensorFlow by default. -If you prefer to use GATK with CUDA-enabled GPU devices, then OpenVINO integration with TensorFlow should be disabled before executing GATK CNNScoreVariants. To disable OpenVINO™ integration with TensorFlow, run the command below in the Jupyter notebook. -``` -! export OPENVINO_TF_DISABLE=1 -``` - -#### Scenario #4: Use OpenVINO™ integration with TensorFlow with CUDA-enabled GPU devices - -Not Supported. - ---------- - - -## Build and Run Instructions - **Locally** - -- ### Clone the repository - - ```bash - git clone https://github.com/DataBiosphere/terra-docker - - ``` - -- ### Build the docker image - - ```bash - cd terra-docker/terra-jupyter-gatk-ovtf - docker build . -t terra-jupyter-gatk-ovtf - ``` - -- ### Run the container, Then navigate a browser to http://localhost:8000/notebooks to access the Jupyter UI - - ```bash - docker run --rm -it -p 8000:8000 terra-jupyter-gatk-ovtf - ``` - -- ### You can gain root access and open a bash terminal as follows: - - ```bash - docker run --rm -it -u root -p 8000:8000 --entrypoint /bin/bash terra-jupyter-gatk-ovtf - ``` - -## Selecting prior versions of this image - -To select an older version this image, you can search the [CHANGELOG.md](./CHANGELOG.md) for a specific package version you need. - -Once you find an image version that you want, simply copy and paste the image url from the changelog into the corresponding custom docker field in the Terra notebook runtime widget. diff --git a/terra-jupyter-gatk-ovtf/build.sh b/terra-jupyter-gatk-ovtf/build.sh deleted file mode 100755 index fd6bc89b..00000000 --- a/terra-jupyter-gatk-ovtf/build.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - - -docker build . -t iotgedge/terra-jupyter-gatk-ovtf diff --git a/terra-jupyter-gatk-ovtf/gatk-ovtf.patch b/terra-jupyter-gatk-ovtf/gatk-ovtf.patch deleted file mode 100644 index 2604c69f..00000000 --- a/terra-jupyter-gatk-ovtf/gatk-ovtf.patch +++ /dev/null @@ -1,104 +0,0 @@ -diff -ru vqsr_cnn/inference.py /opt/conda/lib/python3.7/site-packages/vqsr_cnn/vqsr_cnn/inference.py ---- vqsr_cnn/inference.py 2019-01-29 15:50:10.000000000 +0000 -+++ /opt/conda/lib/python3.7/site-packages/vqsr_cnn/vqsr_cnn/inference.py 2021-09-22 19:47:00.000000000 +0000 -@@ -11,6 +11,11 @@ - # Keras Imports - import keras - import keras.backend as K -+import tensorflow as tf -+from tensorflow.python.framework.convert_to_constants import convert_variables_to_constants_v2 -+ -+import openvino_tensorflow -+openvino_tensorflow.set_backend('CPU') - - # Package Imports - from . import defines -@@ -38,7 +43,9 @@ - VARIANT_TYPE_FIFO_INDEX = 6 - VARIANT_FIFO_FIELDS = 7 - -- -+session = None -+prob_tensor = None -+full_model = None - - CIGAR_CODES_TO_COUNT = [ - defines.CIGAR_CODE['M'], defines.CIGAR_CODE['I'], defines.CIGAR_CODE['S'], defines.CIGAR_CODE['D'] -@@ -86,6 +93,11 @@ - variant_types = [] - variant_data = [] - read_batch = [] -+ -+ global session -+ global prob_tensor -+ global full_model -+ - for _ in range(batch_size): - fifo_line = tool.readDataFIFO() - fifo_data = fifo_line.split(defines.DATA_TYPE_SEPARATOR) -@@ -126,8 +138,24 @@ - predictions = model.predict([np.array(reference_batch), np.array(annotation_batch)], - batch_size=python_batch_size) - elif tensor_type in defines.TENSOR_MAPS_2D: -- predictions = model.predict( -- [np.array(read_batch), np.array(annotation_batch)], batch_size=python_batch_size) -+ if session is None: -+ full_model = tf.function(lambda x: model(x)) -+ full_model = full_model.get_concrete_function( -+ (tf.TensorSpec(model.inputs[0].shape, model.inputs[0].dtype, name="read_tensor"), -+ tf.TensorSpec(model.inputs[1].shape, model.inputs[1].dtype, name="best_practices"))) -+ frozen_func = convert_variables_to_constants_v2(full_model) -+ frozen_func.graph.as_graph_def() -+ session = tf.compat.v1.Session(graph=frozen_func.graph) -+ prob_tensor = frozen_func.graph.get_tensor_by_name("model_1/softmax_predictions/Softmax:0") -+ for i in range(python_batch_size-batch_size): -+ tensor = np.empty(read_batch[0].shape) -+ read_batch.append(tensor) -+ tensor = np.empty(annotation_batch[0].shape) -+ annotation_batch.append(tensor) -+ batch = {} -+ batch["read_tensor:0"] = np.array(read_batch) -+ batch["best_practices:0"] = np.array(annotation_batch) -+ predictions = session.run(prob_tensor, feed_dict=batch) - else: - raise ValueError('Unknown tensor mapping. Check architecture file.', tensor_type) - -diff -ru vqsr_cnn/models.py /opt/conda/lib/python3.7/site-packages/vqsr_cnn/vqsr_cnn/models.py ---- vqsr_cnn/models.py 2019-01-29 15:50:10.000000000 +0000 -+++ /opt/conda/lib/python3.7/site-packages/vqsr_cnn/vqsr_cnn/models.py 2021-09-22 19:47:00.000000000 +0000 -@@ -4,12 +4,13 @@ - # Keras Imports - from keras import layers - from keras import metrics --import keras.backend as K --from keras.optimizers import Adam -+import tensorflow.compat.v1.keras.backend as K -+from keras.optimizers import adam_v2 as Adam - from keras.models import Model, load_model - from keras.layers.convolutional import Conv1D, Conv2D, MaxPooling1D, MaxPooling2D - from keras.callbacks import ModelCheckpoint, EarlyStopping, TensorBoard, ReduceLROnPlateau - from keras.layers import Input, Dense, Dropout, BatchNormalization, SpatialDropout1D, SpatialDropout2D, Activation, Flatten, AlphaDropout -+import tensorflow as tf - - from . import plots - from . import defines -@@ -20,9 +21,9 @@ - def start_session_get_args_and_model(intra_ops, inter_ops, semantics_json, weights_hd5=None, tensor_type=None): - K.clear_session() - K.get_session().close() -- cfg = K.tf.ConfigProto(intra_op_parallelism_threads=intra_ops, inter_op_parallelism_threads=inter_ops) -+ cfg = tf.compat.v1.ConfigProto(intra_op_parallelism_threads=intra_ops, inter_op_parallelism_threads=inter_ops) - cfg.gpu_options.allow_growth = True -- K.set_session(K.tf.Session(config=cfg)) -+ K.set_session(tf.compat.v1.Session(config=cfg)) - return args_and_model_from_semantics(semantics_json, weights_hd5, tensor_type) - - -@@ -609,4 +610,5 @@ - with open(json_file_name, 'w') as outfile: - json.dump(semantics, outfile) - -- print('Saved model semantics at:', json_file_name) -\ No newline at end of file -+ print('Saved model semantics at:', json_file_name) -+ diff --git a/terra-jupyter-gatk-ovtf/ovtf-cuda-disabling.patch b/terra-jupyter-gatk-ovtf/ovtf-cuda-disabling.patch deleted file mode 100644 index 169bbbe4..00000000 --- a/terra-jupyter-gatk-ovtf/ovtf-cuda-disabling.patch +++ /dev/null @@ -1,12 +0,0 @@ ---- /opt/conda/lib/python3.7/site-packages/openvino_tensorflow/__init__.py 2021-12-09 22:39:36.000000000 +0000 -+++ __init__.py 2021-12-10 00:20:28.433852578 +0000 -@@ -30,6 +30,9 @@ - - import ctypes - -+if (os.environ.get("OPENVINO_TF_DISABLE") != "1"): -+ os.environ["CUDA_VISIBLE_DEVICES"] = "-1" -+ - __all__ = [ - 'enable', 'disable', 'is_enabled', 'list_backends', - 'set_backend', 'get_backend', diff --git a/terra-jupyter-gatk-ovtf/run_gatk.sh b/terra-jupyter-gatk-ovtf/run_gatk.sh deleted file mode 100644 index fa821efd..00000000 --- a/terra-jupyter-gatk-ovtf/run_gatk.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -gatk CNNScoreVariants \ - -I gs://gatk-tutorials/workshop_2002/2-germline/CNNScoreVariants/bams/g94982_chr20_1m_10m_bamout.bam \ - -V gs://gatk-tutorials/workshop_2002/2-germline/CNNScoreVariants/vcfs/g94982_b37_chr20_1m_15871.vcf.gz \ - -R gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta \ - -O my_2d_cnn_scored.vcf \ - --tensor-type read_tensor \ - --transfer-batch-size 256 \ - --inference-batch-size 256 diff --git a/terra-jupyter-gatk-ovtf/terra-ovtf-gatk.png b/terra-jupyter-gatk-ovtf/terra-ovtf-gatk.png deleted file mode 100644 index 332ffb54..00000000 Binary files a/terra-jupyter-gatk-ovtf/terra-ovtf-gatk.png and /dev/null differ diff --git a/terra-jupyter-gatk-ovtf/tests/ovtf_smoke_test.ipynb b/terra-jupyter-gatk-ovtf/tests/ovtf_smoke_test.ipynb deleted file mode 100644 index aed91ade..00000000 --- a/terra-jupyter-gatk-ovtf/tests/ovtf_smoke_test.ipynb +++ /dev/null @@ -1,144 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "source": [ - "# Test cases requiring or benefiting from the context of a notebook\n", - "\n", - "If the notebook runs successfully from start to finish, the test is successful!\n", - "\n", - "TODO(all): Add additional tests and/or tests with particular assertions, as we encounter Python package version incompatibilities not currently detected by these tests.\n", - "\n", - "In general, only add test cases here that require the context of a notebook. This is because this notebook, as currently written, will abort at the **first** failure. Compare this to a proper test suite where all cases are run, giving much more information about the full extent of any problems encountered." - ], - "metadata": {} - }, - { - "cell_type": "markdown", - "source": [ - "# Package versions" - ], - "metadata": {} - }, - { - "cell_type": "code", - "execution_count": null, - "source": [ - "!pip3 freeze" - ], - "outputs": [], - "metadata": {} - }, - { - "cell_type": "markdown", - "source": [ - "# Test cases requiring the context of a notebook " - ], - "metadata": {} - }, - { - "cell_type": "markdown", - "source": [ - "## Test OpenVINO integration with Tensorflow (OVTF)" - ], - "metadata": {} - }, - { - "cell_type": "code", - "execution_count": null, - "source": [ - "import tensorflow as tf\n", - "import openvino_tensorflow\n", - "\n", - "print('TensorFlow version: ',tf.__version__)\n", - "print(openvino_tensorflow.__version__)" - ], - "outputs": [], - "metadata": {} - }, - { - "cell_type": "markdown", - "source": [ - "## Test GATK" - ], - "metadata": {} - }, - { - "cell_type": "code", - "execution_count": null, - "source": [ - "!gatk -version" - ], - "outputs": [], - "metadata": {} - }, - { - "cell_type": "code", - "execution_count": null, - "source": [ - "!gatk --dry-run" - ], - "outputs": [], - "metadata": {} - }, - { - "cell_type": "markdown", - "source": [ - "## Test Run the default 1D model on the VCF with CNNScoreVariants" - ], - "metadata": {} - }, - { - "cell_type": "code", - "execution_count": null, - "source": [ - "!gatk CNNScoreVariants \\\n", - " -I gs://gatk-tutorials/workshop_2002/2-germline/CNNScoreVariants/bams/g94982_chr20_1m_10m_bamout.bam \\\n", - " -V gs://gatk-tutorials/workshop_2002/2-germline/CNNScoreVariants/vcfs/g94982_b37_chr20_1m_895.vcf \\\n", - " -O output_my_1d_cnn_scored.vcf \\\n", - " -R gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta" - ], - "outputs": [], - "metadata": {} - } - ], - "metadata": { - "environment": { - "name": "r-cpu.3-6.m56", - "type": "gcloud", - "uri": "gcr.io/deeplearning-platform-release/r-cpu.3-6:m56" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.10" - }, - "toc": { - "base_numbering": 1, - "nav_menu": {}, - "number_sections": true, - "sideBar": true, - "skip_h1_title": false, - "title_cell": "Table of Contents", - "title_sidebar": "Contents", - "toc_cell": false, - "toc_position": {}, - "toc_section_display": true, - "toc_window_display": true - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} \ No newline at end of file diff --git a/terra-jupyter-gatk-ovtf/tests/ovtf_smoke_test.py b/terra-jupyter-gatk-ovtf/tests/ovtf_smoke_test.py deleted file mode 100644 index 5de2c8ef..00000000 --- a/terra-jupyter-gatk-ovtf/tests/ovtf_smoke_test.py +++ /dev/null @@ -1,22 +0,0 @@ -"""Brief smoke tests for terra-jupyter-gatk-ovtf. - -Minimal tests simply import packages and run 'hello world' code to determine whether any exceptions -are thrown. - -TODO(all): Add additional tests and/or tests with particular assertions, as we encounter Python package version -incompatibilities not currently detected by these tests. -""" - -from __future__ import print_function -import os -import pytest - -import tensorflow as tf -import openvino_tensorflow - - -def test_ovtf(): - print("TensorFlow version : ", tf.version.GIT_VERSION, tf.version.VERSION) - - openvino_tensorflow.enable() - assert openvino_tensorflow.is_enabled() == 1 diff --git a/terra-jupyter-gatk/CHANGELOG.md b/terra-jupyter-gatk/CHANGELOG.md index d9dda6af..719b7520 100644 --- a/terra-jupyter-gatk/CHANGELOG.md +++ b/terra-jupyter-gatk/CHANGELOG.md @@ -1,3 +1,9 @@ +## 2.3.0 - 2023-06-23 + +- Update python 3.7 to 3.10 + +Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-gatk:2.3.0` + ## 2.2.14 - 2023-06-09 - Actually install bedtools to accessible path diff --git a/terra-jupyter-gatk/Dockerfile b/terra-jupyter-gatk/Dockerfile index 87a7b0b7..2b7f315a 100644 --- a/terra-jupyter-gatk/Dockerfile +++ b/terra-jupyter-gatk/Dockerfile @@ -1,12 +1,11 @@ -FROM us.gcr.io/broad-dsp-gcr-public/terra-jupyter-python:1.0.15 AS python +FROM us.gcr.io/broad-dsp-gcr-public/terra-jupyter-python:1.1.0 AS python -FROM us.gcr.io/broad-dsp-gcr-public/terra-jupyter-r:2.1.10 +FROM us.gcr.io/broad-dsp-gcr-public/terra-jupyter-r:2.2.0 USER root # need to apt-get everything for python since we can only copy pip installed packages RUN apt-get update && apt-get install -yq --no-install-recommends \ - python3.7-dev \ python-tk \ openjdk-8-jdk \ tk-dev \ @@ -77,11 +76,11 @@ RUN mkdir -p /tmp/bedtools && \ rm -rf /tmp/bedtools RUN pip install /etc/gatk-$GATK_VERSION/gatkPythonPackageArchive.zip -RUN pip3 install --upgrade markupsafe==2.0.1 +RUN pip3 install --upgrade markupsafe==2.1.2 COPY cnn-models.patch /etc/gatk-$GATK_VERSION/cnn-models.patch -RUN patch -u /opt/conda/lib/python3.7/site-packages/vqsr_cnn/vqsr_cnn/models.py -i /etc/gatk-$GATK_VERSION/cnn-models.patch +RUN patch -u /opt/conda/lib/python3.10/site-packages/vqsr_cnn/vqsr_cnn/models.py -i /etc/gatk-$GATK_VERSION/cnn-models.patch ENV PIP_USER=true diff --git a/terra-jupyter-gatk/tests/gatk_smoke_test.ipynb b/terra-jupyter-gatk/tests/gatk_smoke_test.ipynb index ccabbbc2..af6293cc 100644 --- a/terra-jupyter-gatk/tests/gatk_smoke_test.ipynb +++ b/terra-jupyter-gatk/tests/gatk_smoke_test.ipynb @@ -151,7 +151,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.10" + "version": "3.10.9" }, "toc": { "base_numbering": 1, diff --git a/terra-jupyter-hail/CHANGELOG.md b/terra-jupyter-hail/CHANGELOG.md index 9f6dd4f3..60df676a 100644 --- a/terra-jupyter-hail/CHANGELOG.md +++ b/terra-jupyter-hail/CHANGELOG.md @@ -1,3 +1,9 @@ +## 1.1.0 - 2023-06-23 + +- Update python 3.7 to 3.10 + +Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-hail:1.1.0` + ## 1.0.26 - 2023-06-07 - Force the regeneration of 1.0.26 that was never published diff --git a/terra-jupyter-hail/Dockerfile b/terra-jupyter-hail/Dockerfile index e5118944..dfd26af5 100644 --- a/terra-jupyter-hail/Dockerfile +++ b/terra-jupyter-hail/Dockerfile @@ -1,6 +1,8 @@ -FROM us.gcr.io/broad-dsp-gcr-public/terra-jupyter-python:1.0.15 +FROM us.gcr.io/broad-dsp-gcr-public/terra-jupyter-python:1.1.0 USER root + +# If anything below this is updated, make the corresponding changes in the aou image ENV PIP_USER=false ENV PYTHONPATH $PYTHONPATH:/usr/lib/spark/python ENV PYSPARK_PYTHON=python3 diff --git a/terra-jupyter-hail/tests/hail_smoke_test.ipynb b/terra-jupyter-hail/tests/hail_smoke_test.ipynb index 977a6b16..18e15201 100644 --- a/terra-jupyter-hail/tests/hail_smoke_test.ipynb +++ b/terra-jupyter-hail/tests/hail_smoke_test.ipynb @@ -74,7 +74,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.8" + "version": "3.10.9" }, "toc": { "base_numbering": 1, diff --git a/terra-jupyter-python/CHANGELOG.md b/terra-jupyter-python/CHANGELOG.md index b5888365..a68a2385 100644 --- a/terra-jupyter-python/CHANGELOG.md +++ b/terra-jupyter-python/CHANGELOG.md @@ -1,3 +1,9 @@ +## 1.1.0 - 2023-06-23 + +- Update python 3.7 to 3.10 + +Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-python:1.1.0` + ## 1.0.15 - 2023-03-13T17:26:34.179905Z - Update `terra-jupyter-base` to `1.0.14` diff --git a/terra-jupyter-python/Dockerfile b/terra-jupyter-python/Dockerfile index ed9a32f4..69b57057 100644 --- a/terra-jupyter-python/Dockerfile +++ b/terra-jupyter-python/Dockerfile @@ -1,4 +1,4 @@ -FROM us.gcr.io/broad-dsp-gcr-public/terra-jupyter-base:1.0.14 +FROM us.gcr.io/broad-dsp-gcr-public/terra-jupyter-base:1.1.0 USER root # This makes it so pip runs as root, not the user. ENV PIP_USER=false @@ -41,14 +41,14 @@ RUN pip3 -V \ # for ''`. To work-around this uninstall the dependency so that flag `--use_rest_api` can be used # with `%%bigquery` to use the older, slower mechanism for data transfer. && pip3 uninstall -y google-cloud-bigquery-storage \ - && sed -i 's/pandas.lib/pandas/g' /opt/conda/lib/python3.7/site-packages/ggplot/stats/smoothers.py \ + && sed -i 's/pandas.lib/pandas/g' /opt/conda/lib/python3.10/site-packages/ggplot/stats/smoothers.py \ # the next few `sed` lines are workaround for a ggplot bug. See https://github.com/yhat/ggpy/issues/662 - && sed -i 's/pandas.tslib.Timestamp/pandas.Timestamp/g' /opt/conda/lib/python3.7/site-packages/ggplot/stats/smoothers.py \ - && sed -i 's/pd.tslib.Timestamp/pd.Timestamp/g' /opt/conda/lib/python3.7/site-packages/ggplot/stats/smoothers.py \ - && sed -i 's/pd.tslib.Timestamp/pd.Timestamp/g' /opt/conda/lib/python3.7/site-packages/ggplot/utils.py + && sed -i 's/pandas.tslib.Timestamp/pandas.Timestamp/g' /opt/conda/lib/python3.10/site-packages/ggplot/stats/smoothers.py \ + && sed -i 's/pd.tslib.Timestamp/pd.Timestamp/g' /opt/conda/lib/python3.10/site-packages/ggplot/stats/smoothers.py \ + && sed -i 's/pd.tslib.Timestamp/pd.Timestamp/g' /opt/conda/lib/python3.10/site-packages/ggplot/utils.py -RUN pip3 install --upgrade markupsafe==2.0.1 +RUN pip3 install --upgrade markupsafe==2.1.2 ENV USER jupyter USER $USER diff --git a/terra-jupyter-python/requirements.txt b/terra-jupyter-python/requirements.txt index 97bc7fc9..60f1a3cc 100644 --- a/terra-jupyter-python/requirements.txt +++ b/terra-jupyter-python/requirements.txt @@ -21,12 +21,8 @@ cycler h5py html5lib joblib -# Remove the version pin for keras when tensorflow is updated to 2.7 or higher. -keras==2.7.0 patsy pymc3 -# fix to 2.4.7 so that tensorflow still works -pyparsing==2.4.7 Cython pysam --no-binary=pysam @@ -45,5 +41,4 @@ google-resumable-media #adding intel optimized xgboost and intel extension for scikit-learn scikit-learn-intelex xgboost -exceptiongroup -google-auth \ No newline at end of file +exceptiongroup \ No newline at end of file diff --git a/terra-jupyter-python/tests/smoke_test.ipynb b/terra-jupyter-python/tests/smoke_test.ipynb index 9902f1c3..7d2e9037 100644 --- a/terra-jupyter-python/tests/smoke_test.ipynb +++ b/terra-jupyter-python/tests/smoke_test.ipynb @@ -85,9 +85,9 @@ "metadata": {}, "outputs": [], "source": [ - "output = !pip3 show rich\n", + "output = !pip3 show pendulum\n", "print(output) # Should show not yet installed.\n", - "assert(0 == output.count('Name: rich'))" + "assert(0 == output.count('Name: pendulum'))" ] }, { @@ -96,7 +96,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install rich==10.16.1" + "%pip install pendulum==2.1.2" ] }, { @@ -105,9 +105,9 @@ "metadata": {}, "outputs": [], "source": [ - "output = !pip3 show rich\n", + "output = !pip3 show pendulum\n", "print(output) # Should show that it is now installed!\n", - "assert(1 == output.count('Name: rich'))" + "assert(1 == output.count('Name: pendulum'))" ] }, { @@ -123,9 +123,10 @@ "metadata": {}, "outputs": [], "source": [ - "output = !pip3 show docstring-parser\n", + "# python setup.py install\n", + "output = !pip3 show thefuzz\n", "print(output) # Should show not yet installed.\n", - "assert(0 == output.count('Name: docstring-parser'))" + "assert(0 == output.count('Name: thefuzz'))" ] }, { @@ -134,7 +135,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install docstring-parser==0.13" + "%pip install thefuzz" ] }, { @@ -143,9 +144,9 @@ "metadata": {}, "outputs": [], "source": [ - "output = !pip3 show docstring-parser\n", + "output = !pip3 show thefuzz\n", "print(output) # Should show that it is now installed!\n", - "assert(1 == output.count('Name: docstring-parser'))" + "assert(1 == output.count('Name: thefuzz'))" ] }, { @@ -401,6 +402,13 @@ "gsutil ls gs://gcp-public-data--gnomad" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, { "cell_type": "code", "execution_count": null, @@ -516,8 +524,15 @@ "outputs": [], "source": [ "import tensorflow as tf\n", - "import keras\n", - "\n", + "import keras" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ "print(\"TensorFlow version:\", tf.__version__)\n", "print(\"Keras version:\", keras.__version__)\n", "print(\"TensorFlow executing_eagerly:\", tf.executing_eagerly())" @@ -867,7 +882,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.12" + "version": "3.10.9" }, "toc": { "base_numbering": 1, diff --git a/terra-jupyter-r/CHANGELOG.md b/terra-jupyter-r/CHANGELOG.md index c12d23cd..3cff50a4 100644 --- a/terra-jupyter-r/CHANGELOG.md +++ b/terra-jupyter-r/CHANGELOG.md @@ -1,3 +1,9 @@ +## 2.2.0 - 2023-06-23 + +- Python upgrade 3.7 to 3.10 + +Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-r:2.2.0` + ## 2.1.10 - 2023-06-01T17:49:47.585402026Z - Bioconductor 3.17 release with R 4.3 diff --git a/terra-jupyter-r/Dockerfile b/terra-jupyter-r/Dockerfile index cdb5f208..db423e0d 100644 --- a/terra-jupyter-r/Dockerfile +++ b/terra-jupyter-r/Dockerfile @@ -1,17 +1,17 @@ -FROM us.gcr.io/broad-dsp-gcr-public/terra-jupyter-base:1.0.14 +FROM us.gcr.io/broad-dsp-gcr-public/terra-jupyter-base:1.1.0 USER root COPY scripts $JUPYTER_HOME/scripts # Add env vars to identify binary package installation -ENV TERRA_R_PLATFORM="terra-jupyter-r" +ENV TERRA_R_PLATFORM="terra-jupyter-r-1.1.0" ENV TERRA_R_PLATFORM_BINARY_VERSION=3.17 -# Install protobuf 3.19.1. Note this version comes from base deep learning image. Use `conda list` to see what's installed +# Install protobuf 3.20.3. Note this version comes from base deep learning image. Use `conda list` to see what's installed RUN cd /tmp \ - && wget https://github.com/protocolbuffers/protobuf/releases/download/v3.19.1/protobuf-all-3.19.1.tar.gz \ - && tar -xvzf protobuf-all-3.19.1.tar.gz \ - && cd protobuf-3.19.1/ \ + && wget https://github.com/protocolbuffers/protobuf/releases/download/v3.20.3/protobuf-all-3.20.3.tar.gz \ + && tar -xvzf protobuf-all-3.20.3.tar.gz \ + && cd protobuf-3.20.3/ \ && ./configure \ && make \ && make check \ diff --git a/updateVersions.sc b/updateVersions.sc index a73a4408..eca9d83a 100755 --- a/updateVersions.sc +++ b/updateVersions.sc @@ -5,9 +5,9 @@ val workbenchUtil2 = "0.10-c3ef6b80-SNAP" -val circeVersion = "0.13.0" +val circeVersion = "0.14.0" interp.load.ivy( - "co.fs2" %% "fs2-io" % "2.4.2", + "co.fs2" %% "fs2-io" % "2.5.10", "io.circe" %% "circe-core" % circeVersion, "io.circe" %% "circe-parser" % circeVersion, "io.circe" %% "circe-generic" % circeVersion, @@ -78,21 +78,18 @@ def main(updatedImage: String, updatedImageReleaseNote: String, bumpMajorVersion "terra-jupyter-r", "terra-jupyter-gatk", "terra-jupyter-aou", - "terra-jupyter-gatk-ovtf" ) case "terra-jupyter-r" => List( "terra-jupyter-r", "terra-jupyter-bioconductor", "terra-jupyter-gatk", "terra-jupyter-aou", - "terra-jupyter-gatk-ovtf" ) case "terra-jupyter-python" => List( "terra-jupyter-python", "terra-jupyter-hail", "terra-jupyter-gatk", "terra-jupyter-aou", - "terra-jupyter-gatk-ovtf" ) case "terra-jupyter-hail" => List( "terra-jupyter-hail" @@ -100,9 +97,6 @@ def main(updatedImage: String, updatedImageReleaseNote: String, bumpMajorVersion case "terra-jupyter-gatk" => List( "terra-jupyter-gatk" ) - case "terra-jupyter-gatk-ovtf" => List( - "terra-jupyter-gatk-ovtf" - ) case updatedImage => throw new Exception(s"${updatedImage} is not supported yet. Please update the script to support the image") }