From 6d7b21273279cb8c75cb6deb8f009630813b25a4 Mon Sep 17 00:00:00 2001 From: Jennings Zhang Date: Tue, 2 Feb 2021 00:25:48 -0500 Subject: [PATCH 01/12] Refactor - change base image to python:3.9.1-buster - remove pudb - don't use sys.exit - add requirements.txt --- Dockerfile | 49 +++++++----------------------------------------- lungct/lungct.py | 13 +++++-------- requirements.txt | 1 + setup.py | 2 +- 4 files changed, 14 insertions(+), 51 deletions(-) create mode 100644 requirements.txt diff --git a/Dockerfile b/Dockerfile index b6dacab..72f25e8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,50 +1,15 @@ -# Docker file for lungct ChRIS plugin app -# -# Build with -# -# docker build -t . -# -# For example if building a local version, you could do: -# -# docker build --build-arg UID=$UID -t local/pl-lungct . -# -# In the case of a proxy (located at 192.168.13.14:3128), do: -# -# export PROXY=http://192.168.13.14:3128 -# docker build --build-arg http_proxy=$PROXY --build-arg UID=$UID -t local/pl-lungct . -# -# To run an interactive shell inside this container, do: -# -# docker run -ti --entrypoint /bin/bash local/pl-lungct -# -# To pass an env var HOST_IP to container, do: -# -# docker run -ti -e HOST_IP=$(ip route | grep -v docker | awk '{if(NF==11) print $9}') --entrypoint /bin/bash local/pl-lungct -# -# To debug from within a container: -# -# docker run -ti -v $(pwd)/lungct:/usr/local/lib/python3.8/dist-packages/lungct -v $(pwd)/out:/outgoing local/pl-lungct lungct /outgoing -# - -FROM fnndsc/ubuntu-python3:latest +FROM python:3.9.1-buster LABEL MAINTAINER="dev@babymri.org" -# Pass a UID on build command line (see above) to set internal UID -ARG UID=1001 -ENV UID=$UID DEBIAN_FRONTEND=noninteractive - WORKDIR /usr/local/src + +COPY requirements.txt . +RUN ["pip", "install", "-r", "requirements.txt"] + COPY . . -RUN pip --disable-pip-version-check install . \ - && useradd -u $UID -ms /bin/bash localuser \ - && apt update \ - && apt-get install -y sudo \ - && echo "localuser:localuser" | chpasswd \ - && addgroup localuser sudo \ - && echo "localuser ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers +RUN ["pip", "install", "."] -# Start as user localuser -# USER localuser +WORKDIR /usr/local/bin WORKDIR /usr/local/bin CMD ["lungct", "--help"] diff --git a/lungct/lungct.py b/lungct/lungct.py index d110e82..74d531c 100755 --- a/lungct/lungct.py +++ b/lungct/lungct.py @@ -16,7 +16,6 @@ from os.path import abspath, basename, isdir from distutils.dir_util import copy_tree import shutil -import pudb import sys import time import glob @@ -175,13 +174,11 @@ def run(self, options): print(Gstr_title) print('Version: %s' % self.get_version()) - if len(options.dir): - print("Copying tree %s..." % options.dir) - copy_tree(options.dir, options.outputdir) - sys.exit(0) - else: - print("No directory specified and no copy performed.") - sys.exit(1) + if not isdir(options.dir): + raise ValueError("No directory specified and no copy performed.") + + print("Copying tree %s..." % options.dir) + copy_tree(options.dir, options.outputdir) def show_man_page(self): """ diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..f01c051 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +chrisapp~=2.0.0 diff --git a/setup.py b/setup.py index 338a8b6..0261c9f 100755 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ author_email = 'dev@babyMRI.org', url = 'http://wiki', packages = ['lungct', 'data'], - install_requires = ['chrisapp~=2.0.0', 'pudb'], + install_requires = ['chrisapp'], test_suite = 'nose.collector', tests_require = ['nose'], license = 'MIT', From 43a09e3fab2a43ea6632ed4936a40b05aff3856d Mon Sep 17 00:00:00 2001 From: Jennings Zhang Date: Tue, 2 Feb 2021 00:26:36 -0500 Subject: [PATCH 02/12] Add Github Actions --- .github/workflows/ci.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..00eb412 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,24 @@ +on: [push, pull_request, release] + +jobs: + test: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - name: build + run: docker build -t "${GITHUB_REPOSITORY,,}" . + - name: nose tests + run: docker run -w /usr/local/src --entrypoint /bin/sh "${GITHUB_REPOSITORY,,}" -c 'pip install nose && nosetests' + + publish: + runs-on: ubuntu-20.04 + needs: [test] + if: github.event_name == 'push' || github.event_name == 'release' + steps: + - uses: fnndsc/chrisstore-build-and-publish@v1 + with: + description: "A lung CT scan data pack." + docker_username: ${{ secrets.DOCKERHUB_USERNAME }} + docker_password: ${{ secrets.DOCKERHUB_PASSWORD }} + chrisstore_token: ${{ secrets.CHRIS_STORE_USER }} + readme-filepath: ./README.rst From 17b3edfeeef8cbb012de99d0a16e031fa56b9ae2 Mon Sep 17 00:00:00 2001 From: Jennings Zhang Date: Tue, 2 Feb 2021 10:17:59 -0500 Subject: [PATCH 03/12] Add everything --- .github/workflows/ci.yml | 95 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 90 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 00eb412..16e6b22 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,11 +14,96 @@ jobs: runs-on: ubuntu-20.04 needs: [test] if: github.event_name == 'push' || github.event_name == 'release' + + # we want to both push the build to DockerHub, but also + # keep a local copy so that we can run + # + # docker run fnndsc/pl-app app --json > App.json + # + # buildx currently does not support multiple output locations, + # neither can multi-architectural builds be loaded into docker. + # Here we use a local registry to cache the build. + services: + registry: + image: registry:2 + ports: + - 5000:5000 steps: - - uses: fnndsc/chrisstore-build-and-publish@v1 + - name: Get git tag + id: git_info + if: startsWith(github.ref, 'refs/tags/') + run: echo "::set-output name=tag::${GITHUB_REF##*/}" + - name: Decide image tag name + id: determine + env: + git_tag: ${{ steps.git_info.outputs.tag }} + run: | + dock_image="${GITHUB_REPOSITORY,,}" # to lower case + # if build triggered by tag, use tag name + tag="${git_tag:-latest}" + echo "::set-output name=tag::$dock_image:$tag" + + - uses: actions/checkout@v2 + + # QEMU is for emulating non-x86_64 platforms + - uses: docker/setup-qemu-action@v1 + # buildx is the next-generation docker image builder + - uses: docker/setup-buildx-action@v1 + with: + driver-opts: network=host + - name: Cache Docker layers + uses: actions/cache@v2 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + + - name: Login to DockerHub + id: dockerhub_login + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{secrets.DOCKERHUB_PASSWORD}} + + - name: Build and push + uses: docker/build-push-action@v2 + id: docker_build with: - description: "A lung CT scan data pack." - docker_username: ${{ secrets.DOCKERHUB_USERNAME }} - docker_password: ${{ secrets.DOCKERHUB_PASSWORD }} - chrisstore_token: ${{ secrets.CHRIS_STORE_USER }} + context: . + file: ./Dockerfile + tags: | + ${{ steps.determine.outputs.tag }} + localhost:5000/${{ steps.determine.outputs.tag }} + platforms: linux/amd64,linux/ppc64le,linux/arm64 + push: true + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache + + - name: Update DockerHub description + uses: peter-evans/dockerhub-description@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{secrets.DOCKERHUB_PASSWORD}} + short-description: "Trying pl-lungct from Github Actions" readme-filepath: ./README.rst + + - name: Upload to ChRIS Store + run: | + dock_image=${{ steps.determine.outputs.tag }} + docker pull localhost:5000/$dock_image + docker tag localhost:5000/$dock_image $dock_image + docker rmi localhost:5000/$dock_image + script=$(docker inspect --format '{{ (index .Config.Cmd 0) }}' $dock_image) + descriptor_file=$(mktemp --suffix .json) + docker run --rm $dock_image $script --json > $descriptor_file + res=$( + curl -s -u "${{ secrets.CHRIS_STORE_USER }}" "https://chrisstore.co/api/v1/plugins/" \ + -F "name=$(sed 's/^.*\///' <<< $dock_image)" \ + -F "dock_image=$dock_image" \ + -F "descriptor_file=@$descriptor_file" \ + -F "public_repo=https://github.com/${{ github.repository }}" + ) + href="$(jq -r '.collection.items[0].href' <<< "$res")" + echo $href + echo "::set-output name=pluginurl::$href" From 4e64fabc61ba227da54c2c8e5b2977438ba1d731 Mon Sep 17 00:00:00 2001 From: Jennings Zhang Date: Tue, 2 Feb 2021 10:25:50 -0500 Subject: [PATCH 04/12] Print tag to use --- .github/workflows/ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 16e6b22..fce9c7d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,7 +41,9 @@ jobs: dock_image="${GITHUB_REPOSITORY,,}" # to lower case # if build triggered by tag, use tag name tag="${git_tag:-latest}" - echo "::set-output name=tag::$dock_image:$tag" + dock_image=$dock_image:$tag + echo $dock_image + echo "::set-output name=tag::$dock_image" - uses: actions/checkout@v2 From d5b7c1c1865589824429f57e1b372f2e13ce6705 Mon Sep 17 00:00:00 2001 From: Jennings Zhang Date: Tue, 2 Feb 2021 10:34:42 -0500 Subject: [PATCH 05/12] Don't push latest to ChRIS Store --- .github/workflows/ci.yml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fce9c7d..ab9fb58 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -91,6 +91,7 @@ jobs: readme-filepath: ./README.rst - name: Upload to ChRIS Store + if: ! endsWith(steps.determine.outputs.tag, ':latest') run: | dock_image=${{ steps.determine.outputs.tag }} docker pull localhost:5000/$dock_image @@ -106,6 +107,13 @@ jobs: -F "descriptor_file=@$descriptor_file" \ -F "public_repo=https://github.com/${{ github.repository }}" ) - href="$(jq -r '.collection.items[0].href' <<< "$res")" - echo $href - echo "::set-output name=pluginurl::$href" + success=$? + if [ "$success" = "0" ]; then + href="$(jq -r '.collection.items[0].href' <<< "$res")" + echo $href + echo "::set-output name=pluginurl::$href" + else + echo "Failed to upload to ChRIS Store" + echo "$res" + exit $success + fi From f44e50b01712fd2a29f21d217c9a6b40a263f35d Mon Sep 17 00:00:00 2001 From: Jennings Zhang Date: Tue, 2 Feb 2021 10:37:23 -0500 Subject: [PATCH 06/12] Quote negation --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ab9fb58..91b4935 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -91,7 +91,7 @@ jobs: readme-filepath: ./README.rst - name: Upload to ChRIS Store - if: ! endsWith(steps.determine.outputs.tag, ':latest') + if: "!endsWith(steps.determine.outputs.tag, ':latest')" run: | dock_image=${{ steps.determine.outputs.tag }} docker pull localhost:5000/$dock_image From a7d0f6d008b7309ee238e72b85e7715755d1a5cd Mon Sep 17 00:00:00 2001 From: Jennings Zhang Date: Wed, 3 Feb 2021 16:46:45 -0500 Subject: [PATCH 07/12] Change to slim image --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 72f25e8..b7277d2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9.1-buster +FROM python:3.9.1-slim-buster LABEL MAINTAINER="dev@babymri.org" WORKDIR /usr/local/src From d78732b9dbb6efed856b90ac4dd07186a19b4553 Mon Sep 17 00:00:00 2001 From: Jennings Zhang Date: Thu, 4 Feb 2021 15:54:18 -0500 Subject: [PATCH 08/12] Try getting plugin title dynamically --- .github/workflows/ci.yml | 81 +++++++++++++++++++++++++++++----------- Dockerfile | 3 -- 2 files changed, 59 insertions(+), 25 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 91b4935..5483961 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,4 +1,25 @@ -on: [push, pull_request, release] +# Automatically build multi-architectural tagged container images and push them to DockerHub +# https://github.com/FNNDSC/cookiecutter-chrisapp/wiki/Automatic-Builds +# +# - targeted platforms: x86_64, PowerPC64, ARM64 +# - master is built as fnndsc/pl-lungct:latest +# - tagged commits are built as fnndsc/pl-lungct: +# - tagged commits are also uploaded to chrisstore.co +# +# In order to use this workflow, see +# https://github.com/FNNDSC/cookiecutter-chrisapp/wiki/Automatic-Builds#steps-to-enable + +name: ci + +on: + push: + # we have to guess what the name of the default branch is + branches: [ master, main, trunk ] + tags: [ '**' ] + pull_request: + branches: [ master, main, trunk ] + release: + types: [ created ] jobs: test: @@ -8,17 +29,16 @@ jobs: - name: build run: docker build -t "${GITHUB_REPOSITORY,,}" . - name: nose tests - run: docker run -w /usr/local/src --entrypoint /bin/sh "${GITHUB_REPOSITORY,,}" -c 'pip install nose && nosetests' + run: docker run "${GITHUB_REPOSITORY,,}" nosetests publish: - runs-on: ubuntu-20.04 - needs: [test] if: github.event_name == 'push' || github.event_name == 'release' + runs-on: ubuntu-20.04 # we want to both push the build to DockerHub, but also # keep a local copy so that we can run # - # docker run fnndsc/pl-app app --json > App.json + # docker run fnndsc/pl-lungct lungct --json > App.json # # buildx currently does not support multiple output locations, # neither can multi-architectural builds be loaded into docker. @@ -28,6 +48,7 @@ jobs: image: registry:2 ports: - 5000:5000 + steps: - name: Get git tag id: git_info @@ -38,12 +59,13 @@ jobs: env: git_tag: ${{ steps.git_info.outputs.tag }} run: | - dock_image="${GITHUB_REPOSITORY,,}" # to lower case + repo="${GITHUB_REPOSITORY,,}" # to lower case # if build triggered by tag, use tag name tag="${git_tag:-latest}" - dock_image=$dock_image:$tag + dock_image=$repo:$tag echo $dock_image - echo "::set-output name=tag::$dock_image" + echo "::set-output name=dock_image::$dock_image" + echo "::set-output name=repo::$repo" - uses: actions/checkout@v2 @@ -53,6 +75,7 @@ jobs: - uses: docker/setup-buildx-action@v1 with: driver-opts: network=host + # save some time during rebuilds - name: Cache Docker layers uses: actions/cache@v2 with: @@ -66,7 +89,7 @@ jobs: uses: docker/login-action@v1 with: username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{secrets.DOCKERHUB_PASSWORD}} + password: ${{ secrets.DOCKERHUB_PASSWORD }} - name: Build and push uses: docker/build-push-action@v2 @@ -75,34 +98,47 @@ jobs: context: . file: ./Dockerfile tags: | - ${{ steps.determine.outputs.tag }} - localhost:5000/${{ steps.determine.outputs.tag }} + ${{ steps.determine.outputs.dock_image }} + localhost:5000/${{ steps.determine.outputs.dock_image }} platforms: linux/amd64,linux/ppc64le,linux/arm64 push: true cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache + - name: Get plugin meta + id: pluginmeta + run: | + repo=${{ steps.determine.outputs.repo }} + dock_image=${{ steps.determine.outputs.dock_image }} + docker pull localhost:5000/$dock_image + docker tag localhost:5000/$dock_image $dock_image + script=$(docker inspect --format '{{ (index .Config.Cmd 0) }}' $dock_image) + json="$(docker run --rm $dock_image $script --json)" + jq <<< "$json" # pretty print in log + + echo "::set-output name=json::$json" + echo "::set-output name=title::$(jq -r '.title' <<< "$json")" + - name: Update DockerHub description uses: peter-evans/dockerhub-description@v2 + continue-on-error: true # it is not crucial that this works with: username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{secrets.DOCKERHUB_PASSWORD}} - short-description: "Trying pl-lungct from Github Actions" + password: ${{ secrets.DOCKERHUB_PASSWORD }} + short-description: ${{ steps.pluginmeta.outputs.title }} readme-filepath: ./README.rst + repository: ${{ steps.determine.outputs.repo }} - name: Upload to ChRIS Store - if: "!endsWith(steps.determine.outputs.tag, ':latest')" + if: "!endsWith(steps.determine.outputs.dock_image, ':latest')" run: | - dock_image=${{ steps.determine.outputs.tag }} - docker pull localhost:5000/$dock_image - docker tag localhost:5000/$dock_image $dock_image - docker rmi localhost:5000/$dock_image - script=$(docker inspect --format '{{ (index .Config.Cmd 0) }}' $dock_image) + repo=${{ steps.determine.outputs.repo }} + dock_image=${{ steps.determine.outputs.dock_image }} descriptor_file=$(mktemp --suffix .json) - docker run --rm $dock_image $script --json > $descriptor_file + cat > $descriptor_file <<< "${{ steps.pluginmeta.outputs.json }}" res=$( curl -s -u "${{ secrets.CHRIS_STORE_USER }}" "https://chrisstore.co/api/v1/plugins/" \ - -F "name=$(sed 's/^.*\///' <<< $dock_image)" \ + -F "name=$(sed 's/^.*\///' <<< $repo)" \ -F "dock_image=$dock_image" \ -F "descriptor_file=@$descriptor_file" \ -F "public_repo=https://github.com/${{ github.repository }}" @@ -113,7 +149,8 @@ jobs: echo $href echo "::set-output name=pluginurl::$href" else - echo "Failed to upload to ChRIS Store" + echo "::error ::Failed upload to ChRIS Store" echo "$res" exit $success fi + diff --git a/Dockerfile b/Dockerfile index b7277d2..c9df57b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,4 @@ RUN ["pip", "install", "-r", "requirements.txt"] COPY . . RUN ["pip", "install", "."] -WORKDIR /usr/local/bin - -WORKDIR /usr/local/bin CMD ["lungct", "--help"] From 7225e5e73122ed58487029a92767f11389f6ccbc Mon Sep 17 00:00:00 2001 From: Jennings Zhang Date: Thu, 4 Feb 2021 15:56:20 -0500 Subject: [PATCH 09/12] Bump requirements.txt --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f01c051..093d3c6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ -chrisapp~=2.0.0 +chrisapp~=2.3.0 +nose~=1.3.7 From 42552dfb0a34bbf55c90dbcf0caf9f7c688b7d7b Mon Sep 17 00:00:00 2001 From: Jennings Zhang Date: Wed, 7 Jul 2021 02:08:16 -0400 Subject: [PATCH 10/12] Bump chrisapp!=3.0.0rc2 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 093d3c6..6a2c31b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ -chrisapp~=2.3.0 +chrisapp~=3.0.0rc2 nose~=1.3.7 From 5b02f839ad65fbee4fbf3aa672db405726b0d297 Mon Sep 17 00:00:00 2001 From: Jennings Zhang Date: Wed, 7 Jul 2021 02:10:21 -0400 Subject: [PATCH 11/12] Update README badges --- README.rst | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 777085c..462926b 100644 --- a/README.rst +++ b/README.rst @@ -1,11 +1,15 @@ pl-lungct ================================ -.. image:: https://travis-ci.org/FNNDSC/lungct.svg?branch=master - :target: https://travis-ci.org/FNNDSC/lungct +.. image:: https://img.shields.io/docker/v/fnndsc/pl-lungct?sort=semver + :target: https://hub.docker.com/r/fnndsc/pl-lungct + +.. image:: https://img.shields.io/github/license/fnndsc/pl-lungct + :target: https://github.com/FNNDSC/pl-lungct/blob/master/LICENSE + +.. image:: https://github.com/FNNDSC/pl-lungct/workflows/ci/badge.svg + :target: https://github.com/FNNDSC/pl-lungct/actions -.. image:: https://img.shields.io/badge/python-3.8%2B-blue.svg - :target: https://github.com/FNNDSC/pl-lungCT/blob/master/setup.py .. contents:: Table of Contents From 35ff33f1b2c8986b5493f4bcec5d8ef1c7be120b Mon Sep 17 00:00:00 2001 From: Jennings Zhang Date: Wed, 7 Jul 2021 02:10:49 -0400 Subject: [PATCH 12/12] Bump LICENSE --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index 4c9df85..2c69a66 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2017-2020 FNNDSC / BCH +Copyright (c) 2017-2021 FNNDSC / BCH Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal