workflows.yml

---
name: "CI build and test jobs"

on:
  pull_request:
    branches-ignore:
      - /doc\/.*/
  push:
    branches:
      - dev
      - maint/*
    tags:
      - '*'
  schedule:
    # 7am EST / 8am EDT, daily
    - cron: '0 12 * * *'

env:
  REGISTRY: ghcr.io
  IMAGE_NAME: ${{ github.repository }}
  INSTALL_TYPE: pip

defaults:
  run:
    shell: bash -l {0}

jobs:
  build:
    runs-on: ubuntu-latest
    steps:
      - name: Setup python
        uses: actions/setup-python@v4
        with:
          python-version: 3.9

      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          submodules: recursive

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Add version to environment
        run: |
          python -c 'import fitlins; print(f"VERSION={fitlins.__version__}")' >> $GITHUB_ENV

      - name: Check version matches tag
        if: startsWith(github.ref, 'refs/tags/')
        run: test "$VERSION" = "${{ github.ref_name }}"

      - name: Build image and export
        timeout-minutes: 30
        uses: docker/build-push-action@v5
        with:
          context: .
          build-args: |
            BUILD_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")"
            VCS_REF="$(git rev-parse --short HEAD)"
            VERSION=${{ env.VERSION }}
          push: false
          tags: |
            ${{ env.IMAGE_NAME }}:latest
            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
          load: true

      - name: Verify executable can be run
        run: |
          docker images
          docker run --rm $IMAGE_NAME:latest --help

      - name: Check version
        run: |
          DOCKERVERSION=$(docker run --rm $IMAGE_NAME:latest --version \
                           | tail -n 1 | sed -e 's/.*fit/fit/' -e 's/[\r\n]//g')
          echo "$VERSION"
          echo "$DOCKERVERSION"
          test "$DOCKERVERSION" = "fitlins v$VERSION"

      - name: Export docker image to a tar file
        run: |
          docker save ${{ github.repository }} > /tmp/docker.tar
          ls -l /tmp/

      - name: Upload docker.tar as artifact
        uses: actions/upload-artifact@v3
        with:
          name: docker
          path: /tmp/docker.tar

  cache_test_data:
    runs-on: ubuntu-latest
    steps:
      - name: "Make cachedir"
        run: |
          mkdir -p /tmp/.cache/data && \
          chmod 777 /tmp/.cache/data

      - name: Set git ID
        run: |
          git config --global user.name 'Fitlins GH-Action User' && \
          git config --global user.email 'shashankbansal56@gmail.com'

      - name: Set up Python 3.9
        uses: actions/setup-python@v4
        with:
          python-version: 3.9

      - name: Install dependencies
        shell: bash -l {0}
        run: |
          sudo apt-get update -y
          sudo apt-get install -y git-annex
          python -m pip install --upgrade pip
          pip install datalad==0.14.7

      - name: Datalad WTF
        run: |
         datalad wtf

      - name: Install cached data
        uses: actions/cache@v3
        id: cache
        with:
          path: /tmp/.cache/data
          key: ds003-v5-${{ github.ref_name }}-${{ github.sha }}
          restore-keys: |
            ds003-v5-${{ github.ref_name }}-
            ds003-v5-

      - name: "Install fMRIPrep derivatives of ds000003 and reference data"
        # if: steps.cache.outputs.cache-hit != 'true'
        run: |
          cd /tmp/.cache/data
          datalad install -r -s https://gin.g-node.org/markiewicz/fitlins-tests
          datalad update --merge -d fitlins-tests
          cd fitlins-tests
          datalad get inputs/ds000003-fmriprep/sub-0{1,2,3}/func/*_space-MNI152NLin2009cAsym_desc-*.nii.gz \
                      inputs/ds000003-fmriprep/sub-0{1,2,3}/func/*_desc-confounds_*.tsv \
                      inputs/ds000003-fmriprep/dataset_description.json \
                      inputs/ds000003-fmriprep/sub-*/*/*.json \
                      outputs
          datalad status

      - name: "Check installed derivatives data"
        run: |
          cd /tmp/.cache/data/fitlins-tests/
          git log --oneline --graph
          du -sh .

  run_pytest:
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: ['ubuntu-latest']
        python-version: ["3.7", "3.8", "3.9", "3.10"]
        pip-flags: ['']
        check: ['test']
    env:
      CHECK_TYPE: ${{ matrix.check }}
      EXTRA_PIP_FLAGS: ${{ matrix.pip-flags }}
      OS_TYPE: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: recursive
          fetch-depth: 0
      - uses: conda-incubator/setup-miniconda@v2
        with:
          miniconda-version: "latest"
          auto-update-conda: true
          activate-environment: fitlins-ci
          environment-file: tools/ci/conda_environment.yml
          python-version: ${{ matrix.python-version }}
          show-channel-urls: true
          auto-activate-base: false
      - name: Test conda installation
        run: |
          conda info
          conda list
          conda env list
          conda config --show-sources
          conda config --show
      - name: Install dependencies
        run: |
          conda install -y --channel leej3 "afni-minimal"
          conda list | grep afni
      - name: Display Python version
        run: |
          which python
          which pip
          python -c "import sys; print(sys.version)"
      - name: Install fitlins
        run: tools/ci/install.sh
      - name: Run tests
        run: tools/ci/check.sh
      - uses: codecov/codecov-action@v3
        with:
          verbose: true
        if: ${{ always() }}

  pre-release:
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: ['ubuntu-latest']
        python-version: ['3.10']
        # Check pre-release pybids and pre-release everything
        pip-flags: ['', 'PRE_PIP_FLAGS']
        check: ['test']
    env:
      CHECK_TYPE: ${{ matrix.check }}
      EXTRA_PIP_FLAGS: ${{ matrix.pip-flags }}
      OS_TYPE: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: recursive
          fetch-depth: 0
      - uses: conda-incubator/setup-miniconda@v2
        with:
          activate-environment: fitlins-ci
          environment-file: tools/ci/conda_environment.yml
          python-version: ${{ matrix.python-version }}
      - name: Test conda installation
        run: |
          conda info
          conda list
          conda env list
          conda config --show-sources
          conda config --show
      - name: Install dependencies
        run: |
          conda install -y --channel leej3 "afni-minimal"
          conda list | grep afni
      - name: Display Python version
        run: |
          which python
          which pip
          python -c "import sys; print(sys.version)"
      - name: Install fitlins
        run: tools/ci/install.sh
      - name: Install pybids from GitHub
        run: pip install git+https://github.com/bids-standard/pybids.git
      - name: Run tests
        run: tools/ci/check.sh
      - uses: codecov/codecov-action@v3
        with:
          verbose: true
        if: ${{ always() }}

  test_ds003_nistats_smooth:
    needs: [cache_test_data]
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        python-version: [3.9]
        test_name: ["nistats_smooth"]
        pip-flags: ['']
        check: ['workflow']
    env:
      CHECK_TYPE: ${{ matrix.check }}
      EXTRA_PIP_FLAGS: ${{ matrix.pip-flags }}
      OS_TYPE: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: recursive
          fetch-depth: 0
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install fitlins
        run: tools/ci/install.sh

      - name: Restore cached test data
        uses: actions/cache@v3
        id: cache
        with:
          path: /tmp/.cache/data
          key: ds003-v5-${{ github.ref_name }}-${{ github.sha }}
          
      - name: Run fitlins tests
        timeout-minutes: 240
        run: |
          mkdir -p $WORK_DIR $OUTPUT_DIR
          chmod 777 $WORK_DIR $OUTPUT_DIR
          tools/ci/check.sh
        env:
          DATA: /tmp/.cache/data/fitlins-tests
          TEST_NAME: ${{ matrix.test_name }}
          OUTPUT_DIR: /tmp/ds003/derivatives
          WORK_DIR: /tmp/ds003/work
      - name: Combine coverage and submit
        uses: codecov/codecov-action@v3
        with:
          verbose: true

  test_ds003_afni:
    needs: [cache_test_data]
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        python-version: [3.9]
        test_name: ["afni_smooth", "afni_blurto"]
        pip-flags: ['']
        check: ['workflow']
    env:
      CHECK_TYPE: ${{ matrix.check }}
      EXTRA_PIP_FLAGS: ${{ matrix.pip-flags }}
      OS_TYPE: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: recursive
          fetch-depth: 0
      - uses: conda-incubator/setup-miniconda@v2
        with:
          miniconda-version: "latest"
          auto-update-conda: true
          activate-environment: fitlins-ci
          environment-file: tools/ci/conda_environment.yml
          python-version: ${{ matrix.python-version }}
          show-channel-urls: true
          auto-activate-base: false
      - name: Test conda installation
        run: |
          conda info
          conda list
          conda env list
      - name: Install dependencies
        run: |
          conda install -y --channel leej3 "afni-minimal"
          conda list | grep afni
      - name: Install fitlins
        run: tools/ci/install.sh

      - name: Restore cached test data
        uses: actions/cache@v3
        id: cache
        with:
          path: /tmp/.cache/data
          key: ds003-v5-${{ github.ref_name }}-${{ github.sha }}
          
      - name: Run fitlins tests
        timeout-minutes: 240
        run: |
          mkdir -p $WORK_DIR $OUTPUT_DIR
          chmod 777 $WORK_DIR $OUTPUT_DIR
          tools/ci/check.sh
        env:
          DATA: /tmp/.cache/data/fitlins-tests
          TEST_NAME: ${{ matrix.test_name }}
          OUTPUT_DIR: /tmp/ds003/derivatives
          WORK_DIR: /tmp/ds003/work
      - name: Combine coverage and submit
        uses: codecov/codecov-action@v3
        with:
          verbose: true

  test_ds003_nistats_blurto:
    needs: [build, cache_test_data]
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        test_name: ["nistats_blurto"]
    steps:
      - uses: actions/checkout@v4

      - name: Set up docker buildx
        uses: docker/setup-buildx-action@v3

      - name: Download artifact
        uses: actions/download-artifact@v3
        with:
          name: docker
          path: /tmp

      - name: Load docker image
        run: |
          docker info
          set +o pipefail
          ls -l /tmp/docker.tar
          docker load < /tmp/docker.tar
          docker images

      - name: Restore cached test data
        uses: actions/cache@v3
        id: cache
        with:
          path: /tmp/.cache/data
          key: ds003-v5-${{ github.ref_name }}-${{ github.sha }}

      - name: Run fitlins tests
        timeout-minutes: 240
        run: |
          mkdir -p $WORK_DIR $OUTPUT_DIR
          chmod 777 $WORK_DIR $OUTPUT_DIR
          docker run --rm \
              -v $DATA:$DATA \
              -v $WORK_DIR:$WORK_DIR \
              -v $OUTPUT_DIR:$OUTPUT_DIR \
              -v $PWD:/work \
              -e CHECK_TYPE -e TEST_NAME \
              -e DATA -e WORK_DIR -e OUTPUT_DIR \
              -u $UID \
              --entrypoint=/neurodocker/startup.sh \
              $IMAGE tools/ci/check.sh
        env:
          IMAGE: ${{ github.repository }}:latest
          CHECK_TYPE: workflow
          DATA: /tmp/.cache/data/fitlins-tests
          TEST_NAME: ${{ matrix.test_name }}
          OUTPUT_DIR: /tmp/ds003/derivatives
          WORK_DIR: /tmp/ds003/work

      - name: Combine coverage and submit
        uses: codecov/codecov-action@v3
        with:
          verbose: true

  deploy_docker:
    needs: [test_ds003_afni, test_ds003_nistats_smooth, test_ds003_nistats_blurto]
    if: github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags/')
    runs-on: ubuntu-latest
    steps:
      - name: Set up docker buildx
        uses: docker/setup-buildx-action@v3

      - name: Download artifact
        uses: actions/download-artifact@v3
        with:
          name: docker
          path: /tmp

      - name: Load docker image
        run: |
          docker info
          set +o pipefail
          ls -l /tmp/docker.tar
          docker load < /tmp/docker.tar
          docker images

      - name: Set version/dev tags
        run: |
          # GHCR
          docker tag $IMAGE_NAME $REGISTRY/$IMAGE_NAME:${{ github.ref_name }}
          # DockerHub
          docker tag $IMAGE_NAME $IMAGE_NAME:${{ github.ref_name }}

      - name: Check images
        run: docker images

      - name: Login to Github Container Registry
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.repository_owner }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Deploy to GHCR
        timeout-minutes: 60
        run: docker push $REGISTRY/$IMAGE_NAME:${{ github.ref_name }}

      - name: Login to DockerHub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}

      - name: Deploy to DockerHub
        timeout-minutes: 60
        run: docker push $IMAGE_NAME:${{ github.ref_name }}

      - name: Deploy ":latest" to DockerHub
        timeout-minutes: 10
        if: startsWith(github.ref, 'refs/tags/')
        run: docker push $IMAGE_NAME:latest