From 2d28ad05013a597c31bcac10339c10c6d1118d69 Mon Sep 17 00:00:00 2001 From: Chuck Tang Date: Wed, 25 Sep 2024 15:11:10 -0700 Subject: [PATCH 1/5] commit change --- .github/workflows/docker.yaml | 80 +++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 .github/workflows/docker.yaml diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml new file mode 100644 index 00000000..ecdbc036 --- /dev/null +++ b/.github/workflows/docker.yaml @@ -0,0 +1,80 @@ +name: Docker +on: + push: + branches: + - main + pull_request: + branches: + - main + paths: + - Dockerfile + - .github/workflows/docker.yaml + workflow_dispatch: {} +jobs: + docker-build: + runs-on: mosaic-8wide + if: github.repository_owner == 'mosaicml' + strategy: + matrix: + include: + - name: "2.4.0_cu124" + base_image: mosaicml/pytorch:2.4.0_cu124-python3.11-ubuntu20.04 + dep_groups: "[all]" + - name: "2.4.0_cu124_aws" + base_image: mosaicml/pytorch:2.4.0_cu124-python3.11-ubuntu20.04 + dep_groups: "[all]" + steps: + + - name: Checkout + uses: actions/checkout@v3 + + - name: Setup QEMU + uses: docker/setup-qemu-action@v2 + + - name: Setup Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_PASSWORD }} + + - name: Calculate Docker Image Variables + run: | + set -euxo pipefail + + ################### + # Calculate the tag + ################### + GIT_SHA=$(echo ${{ github.sha }} | cut -c1-7) + echo "IMAGE_TAG=${GIT_SHA}" >> ${GITHUB_ENV} + + if [ "${{ github.event_name }}" == "pull_request" ]; then + echo "Triggered by pull_request event." + STAGING_REPO="mosaicml/ci-staging" + IMAGE_TAG="${STAGING_REPO}:${{matrix.name}}-${GIT_SHA}" + IMAGE_CACHE="${STAGING_REPO}:${{matrix.name}}-buildcache" + else + # Triggered by push or workflow_dispatch event + echo "Triggered by ${{ github.event_name }} event, releasing to prod" + PROD_REPO="mosaicml/diffusion" + IMAGE_TAG="${PROD_REPO}:${{matrix.name}}-${GIT_SHA},${PROD_REPO}:${{matrix.name}}-latest" + IMAGE_CACHE="${PROD_REPO}:${{matrix.name}}-buildcache" + fi + + echo "IMAGE_TAG=${IMAGE_TAG}" >> ${GITHUB_ENV} + echo "IMAGE_CACHE=${IMAGE_CACHE}" >> ${GITHUB_ENV} + + - name: Build and Push the Docker Image + uses: docker/build-push-action@v3 + with: + context: . + tags: ${{ env.IMAGE_TAG }} + push: true + cache-from: type=registry,ref=${{ env.IMAGE_CACHE }} + cache-to: type=registry,ref=${{ env.IMAGE_CACHE }},mode=max + build-args: | + BRANCH_NAME=${{ github.head_ref || github.ref_name }} + BASE_IMAGE=${{ matrix.base_image }} + DEP_GROUPS=${{ matrix.dep_groups }} From 8c329824d4313ee721569e813412739c21c88662 Mon Sep 17 00:00:00 2001 From: Chuck Tang Date: Wed, 25 Sep 2024 16:34:52 -0700 Subject: [PATCH 2/5] commit change --- .github/workflows/docker.yaml | 10 ++++++++-- Dockerfile | 21 +++++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) create mode 100644 Dockerfile diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index ecdbc036..164b5ccb 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -17,14 +17,20 @@ jobs: strategy: matrix: include: + - name: "2.1.2_cu121" + base_image: mosaicml/pytorch:2.1.2_cu121-python3.10-ubuntu20.04 + dep_groups: "[all]" + - name: "2.1.2_cu121_aws" + base_image: mosaicml/pytorch:2.1.2_cu121-python3.10-ubuntu20.04-aws + dep_groups: "[all]" - name: "2.4.0_cu124" base_image: mosaicml/pytorch:2.4.0_cu124-python3.11-ubuntu20.04 dep_groups: "[all]" - name: "2.4.0_cu124_aws" - base_image: mosaicml/pytorch:2.4.0_cu124-python3.11-ubuntu20.04 + base_image: mosaicml/pytorch:2.4.0_cu124-python3.11-ubuntu20.04-aws dep_groups: "[all]" steps: - + - name: Checkout uses: actions/checkout@v3 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..2496ddb8 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,21 @@ +# Copyright 2022 MosaicML Diffusion authors +# SPDX-License-Identifier: Apache-2.0 + +ARG BASE_IMAGE +FROM $BASE_IMAGE + +ARG BRANCH_NAME +ARG DEP_GROUPS + +ENV TORCH_CUDA_ARCH_LIST="8.0 8.6 8.7 8.9 9.0" + +# Check for changes in setup.py. +# If there are changes, the docker cache is invalidated and a fresh pip installation is triggered. +ADD https://raw.githubusercontent.com/mosaicml/diffusion/$BRANCH_NAME/setup.py setup.py +RUN rm setup.py + +# Install and uninstall diffusion to cache diffusion requirements +RUN git clone -b $BRANCH_NAME https://github.com/mosaicml/diffusion.git +RUN pip install --no-cache-dir "./diffusion${DEP_GROUPS}" +RUN pip uninstall -y diffusion +RUN rm -rf diffusion From a496525ddfc4495e560df0b2723da48aacbf88f0 Mon Sep 17 00:00:00 2001 From: Chuck Tang Date: Wed, 25 Sep 2024 18:09:10 -0700 Subject: [PATCH 3/5] commit change --- .github/workflows/coverage.yaml | 31 +++++++++++-------------------- 1 file changed, 11 insertions(+), 20 deletions(-) diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml index 724497b1..2629c453 100644 --- a/.github/workflows/coverage.yaml +++ b/.github/workflows/coverage.yaml @@ -10,23 +10,14 @@ jobs: timeout-minutes: 5 runs-on: ubuntu-latest steps: - - name: Checkout Repo - uses: actions/checkout@v3 - - name: Setup - run: | - set -ex - python -m pip install --upgrade 'pip<23' wheel - pip install coverage[toml]==6.5.0 - - name: Download artifacts - uses: actions/download-artifact@v3 - with: - path: ${{ inputs.download-path }} - - name: Generate coverage report - run: | - set -ex - - # Flatten the coverage files - ls ${{ inputs.download-path }} | while read x; do mv ${{ inputs.download-path }}/$x/.coverage .coverage.$x; done - - python -m coverage combine - python -m coverage report + - name: Checkout Repo + uses: actions/checkout@v3 + - name: Get composite run steps repository + uses: actions/checkout@v3 + with: + repository: mosaicml/ci-testing + ref: v0.2.2 + path: ./ci-testing + - uses: ./ci-testing/.github/actions/coverage + with: + download-path: ${{ inputs.download-path }} \ No newline at end of file From d0789cf05eec58183ff4a9c48c8339d58a9fe7f4 Mon Sep 17 00:00:00 2001 From: Chuck Tang Date: Thu, 26 Sep 2024 11:11:43 -0700 Subject: [PATCH 4/5] commit change --- .github/workflows/coverage.yaml | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml index 2629c453..724497b1 100644 --- a/.github/workflows/coverage.yaml +++ b/.github/workflows/coverage.yaml @@ -10,14 +10,23 @@ jobs: timeout-minutes: 5 runs-on: ubuntu-latest steps: - - name: Checkout Repo - uses: actions/checkout@v3 - - name: Get composite run steps repository - uses: actions/checkout@v3 - with: - repository: mosaicml/ci-testing - ref: v0.2.2 - path: ./ci-testing - - uses: ./ci-testing/.github/actions/coverage - with: - download-path: ${{ inputs.download-path }} \ No newline at end of file + - name: Checkout Repo + uses: actions/checkout@v3 + - name: Setup + run: | + set -ex + python -m pip install --upgrade 'pip<23' wheel + pip install coverage[toml]==6.5.0 + - name: Download artifacts + uses: actions/download-artifact@v3 + with: + path: ${{ inputs.download-path }} + - name: Generate coverage report + run: | + set -ex + + # Flatten the coverage files + ls ${{ inputs.download-path }} | while read x; do mv ${{ inputs.download-path }}/$x/.coverage .coverage.$x; done + + python -m coverage combine + python -m coverage report From bc3c4cdacddd76ffc68d1f5a4fb0fc17ec61470a Mon Sep 17 00:00:00 2001 From: Chuck Tang Date: Thu, 26 Sep 2024 11:30:51 -0700 Subject: [PATCH 5/5] commit change --- .github/workflows/pytest-cpu.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pytest-cpu.yaml b/.github/workflows/pytest-cpu.yaml index 5a39cad7..348844e7 100644 --- a/.github/workflows/pytest-cpu.yaml +++ b/.github/workflows/pytest-cpu.yaml @@ -62,3 +62,4 @@ jobs: with: name: coverage-${{ github.sha }}-${{ inputs.name }} path: .coverage + include-hidden-files: true