From 9170191eccb65de2f17009f68952a18fc008fa6a Mon Sep 17 00:00:00 2001 From: NERDDISCO <492378+TimPietrusky@users.noreply.github.com> Date: Mon, 19 Aug 2024 13:29:53 +0200 Subject: [PATCH] feat: added FLUX.1 schnell & dev --- .github/workflows/dev.yml | 4 +- Dockerfile | 10 ++ README.md | 87 +++++++----- docker-bake.hcl | 27 +++- src/extra_model_paths.yaml | 1 + .../workflows/workflow_flux1_dev.json | 130 ++++++++++++++++++ .../workflows/workflow_flux1_schnell.json | 130 ++++++++++++++++++ 7 files changed, 354 insertions(+), 35 deletions(-) create mode 100644 test_resources/workflows/workflow_flux1_dev.json create mode 100644 test_resources/workflows/workflow_flux1_schnell.json diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 3731707..ede8516 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -8,7 +8,7 @@ on: jobs: dev: - runs-on: ubuntu-latest-m + runs-on: ubuntu-latest-l steps: - name: Checkout uses: actions/checkout@v3 @@ -55,4 +55,4 @@ jobs: *.args.DOCKERHUB_REPO=${{ env.DOCKERHUB_REPO }} *.args.DOCKERHUB_IMG=${{ env.DOCKERHUB_IMG }} *.args.RELEASE_VERSION=${{ env.RELEASE_VERSION }} - sd3.args.HUGGINGFACE_ACCESS_TOKEN=${{ env.HUGGINGFACE_ACCESS_TOKEN }} + *.args.HUGGINGFACE_ACCESS_TOKEN=${{ env.HUGGINGFACE_ACCESS_TOKEN }} diff --git a/Dockerfile b/Dockerfile index 25696b3..94236d3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -57,6 +57,16 @@ RUN if [ "$MODEL_TYPE" = "sdxl" ]; then \ wget -O models/vae/sdxl-vae-fp16-fix.safetensors https://huggingface.co/madebyollin/sdxl-vae-fp16-fix/resolve/main/sdxl_vae.safetensors; \ elif [ "$MODEL_TYPE" = "sd3" ]; then \ wget --header="Authorization: Bearer ${HUGGINGFACE_ACCESS_TOKEN}" -O models/checkpoints/sd3_medium_incl_clips_t5xxlfp8.safetensors https://huggingface.co/stabilityai/stable-diffusion-3-medium/resolve/main/sd3_medium_incl_clips_t5xxlfp8.safetensors; \ + elif [ "$MODEL_TYPE" = "flux1-schnell" ]; then \ + wget -O models/unet/flux1-schnell.safetensors https://huggingface.co/black-forest-labs/FLUX.1-schnell/resolve/main/flux1-schnell.safetensors && \ + wget -O models/clip/clip_l.safetensors https://huggingface.co/comfyanonymous/flux_text_encoders/resolve/main/clip_l.safetensors && \ + wget -O models/clip/t5xxl_fp8_e4m3fn.safetensors https://huggingface.co/comfyanonymous/flux_text_encoders/resolve/main/t5xxl_fp8_e4m3fn.safetensors && \ + wget -O models/vae/ae.safetensors https://huggingface.co/black-forest-labs/FLUX.1-schnell/resolve/main/ae.safetensors; \ + elif [ "$MODEL_TYPE" = "flux1-dev" ]; then \ + wget --header="Authorization: Bearer ${HUGGINGFACE_ACCESS_TOKEN}" -O models/unet/flux1-dev.safetensors https://huggingface.co/black-forest-labs/FLUX.1-dev/resolve/main/flux1-dev.safetensors && \ + wget -O models/clip/clip_l.safetensors https://huggingface.co/comfyanonymous/flux_text_encoders/resolve/main/clip_l.safetensors && \ + wget -O models/clip/t5xxl_fp8_e4m3fn.safetensors https://huggingface.co/comfyanonymous/flux_text_encoders/resolve/main/t5xxl_fp8_e4m3fn.safetensors && \ + wget --header="Authorization: Bearer ${HUGGINGFACE_ACCESS_TOKEN}" -O models/vae/ae.safetensors https://huggingface.co/black-forest-labs/FLUX.1-dev/resolve/main/ae.safetensors; \ fi # Stage 3: Final image diff --git a/README.md b/README.md index f8311dd..a1fed9a 100644 --- a/README.md +++ b/README.md @@ -19,27 +19,30 @@ Read our article here: https://blib.la/blog/comfyui-on-runpod - [Quickstart](#quickstart) - [Features](#features) - [Config](#config) - - [Upload image to AWS S3](#upload-image-to-aws-s3) + * [Upload image to AWS S3](#upload-image-to-aws-s3) - [Use the Docker image on RunPod](#use-the-docker-image-on-runpod) + * [Create your template (optional)](#create-your-template-optional) + * [Create your endpoint](#create-your-endpoint) + * [GPU recommendations](#gpu-recommendations) - [API specification](#api-specification) - - [JSON Request Body](#json-request-body) - - [Fields](#fields) - - ["input.images"](#inputimages) + * [JSON Request Body](#json-request-body) + * [Fields](#fields) + + ["input.images"](#inputimages) - [Interact with your RunPod API](#interact-with-your-runpod-api) - - [Health status](#health-status) - - [Generate an image](#generate-an-image) - - [Example request with cURL](#example-request-with-curl) + * [Health status](#health-status) + * [Generate an image](#generate-an-image) + + [Example request for SDXL with cURL](#example-request-for-sdxl-with-curl) - [How to get the workflow from ComfyUI?](#how-to-get-the-workflow-from-comfyui) - [Bring Your Own Models and Nodes](#bring-your-own-models-and-nodes) - - [Network Volume](#network-volume) - - [Custom Docker Image](#custom-docker-image) + * [Network Volume](#network-volume) + * [Custom Docker Image](#custom-docker-image) - [Local testing](#local-testing) - - [Setup](#setup) - - [Setup for Windows](#setup-for-windows) - - [Testing the RunPod handler](#testing-the-runpod-handler) - - [Local API](#local-api) - - [Access the local Worker API](#access-the-local-worker-api) - - [Access local ComfyUI](#access-local-comfyui) + * [Setup](#setup) + + [Setup for Windows](#setup-for-windows) + * [Testing the RunPod handler](#testing-the-runpod-handler) + * [Local API](#local-api) + + [Access the local Worker API](#access-the-local-worker-api) + + [Access local ComfyUI](#access-local-comfyui) - [Automatically deploy to Docker hub with GitHub Actions](#automatically-deploy-to-docker-hub-with-github-actions) - [Acknowledgments](#acknowledgments) @@ -49,15 +52,14 @@ Read our article here: https://blib.la/blog/comfyui-on-runpod ## Quickstart -- 🐳 Choose one of the three available images for your serverless endpoint: - - `timpietruskyblibla/runpod-worker-comfy:3.0.0-base`: doesn't contain any checkpoints, just a clean ComfyUI - - `timpietruskyblibla/runpod-worker-comfy:3.0.0-sdxl`: contains the checkpoints and VAE for Stable Diffusion XL - - Checkpoint: [sd_xl_base_1.0.safetensors](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0) - - VAEs: - - [sdxl_vae.safetensors](https://huggingface.co/stabilityai/sdxl-vae/) - - [sdxl-vae-fp16-fix](https://huggingface.co/madebyollin/sdxl-vae-fp16-fix/) - - `timpietruskyblibla/runpod-worker-comfy:3.0.0-sd3`: contains the [sd3_medium_incl_clips_t5xxlfp8.safetensors](https://huggingface.co/stabilityai/stable-diffusion-3-medium) checkpoint for Stable Diffusion 3 +- 🐳 Choose one of the five available images for your serverless endpoint: + - `timpietruskyblibla/runpod-worker-comfy:3.1.0-base`: doesn't contain anything, just a clean ComfyUI + - `timpietruskyblibla/runpod-worker-comfy:3.1.0-flux1-schnell`: contains the checkpoint, text encoders and VAE for [FLUX.1 schnell](https://huggingface.co/black-forest-labs/FLUX.1-schnell) + - `timpietruskyblibla/runpod-worker-comfy:3.1.0-flux1-dev`: contains the checkpoint, text encoders and VAE for [FLUX.1 dev](https://huggingface.co/black-forest-labs/FLUX.1-dev) + - `timpietruskyblibla/runpod-worker-comfy:3.1.0-sdxl`: contains the checkpoint and VAE for [Stable Diffusion XL](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0) + - `timpietruskyblibla/runpod-worker-comfy:3.1.0-sd3`: contains the checkpoint for [Stable Diffusion 3 medium](https://huggingface.co/stabilityai/stable-diffusion-3-medium) - ℹ️ [Use the Docker image on RunPod](#use-the-docker-image-on-runpod) +- 🧪 Pick an [example workflow](./test_resources/workflows/) & [send it to your deployed endpoint](#interact-with-your-runpod-api) ## Features @@ -66,10 +68,15 @@ Read our article here: https://blib.la/blog/comfyui-on-runpod - The generated image is either: - Returned as base64-encoded string (default) - Uploaded to AWS S3 ([if AWS S3 is configured](#upload-image-to-aws-s3)) -- There are three different Docker images to choose from: - - `timpietruskyblibla/runpod-worker-comfy:3.0.0-base`: doesn't contain anything, just a clean ComfyUI - - `timpietruskyblibla/runpod-worker-comfy:3.0.0-sdxl`: contains the checkpoint and VAE for Stable Diffusion XL - - `timpietruskyblibla/runpod-worker-comfy:3.0.0-sd3`: contains the checkpoint for Stable Diffusion 3 +- There are a few different Docker images to choose from: + - `timpietruskyblibla/runpod-worker-comfy:3.1.0-flux1-schnell`: contains the [flux1-schnell.safetensors](https://huggingface.co/black-forest-labs/FLUX.1-schnell) checkpoint, the [clip_l.safetensors](https://huggingface.co/comfyanonymous/flux_text_encoders/resolve/main/clip_l.safetensors) + [t5xxl_fp8_e4m3fn.safetensors](https://huggingface.co/comfyanonymous/flux_text_encoders/resolve/main/t5xxl_fp8_e4m3fn.safetensors) text encoders and [ae.safetensors](https://huggingface.co/black-forest-labs/FLUX.1-schnell/resolve/main/ae.safetensors) VAE for FLUX.1-schnell + - `timpietruskyblibla/runpod-worker-comfy:3.1.0-flux1-dev`: contains the [flux1-dev.safetensors](https://huggingface.co/black-forest-labs/FLUX.1-dev) checkpoint, the [clip_l.safetensors](https://huggingface.co/comfyanonymous/flux_text_encoders/resolve/main/clip_l.safetensors) + [t5xxl_fp8_e4m3fn.safetensors](https://huggingface.co/comfyanonymous/flux_text_encoders/resolve/main/t5xxl_fp8_e4m3fn.safetensors) text encoders and [ae.safetensors](https://huggingface.co/black-forest-labs/FLUX.1-dev/resolve/main/ae.safetensors) VAE for FLUX.1-dev + - `timpietruskyblibla/runpod-worker-comfy:3.1.0-sdxl`: contains the checkpoints and VAE for Stable Diffusion XL + - Checkpoint: [sd_xl_base_1.0.safetensors](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0) + - VAEs: + - [sdxl_vae.safetensors](https://huggingface.co/stabilityai/sdxl-vae/) + - [sdxl-vae-fp16-fix](https://huggingface.co/madebyollin/sdxl-vae-fp16-fix/) + - `timpietruskyblibla/runpod-worker-comfy:3.1.0-sd3`: contains the [sd3_medium_incl_clips_t5xxlfp8.safetensors](https://huggingface.co/stabilityai/stable-diffusion-3-medium) checkpoint for Stable Diffusion 3 medium - [Bring your own models](#bring-your-own-models) - Based on [Ubuntu + NVIDIA CUDA](https://hub.docker.com/r/nvidia/cuda) @@ -99,30 +106,46 @@ This is only needed if you want to upload the generated picture to AWS S3. If yo ## Use the Docker image on RunPod +### Create your template (optional) + - Create a [new template](https://runpod.io/console/serverless/user/templates) by clicking on `New Template` - In the dialog, configure: - Template Name: `runpod-worker-comfy` (it can be anything you want) - Template Type: serverless (change template type to "serverless") - - Container Image: `/:tag`, in this case: `timpietruskyblibla/runpod-worker-comfy:3.0.0-sd3` (or `-base` for a clean image or `-sdxl` for Stable Diffusion XL) + - Container Image: `/:tag`, in this case: `timpietruskyblibla/runpod-worker-comfy:3.1.0-sd3` (or `-base` for a clean image or `-sdxl` for Stable Diffusion XL or `-flex1-schnell` for FLUX.1 schnell) - Container Registry Credentials: You can leave everything as it is, as this repo is public - Container Disk: `20 GB` - (optional) Environment Variables: [Configure S3](#upload-image-to-aws-s3) - Note: You can also not configure it, the images will then stay in the worker. In order to have them stored permanently, [we have to add the network volume](https://github.com/blib-la/runpod-worker-comfy/issues/1) - Click on `Save Template` + +### Create your endpoint + - Navigate to [`Serverless > Endpoints`](https://www.runpod.io/console/serverless/user/endpoints) and click on `New Endpoint` - In the dialog, configure: + - Endpoint Name: `comfy` - - Select Template: `runpod-worker-comfy` (or whatever name you gave your template) + - Worker configuration: Select a GPU that can run the model you have chosen (see [GPU recommendations](#gpu-recommendations)) - Active Workers: `0` (whatever makes sense for you) - Max Workers: `3` (whatever makes sense for you) + - GPUs/Worker: `1` - Idle Timeout: `5` (you can leave the default) - Flash Boot: `enabled` (doesn't cost more, but provides faster boot of our worker, which is good) + - Select Template: `runpod-worker-comfy` (or whatever name you gave your template) - (optional) Advanced: If you are using a Network Volume, select it under `Select Network Volume`. Otherwise leave the defaults. - - Select a GPU that has some availability - - GPUs/Worker: `1` + - Click `deploy` - Your endpoint will be created, you can click on it to see the dashboard +### GPU recommendations + +| Model | Image | Minimum VRAM Required | Container Size | +| ------------------------- | --------------- | --------------------- | -------------- | +| Stable Diffusion XL | `sdxl` | 8 GB | 15 GB | +| Stable Diffusion 3 Medium | `sd3` | 5 GB | 20 GB | +| FLUX.1 Schnell | `flux1-schnell` | 24 GB | 30 GB | +| FLUX.1 dev | `flux1-dev` | 24 GB | 30 GB | + ## API specification The following describes which fields exist when doing requests to the API. We only describe the fields that are sent via `input` as those are needed by the worker itself. For a full list of fields, please take a look at the [official documentation](https://docs.runpod.io/docs/serverless-usage). @@ -193,7 +216,7 @@ The API expects a [JSON in this form](#json-request-body), where `workflow` is t Please also take a look at the [test_input.json](./test_input.json) to see how the API input should look like. -#### Example request with cURL +#### Example request for SDXL with cURL ```bash curl -X POST -H "Authorization: Bearer " -H "Content-Type: application/json" -d '{"input":{"workflow":{"3":{"inputs":{"seed":1337,"steps":20,"cfg":8,"sampler_name":"euler","scheduler":"normal","denoise":1,"model":["4",0],"positive":["6",0],"negative":["7",0],"latent_image":["5",0]},"class_type":"KSampler"},"4":{"inputs":{"ckpt_name":"sd_xl_base_1.0.safetensors"},"class_type":"CheckpointLoaderSimple"},"5":{"inputs":{"width":512,"height":512,"batch_size":1},"class_type":"EmptyLatentImage"},"6":{"inputs":{"text":"beautiful scenery nature glass bottle landscape, purple galaxy bottle,","clip":["4",1]},"class_type":"CLIPTextEncode"},"7":{"inputs":{"text":"text, watermark","clip":["4",1]},"class_type":"CLIPTextEncode"},"8":{"inputs":{"samples":["3",0],"vae":["4",2]},"class_type":"VAEDecode"},"9":{"inputs":{"filename_prefix":"ComfyUI","images":["8",0]},"class_type":"SaveImage"}}}}' https://api.runpod.ai/v2//runsync diff --git a/docker-bake.hcl b/docker-bake.hcl index 99d06fd..b9464fe 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -15,7 +15,7 @@ variable "HUGGINGFACE_ACCESS_TOKEN" { } group "default" { - targets = ["base", "sdxl", "sd3"] + targets = ["base", "sdxl", "sd3", "flux1-schnell", "flux1-dev"] } target "base" { @@ -47,3 +47,28 @@ target "sd3" { tags = ["${DOCKERHUB_REPO}/${DOCKERHUB_IMG}:${RELEASE_VERSION}-sd3"] inherits = ["base"] } + +target "flux1-schnell" { + context = "." + dockerfile = "Dockerfile" + target = "final" + args = { + MODEL_TYPE = "flux1-schnell" + HUGGINGFACE_ACCESS_TOKEN = "${HUGGINGFACE_ACCESS_TOKEN}" + } + tags = ["${DOCKERHUB_REPO}/${DOCKERHUB_IMG}:${RELEASE_VERSION}-flux1-schnell"] + inherits = ["base"] +} + +target "flux1-dev" { + context = "." + dockerfile = "Dockerfile" + target = "final" + args = { + MODEL_TYPE = "flux1-dev" + HUGGINGFACE_ACCESS_TOKEN = "${HUGGINGFACE_ACCESS_TOKEN}" + } + tags = ["${DOCKERHUB_REPO}/${DOCKERHUB_IMG}:${RELEASE_VERSION}-flux1-dev"] + inherits = ["base"] +} + diff --git a/src/extra_model_paths.yaml b/src/extra_model_paths.yaml index f2c82f9..676f615 100644 --- a/src/extra_model_paths.yaml +++ b/src/extra_model_paths.yaml @@ -9,3 +9,4 @@ runpod_worker_comfy: loras: models/loras/ upscale_models: models/upscale_models/ vae: models/vae/ + unet: models/unet/ diff --git a/test_resources/workflows/workflow_flux1_dev.json b/test_resources/workflows/workflow_flux1_dev.json new file mode 100644 index 0000000..9d6c444 --- /dev/null +++ b/test_resources/workflows/workflow_flux1_dev.json @@ -0,0 +1,130 @@ +{ + "input": { + "workflow": { + "5": { + "inputs": { + "width": 1024, + "height": 1024, + "batch_size": 1 + }, + "class_type": "EmptyLatentImage", + "_meta": { + "title": "Empty Latent Image" + } + }, + "6": { + "inputs": { + "text": "grey cat wearing a harry potter hat and programming in javascript in its ultramodern computer", + "clip": ["11", 0] + }, + "class_type": "CLIPTextEncode", + "_meta": { + "title": "CLIP Text Encode (Prompt)" + } + }, + "8": { + "inputs": { + "samples": ["13", 0], + "vae": ["10", 0] + }, + "class_type": "VAEDecode", + "_meta": { + "title": "VAE Decode" + } + }, + "9": { + "inputs": { + "filename_prefix": "ComfyUI", + "images": ["8", 0] + }, + "class_type": "SaveImage", + "_meta": { + "title": "Save Image" + } + }, + "10": { + "inputs": { + "vae_name": "ae.safetensors" + }, + "class_type": "VAELoader", + "_meta": { + "title": "Load VAE" + } + }, + "11": { + "inputs": { + "clip_name1": "t5xxl_fp8_e4m3fn.safetensors", + "clip_name2": "clip_l.safetensors", + "type": "flux" + }, + "class_type": "DualCLIPLoader", + "_meta": { + "title": "DualCLIPLoader" + } + }, + "12": { + "inputs": { + "unet_name": "flux1-dev.safetensors", + "weight_dtype": "fp8_e4m3fn" + }, + "class_type": "UNETLoader", + "_meta": { + "title": "Load Diffusion Model" + } + }, + "13": { + "inputs": { + "noise": ["25", 0], + "guider": ["22", 0], + "sampler": ["16", 0], + "sigmas": ["17", 0], + "latent_image": ["5", 0] + }, + "class_type": "SamplerCustomAdvanced", + "_meta": { + "title": "SamplerCustomAdvanced" + } + }, + "16": { + "inputs": { + "sampler_name": "euler" + }, + "class_type": "KSamplerSelect", + "_meta": { + "title": "KSamplerSelect" + } + }, + "17": { + "inputs": { + "scheduler": "sgm_uniform", + "steps": 4, + "denoise": 1, + "model": ["12", 0] + }, + "class_type": "BasicScheduler", + "_meta": { + "title": "BasicScheduler" + } + }, + "22": { + "inputs": { + "model": ["12", 0], + "conditioning": ["6", 0] + }, + "class_type": "BasicGuider", + "_meta": { + "title": "BasicGuider" + } + }, + "25": { + "inputs": { + "noise_seed": 108076821791990 + }, + "class_type": "RandomNoise", + "_meta": { + "title": "RandomNoise" + } + } + } + } +} diff --git a/test_resources/workflows/workflow_flux1_schnell.json b/test_resources/workflows/workflow_flux1_schnell.json new file mode 100644 index 0000000..9d52765 --- /dev/null +++ b/test_resources/workflows/workflow_flux1_schnell.json @@ -0,0 +1,130 @@ +{ + "input": { + "workflow": { + "5": { + "inputs": { + "width": 1024, + "height": 1024, + "batch_size": 1 + }, + "class_type": "EmptyLatentImage", + "_meta": { + "title": "Empty Latent Image" + } + }, + "6": { + "inputs": { + "text": "grey cat wearing a harry potter hat and programming in javascript in its ultramodern computer", + "clip": ["11", 0] + }, + "class_type": "CLIPTextEncode", + "_meta": { + "title": "CLIP Text Encode (Prompt)" + } + }, + "8": { + "inputs": { + "samples": ["13", 0], + "vae": ["10", 0] + }, + "class_type": "VAEDecode", + "_meta": { + "title": "VAE Decode" + } + }, + "9": { + "inputs": { + "filename_prefix": "ComfyUI", + "images": ["8", 0] + }, + "class_type": "SaveImage", + "_meta": { + "title": "Save Image" + } + }, + "10": { + "inputs": { + "vae_name": "ae.safetensors" + }, + "class_type": "VAELoader", + "_meta": { + "title": "Load VAE" + } + }, + "11": { + "inputs": { + "clip_name1": "t5xxl_fp8_e4m3fn.safetensors", + "clip_name2": "clip_l.safetensors", + "type": "flux" + }, + "class_type": "DualCLIPLoader", + "_meta": { + "title": "DualCLIPLoader" + } + }, + "12": { + "inputs": { + "unet_name": "flux1-schnell.safetensors", + "weight_dtype": "fp8_e4m3fn" + }, + "class_type": "UNETLoader", + "_meta": { + "title": "Load Diffusion Model" + } + }, + "13": { + "inputs": { + "noise": ["25", 0], + "guider": ["22", 0], + "sampler": ["16", 0], + "sigmas": ["17", 0], + "latent_image": ["5", 0] + }, + "class_type": "SamplerCustomAdvanced", + "_meta": { + "title": "SamplerCustomAdvanced" + } + }, + "16": { + "inputs": { + "sampler_name": "euler" + }, + "class_type": "KSamplerSelect", + "_meta": { + "title": "KSamplerSelect" + } + }, + "17": { + "inputs": { + "scheduler": "sgm_uniform", + "steps": 4, + "denoise": 1, + "model": ["12", 0] + }, + "class_type": "BasicScheduler", + "_meta": { + "title": "BasicScheduler" + } + }, + "22": { + "inputs": { + "model": ["12", 0], + "conditioning": ["6", 0] + }, + "class_type": "BasicGuider", + "_meta": { + "title": "BasicGuider" + } + }, + "25": { + "inputs": { + "noise_seed": 108076821791990 + }, + "class_type": "RandomNoise", + "_meta": { + "title": "RandomNoise" + } + } + } + } +}