From 52216ccf0b317a0460a7d3315d7e9afee182135b Mon Sep 17 00:00:00 2001 From: USTC-liuchang <68578066+AlonzoLeeeooo@users.noreply.github.com> Date: Thu, 5 Sep 2024 16:44:20 +0800 Subject: [PATCH 1/9] `ControlNeXtSDVModel` -> `ControlNeXtSVDModel` --- .../pipeline/pipeline_stable_video_diffusion_controlnext.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ControlNeXt-SVD-v2-Training/pipeline/pipeline_stable_video_diffusion_controlnext.py b/ControlNeXt-SVD-v2-Training/pipeline/pipeline_stable_video_diffusion_controlnext.py index d0c1e7c..214d142 100644 --- a/ControlNeXt-SVD-v2-Training/pipeline/pipeline_stable_video_diffusion_controlnext.py +++ b/ControlNeXt-SVD-v2-Training/pipeline/pipeline_stable_video_diffusion_controlnext.py @@ -20,7 +20,7 @@ import PIL.Image import torch from transformers import CLIPImageProcessor, CLIPVisionModelWithProjection -from models.controlnext_vid_svd import ControlNeXtSDVModel +from models.controlnext_vid_svd import ControlNeXtSVDModel from diffusers.image_processor import VaeImageProcessor from diffusers.models import AutoencoderKLTemporalDecoder, UNetSpatioTemporalConditionModel @@ -125,7 +125,7 @@ def __init__( vae: AutoencoderKLTemporalDecoder, image_encoder: CLIPVisionModelWithProjection, unet: UNetSpatioTemporalConditionControlNeXtModel, - controlnext: ControlNeXtSDVModel, + controlnext: ControlNeXtSVDModel, scheduler: EulerDiscreteScheduler, feature_extractor: CLIPImageProcessor, ): From 84553dab8b98cdcfdd56e5d000654dd832297cd3 Mon Sep 17 00:00:00 2001 From: USTC-liuchang <68578066+AlonzoLeeeooo@users.noreply.github.com> Date: Thu, 5 Sep 2024 16:44:50 +0800 Subject: [PATCH 2/9] `ControlNeXtSDVModel` -> `ControlNeXtSVDModel` --- ControlNeXt-SVD-v2-Training/models/controlnext_vid_svd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ControlNeXt-SVD-v2-Training/models/controlnext_vid_svd.py b/ControlNeXt-SVD-v2-Training/models/controlnext_vid_svd.py index f1fd620..e759aed 100644 --- a/ControlNeXt-SVD-v2-Training/models/controlnext_vid_svd.py +++ b/ControlNeXt-SVD-v2-Training/models/controlnext_vid_svd.py @@ -9,7 +9,7 @@ from diffusers.models.resnet import Downsample2D, ResnetBlock2D -class ControlNeXtSDVModel(ModelMixin, ConfigMixin): +class ControlNeXtSVDModel(ModelMixin, ConfigMixin): _supports_gradient_checkpointing = True @register_to_config From e9f849c92cc4f416a0e25992c204d5c31c3f95ba Mon Sep 17 00:00:00 2001 From: USTC-liuchang <68578066+AlonzoLeeeooo@users.noreply.github.com> Date: Thu, 5 Sep 2024 16:45:32 +0800 Subject: [PATCH 3/9] `ControlNeXtSDVModel` -> `ControlNeXtSVDModel` --- ControlNeXt-SVD-v2-Training/train_svd.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ControlNeXt-SVD-v2-Training/train_svd.py b/ControlNeXt-SVD-v2-Training/train_svd.py index 2ba806d..5345cf5 100644 --- a/ControlNeXt-SVD-v2-Training/train_svd.py +++ b/ControlNeXt-SVD-v2-Training/train_svd.py @@ -59,7 +59,7 @@ from utils.vid_dataset import UBCFashion from models.unet_spatio_temporal_condition_controlnext import UNetSpatioTemporalConditionControlNeXtModel from pipeline.pipeline_stable_video_diffusion_controlnext import StableVideoDiffusionPipelineControlNeXt -from models.controlnext_vid_svd import ControlNeXtSDVModel +from models.controlnext_vid_svd import ControlNeXtSVDModel import torch.nn as nn import pdb from diffusers.utils.torch_utils import randn_tensor @@ -920,7 +920,7 @@ def main(): ) logger.info("Initializing controlnext weights from unet") - controlnext = ControlNeXtSDVModel() + controlnext = ControlNeXtSVDModel() if args.controlnet_model_name_or_path: logger.info("Loading existing controlnet weights") From 408c45e75651920726d365ff2839b43466b42f34 Mon Sep 17 00:00:00 2001 From: USTC-liuchang <68578066+AlonzoLeeeooo@users.noreply.github.com> Date: Thu, 5 Sep 2024 16:46:12 +0800 Subject: [PATCH 4/9] Update controlnext_vid_svd.py --- ControlNeXt-SVD/models/controlnext_vid_svd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ControlNeXt-SVD/models/controlnext_vid_svd.py b/ControlNeXt-SVD/models/controlnext_vid_svd.py index 6b72e64..e915d8b 100644 --- a/ControlNeXt-SVD/models/controlnext_vid_svd.py +++ b/ControlNeXt-SVD/models/controlnext_vid_svd.py @@ -337,7 +337,7 @@ def forward( -class ControlNeXtSDVModel(ModelMixin, ConfigMixin): +class ControlNeXtSVDModel(ModelMixin, ConfigMixin): _supports_gradient_checkpointing = True From f74a04556175174ded5ac1879704b51b188d9a22 Mon Sep 17 00:00:00 2001 From: USTC-liuchang <68578066+AlonzoLeeeooo@users.noreply.github.com> Date: Thu, 5 Sep 2024 16:46:56 +0800 Subject: [PATCH 5/9] `ControlNeXtSDVModel` -> `ControlNeXtSVDModel` --- .../pipeline/pipeline_stable_video_diffusion_controlnext.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ControlNeXt-SVD/pipeline/pipeline_stable_video_diffusion_controlnext.py b/ControlNeXt-SVD/pipeline/pipeline_stable_video_diffusion_controlnext.py index 01d2bda..5b001a8 100644 --- a/ControlNeXt-SVD/pipeline/pipeline_stable_video_diffusion_controlnext.py +++ b/ControlNeXt-SVD/pipeline/pipeline_stable_video_diffusion_controlnext.py @@ -20,7 +20,7 @@ import PIL.Image import torch from transformers import CLIPImageProcessor, CLIPVisionModelWithProjection -from models.controlnext_vid_svd import ControlNeXtSDVModel +from models.controlnext_vid_svd import ControlNeXtSVDModel from diffusers.image_processor import VaeImageProcessor from diffusers.models import AutoencoderKLTemporalDecoder, UNetSpatioTemporalConditionModel @@ -125,7 +125,7 @@ def __init__( vae: AutoencoderKLTemporalDecoder, image_encoder: CLIPVisionModelWithProjection, unet: UNetSpatioTemporalConditionControlNeXtModel, - controlnext: ControlNeXtSDVModel, + controlnext: ControlNeXtSVDModel, scheduler: EulerDiscreteScheduler, feature_extractor: CLIPImageProcessor, ): From 406edc4b2cbe448b593816b8865fbb34f34872cf Mon Sep 17 00:00:00 2001 From: USTC-liuchang <68578066+AlonzoLeeeooo@users.noreply.github.com> Date: Thu, 5 Sep 2024 16:47:28 +0800 Subject: [PATCH 6/9] `ControlNeXtSDVModel` -> `ControlNeXtSVDModel` --- ControlNeXt-SVD/run_controlnext.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ControlNeXt-SVD/run_controlnext.py b/ControlNeXt-SVD/run_controlnext.py index d85dd22..a3bfb0e 100644 --- a/ControlNeXt-SVD/run_controlnext.py +++ b/ControlNeXt-SVD/run_controlnext.py @@ -3,7 +3,7 @@ import numpy as np from PIL import Image from pipeline.pipeline_stable_video_diffusion_controlnext import StableVideoDiffusionPipelineControlNeXt -from models.controlnext_vid_svd import ControlNeXtSDVModel +from models.controlnext_vid_svd import ControlNeXtSVDModel from models.unet_spatio_temporal_condition_controlnext import UNetSpatioTemporalConditionControlNeXtModel from transformers import CLIPVisionModelWithProjection import re @@ -202,7 +202,7 @@ def load_tensor(tensor_path): low_cpu_mem_usage=True, variant="fp16", ) - controlnext = ControlNeXtSDVModel() + controlnext = ControlNeXtSVDModel() controlnext.load_state_dict(load_tensor(args.controlnext_path)) unet.load_state_dict(load_tensor(args.unet_path), strict=False) From 69cb03b6ac727d754f83670a4eab0b02c3ee16a3 Mon Sep 17 00:00:00 2001 From: USTC-liuchang <68578066+AlonzoLeeeooo@users.noreply.github.com> Date: Thu, 5 Sep 2024 16:48:18 +0800 Subject: [PATCH 7/9] `ControlNeXtSDVModel` -> `ControlNeXtSVDModel` --- ControlNeXt-SVD-v2/models/controlnext_vid_svd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ControlNeXt-SVD-v2/models/controlnext_vid_svd.py b/ControlNeXt-SVD-v2/models/controlnext_vid_svd.py index c390277..663d79f 100644 --- a/ControlNeXt-SVD-v2/models/controlnext_vid_svd.py +++ b/ControlNeXt-SVD-v2/models/controlnext_vid_svd.py @@ -9,7 +9,7 @@ from diffusers.models.resnet import Downsample2D, ResnetBlock2D -class ControlNeXtSDVModel(ModelMixin, ConfigMixin): +class ControlNeXtSVDModel(ModelMixin, ConfigMixin): _supports_gradient_checkpointing = True @register_to_config From b81bdaf92118b36674f118023a5f7152c1b510f5 Mon Sep 17 00:00:00 2001 From: USTC-liuchang <68578066+AlonzoLeeeooo@users.noreply.github.com> Date: Thu, 5 Sep 2024 16:49:04 +0800 Subject: [PATCH 8/9] `ControlNeXtSDVModel` -> `ControlNeXtSVDModel` --- .../pipeline/pipeline_stable_video_diffusion_controlnext.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ControlNeXt-SVD-v2/pipeline/pipeline_stable_video_diffusion_controlnext.py b/ControlNeXt-SVD-v2/pipeline/pipeline_stable_video_diffusion_controlnext.py index 03466e2..e45711a 100644 --- a/ControlNeXt-SVD-v2/pipeline/pipeline_stable_video_diffusion_controlnext.py +++ b/ControlNeXt-SVD-v2/pipeline/pipeline_stable_video_diffusion_controlnext.py @@ -20,7 +20,7 @@ import PIL.Image import torch from transformers import CLIPImageProcessor, CLIPVisionModelWithProjection -from models.controlnext_vid_svd import ControlNeXtSDVModel +from models.controlnext_vid_svd import ControlNeXtSVDModel from diffusers.image_processor import VaeImageProcessor from diffusers.models import AutoencoderKLTemporalDecoder, UNetSpatioTemporalConditionModel @@ -125,7 +125,7 @@ def __init__( vae: AutoencoderKLTemporalDecoder, image_encoder: CLIPVisionModelWithProjection, unet: UNetSpatioTemporalConditionControlNeXtModel, - controlnext: ControlNeXtSDVModel, + controlnext: ControlNeXtSVDModel, scheduler: EulerDiscreteScheduler, feature_extractor: CLIPImageProcessor, ): From 5390bc4f03a460bf360a513a078c7b317341ce16 Mon Sep 17 00:00:00 2001 From: USTC-liuchang <68578066+AlonzoLeeeooo@users.noreply.github.com> Date: Thu, 5 Sep 2024 16:49:36 +0800 Subject: [PATCH 9/9] `ControlNeXtSDVModel` -> `ControlNeXtSVDModel` --- ControlNeXt-SVD-v2/run_controlnext.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ControlNeXt-SVD-v2/run_controlnext.py b/ControlNeXt-SVD-v2/run_controlnext.py index 3f6a8f2..a41a828 100644 --- a/ControlNeXt-SVD-v2/run_controlnext.py +++ b/ControlNeXt-SVD-v2/run_controlnext.py @@ -3,7 +3,7 @@ import numpy as np from PIL import Image from pipeline.pipeline_stable_video_diffusion_controlnext import StableVideoDiffusionPipelineControlNeXt -from models.controlnext_vid_svd import ControlNeXtSDVModel +from models.controlnext_vid_svd import ControlNeXtSVDModel from models.unet_spatio_temporal_condition_controlnext import UNetSpatioTemporalConditionControlNeXtModel from transformers import CLIPVisionModelWithProjection import re @@ -221,7 +221,7 @@ def load_tensor(tensor_path): subfolder="unet", low_cpu_mem_usage=True, ) - controlnext = ControlNeXtSDVModel() + controlnext = ControlNeXtSVDModel() controlnext.load_state_dict(load_tensor(args.controlnext_path)) unet.load_state_dict(load_tensor(args.unet_path), strict=False) @@ -279,4 +279,4 @@ def load_tensor(tensor_path): final_result, validation_control_images[:num_frames], args.output_dir, - fps=fps) \ No newline at end of file + fps=fps)