add ras-sd35 experimental

Signed-off-by: Vladimir Mandic <mandic00@live.com>

Author: vladmandic

.pylintrc

@@ -23,20 +23,21 @@ ignore-paths=/usr/lib/.*$,
              modules/k-diffusion,
              modules/ldsr,
              modules/meissonic,
+             modules/mod,
              modules/omnigen,
              modules/onnx_impl,
              modules/pag,
              modules/pixelsmith,
              modules/prompt_parser_xhinker.py,
              modules/pulid/eva_clip,
+             modules/ras,
              modules/rife,
              modules/schedulers,
              modules/taesd,
              modules/teacache,
              modules/todo,
              modules/unipc,
              modules/xadapter,
-             modules/mod,
              repositories,
              extensions-builtin/Lora,
              extensions-builtin/sd-webui-agent-scheduler,

.ruff.toml

@@ -17,12 +17,14 @@ exclude = [
     "modules/k-diffusion",
     "modules/ldsr",
     "modules/meissonic",
+    "modules/mod",
     "modules/omnigen",
     "modules/pag",
     "modules/pixelsmith",
     "modules/postprocess/aurasr_arch.py",
     "modules/prompt_parser_xhinker.py",
     "modules/pulid/eva_clip",
+    "modules/ras",
     "modules/rife",
     "modules/schedulers",
     "modules/segmoe",
@@ -31,7 +33,6 @@ exclude = [
     "modules/todo",
     "modules/unipc",
     "modules/xadapter",
-    "modules/mod",
     "repositories",
     "extensions-builtin/Lora",
     "extensions-builtin/sd-extension-chainner/nodes",

CHANGELOG.md

@@ -50,13 +50,16 @@ We're back with another update with nearly 100 commits!
     result is model smaller than *Flux.1-Dev*, but with similar capabilities  
     fully supports offloading and on-the-fly quantization  
     simply select from *networks -> models -> reference*  
-- **Pipelines**  
+- **Functions**  
   - [Mixture-of-Diffusers](https://huggingface.co/posts/elismasilva/251775641926329)  
     Regional tiling type of a solution for SDXL models  
     select from *scripts -> mixture of diffusers*  
   - [Automatic Color Inpaint]  
     Automatically creates mask based on selected color and triggers inpaint  
     simply select in *scripts -> automatic color inpaint* when in img2img mode  
+  - [RAS: Region-Adaptive Sampling](https://github.com/microsoft/RAS) *experimental*  
+    Speeds up SD3.5 models by sampling only regions of interest  
+    Enable in *settings -> pipeline modifiers -> ras*  
 - **Interrogate/Captioning**  
   - Redesigned captioning UI  
     split from Process tab into separate tab  

modules/interrogate/deepseek.py

@@ -20,7 +20,7 @@
 vl_chat_processor = None
 
 
-class fake_attrdict(object):
+class fake_attrdict():
     class AttrDict(dict): # dot notation access to dictionary attributes
         __getattr__ = dict.get
         __setattr__ = dict.__setitem__

modules/interrogate/interrogate.py

@@ -28,7 +28,7 @@ def interrogate(image):
     elif shared.opts.interrogate_default_type == 'VLM':
         shared.log.info(f'Interrogate: type={shared.opts.interrogate_default_type} vlm="{shared.opts.interrogate_vlm_model}" prompt="{shared.opts.interrogate_vlm_prompt}"')
         from modules.interrogate import vqa
-        prompt = vqa.interrogate(image=image, model_name=shared.opts.interrogate_vlm_model, question=shared.opts.interrogate_vlm_prompt)
+        prompt = vqa.interrogate(image=image, model_name=shared.opts.interrogate_vlm_model, question=shared.opts.interrogate_vlm_prompt, prompt=None)
         shared.log.debug(f'Interrogate: time={time.time()-t0:.2f} answer="{prompt}"')
         return prompt
     else:

modules/interrogate/joytag.py

@@ -896,7 +896,7 @@ def __init__(self, config: str):
             ty, line = line.split(":") if ":" in line else (line, "")
             options = line.split(",")
             options = [o.split("=") for o in options] if line else []
-            options = {k: v for k, v in options} # noqa: C416
+            options = {k: v for k, v in options} # pylint: disable=unnecessary-comprehension # noqa: C416
             if ty == 'conv':
                 layers.append(nn.Conv2d(
                     in_channels=channels,

modules/interrogate/openclip.py

@@ -239,11 +239,11 @@ def close(self):
 
 def update_interrogate_params():
     if ci is not None:
-        ci.caption_max_length=shared.opts.interrogate_clip_max_length,
-        ci.chunk_size=shared.opts.interrogate_clip_chunk_size,
-        ci.flavor_intermediate_count=shared.opts.interrogate_clip_flavor_count,
-        ci.clip_offload=shared.opts.interrogate_offload,
-        ci.caption_offload=shared.opts.interrogate_offload,
+        ci.caption_max_length=shared.opts.interrogate_clip_max_length
+        ci.chunk_size=shared.opts.interrogate_clip_chunk_size
+        ci.flavor_intermediate_count=shared.opts.interrogate_clip_flavor_count
+        ci.clip_offload=shared.opts.interrogate_offload
+        ci.caption_offload=shared.opts.interrogate_offload
 
 
 def get_clip_models():

modules/masking.py

@@ -408,7 +408,6 @@ def run_mask(input_image: Image.Image, input_mask: Image.Image = None, return_ty
         debug(f'Mask args legacy: blur={mask_blur} padding={mask_padding}')
     if invert is not None:
         opts.invert = invert
-    print('HERE', opts.mask_blur, mask_blur)
     if mask_blur is not None: # compatibility with old img2img values which uses px values
         opts.mask_blur = round(4 * mask_blur / size, 3)
     if mask_padding is not None: # compatibility with old img2img values which uses px values

modules/postprocess/dcc.py

@@ -1,8 +1,8 @@
 import numpy as np
 
 
-def DetectDirect(A, type, k, T):
-    if type == 1:
+def DetectDirect(A, dcc_type, k, T):
+    if dcc_type == 1:
         # 45 degree diagonal direction
         t1 = abs(A[2,0]-A[0,2])
         t2 = abs(A[4,0]-A[2,2])+abs(A[2,2]-A[0,4])

modules/processing_diffusers.py

@@ -5,7 +5,7 @@
 import torch
 import torchvision.transforms.functional as TF
 from PIL import Image
-from modules import shared, devices, processing, sd_models, errors, sd_hijack_hypertile, processing_vae, sd_models_compile, hidiffusion, timer, modelstats, extra_networks
+from modules import shared, devices, processing, sd_models, errors, sd_hijack_hypertile, processing_vae, sd_models_compile, hidiffusion, timer, modelstats, extra_networks, ras
 from modules.processing_helpers import resize_hires, calculate_base_steps, calculate_hires_steps, calculate_refiner_steps, save_intermediate, update_sampler, is_txt2img, is_refiner_enabled, get_job_name
 from modules.processing_args import set_pipeline_args
 from modules.onnx_impl import preprocess_pipeline as preprocess_onnx_pipeline, check_parameters_changed as olive_check_parameters_changed
@@ -93,6 +93,7 @@ def process_base(p: processing.StableDiffusionProcessing):
             sd_models.move_model(shared.sd_model.transformer, devices.device)
         extra_networks.activate(p, exclude=['text_encoder', 'text_encoder_2', 'text_encoder_3'])
         hidiffusion.apply(p, shared.sd_model_type)
+        ras.apply(shared.sd_model, p)
         timer.process.record('move')
         if hasattr(shared.sd_model, 'tgate') and getattr(p, 'gate_step', -1) > 0:
             base_args['gate_step'] = p.gate_step
@@ -106,6 +107,7 @@ def process_base(p: processing.StableDiffusionProcessing):
         if hasattr(output, 'images'):
             shared.history.add(output.images, info=processing.create_infotext(p), ops=p.ops)
         timer.process.record('pipeline')
+        ras.unapply(shared.sd_model)
         hidiffusion.unapply()
         sd_models_compile.openvino_post_compile(op="base") # only executes on compiled vino models
         sd_models_compile.check_deepcache(enable=False)

modules/prompt_parser.py

@@ -354,7 +354,7 @@ def multiply_range(start_position, multiplier):
             # log.trace(f'Prompt: text="{text[m.start():m.end()]}" section="{section}" weight="{weight}"')
             if len(section) == 0:
                 continue
-            elif section.startswith('\\'):
+            if section.startswith('\\'):
                 if len(res) > 0 and text[m.start()-1] != ' ':
                     res[-1][0] += section[1:] # append literal character to the last section
                 else:

modules/ras/__init__.py

@@ -0,0 +1,36 @@
+# source <https://github.com/Trojaner/RAS_Simplified>
+# original: <https://github.com/microsoft/RAS>
+
+from modules import shared, processing
+
+
+def apply(pipe, p: processing.StableDiffusionProcessing):
+    if shared.sd_model_type != "sd3" or not shared.opts.ras_enable:
+        return
+    from .ras_manager import MANAGER
+    from .ras_scheduler import RASFlowMatchEulerDiscreteScheduler
+    from .ras_attention import RASJointAttnProcessor2_0
+    from .ras_forward import ras_forward
+    scheduler = RASFlowMatchEulerDiscreteScheduler.from_config(pipe.scheduler.config)
+    pipe.scheduler = scheduler
+    MANAGER.num_steps = p.steps
+    MANAGER.scheduler_end_step = p.steps
+    MANAGER.width = p.width
+    MANAGER.height = p.height
+    MANAGER.error_reset_steps = [int(1*p.steps/3), int(2*p.steps/3)]
+    shared.log.info(f'RAS: scheduler={pipe.scheduler.__class__.__name__} {str(MANAGER)}')
+    MANAGER.reset_cache()
+    MANAGER.generate_skip_token_list()
+    pipe.transformer.old_forward = pipe.transformer.forward
+    pipe.transformer.forward = ras_forward.__get__(pipe.transformer, pipe.transformer.__class__) # pylint: disable=no-value-for-parameter
+    for block in pipe.transformer.transformer_blocks:
+        block.attn.set_processor(RASJointAttnProcessor2_0())
+
+
+def unapply(pipe):
+    if hasattr(pipe, 'transformer') and hasattr(pipe.transformer, "old_forward"):
+        from diffusers.models.attention_processor import JointAttnProcessor2_0
+        pipe.transformer.forward = pipe.transformer.old_forward
+        del pipe.transformer.old_forward
+        for block in pipe.transformer.transformer_blocks:
+            block.attn.set_processor(JointAttnProcessor2_0())