support qwen-image inpaint controlnet

diffsynth/configs/model_config.py

@@ -169,6 +169,7 @@ model_loader_configs = [
     (None, "8004730443f55db63092006dd9f7110e", ["qwen_image_text_encoder"], [QwenImageTextEncoder], "diffusers"),
     (None, "ed4ea5824d55ec3107b09815e318123a", ["qwen_image_vae"], [QwenImageVAE], "diffusers"),
     (None, "073bce9cf969e317e5662cd570c3e79c", ["qwen_image_blockwise_controlnet"], [QwenImageBlockWiseControlNet], "civitai"),
+    (None, "a9e54e480a628f0b956a688a81c33bab", ["qwen_image_blockwise_controlnet"], [QwenImageBlockWiseControlNet], "civitai"),
 ]
 huggingface_model_loader_configs = [
     # These configs are provided for detecting model type automatically.

diffsynth/models/qwen_image_controlnet.py

@@ -1,10 +1,7 @@
 import torch
 import torch.nn as nn
-from .qwen_image_dit import QwenEmbedRope, QwenImageTransformerBlock
-from ..vram_management import gradient_checkpoint_forward
-from einops import rearrange
-from .sd3_dit import TimestepEmbeddings, RMSNorm
-
+from .sd3_dit import RMSNorm
+from .utils import hash_state_dict_keys
 
 
 class BlockWiseControlBlock(torch.nn.Module):
@@ -35,10 +32,11 @@ class QwenImageBlockWiseControlNet(torch.nn.Module):
         self,
         num_layers: int = 60,
         in_dim: int = 64,
+        additional_in_dim: int = 0,
         dim: int = 3072,
     ):
         super().__init__()
-        self.img_in = nn.Linear(in_dim, dim)
+        self.img_in = nn.Linear(in_dim + additional_in_dim, dim)
         self.controlnet_blocks = nn.ModuleList(
             [
                 BlockWiseControlBlock(dim)
@@ -68,4 +66,9 @@ class QwenImageBlockWiseControlNetStateDictConverter():
         pass
 
     def from_civitai(self, state_dict):
-        return state_dict
+        hash_value = hash_state_dict_keys(state_dict)
+        extra_kwargs = {}
+        if hash_value == "a9e54e480a628f0b956a688a81c33bab":
+            # inpaint controlnet
+            extra_kwargs = {"additional_in_dim": 4}
+        return state_dict, extra_kwargs