diff --git a/diffsynth/configs/model_config.py b/diffsynth/configs/model_config.py index 963af72..55ab380 100644 --- a/diffsynth/configs/model_config.py +++ b/diffsynth/configs/model_config.py @@ -204,7 +204,6 @@ preset_models_on_huggingface = { ("lllyasviel/omost-llama-3-8b-4bits", "model.safetensors.index.json", "models/OmostPrompt/omost-llama-3-8b-4bits"), ("lllyasviel/omost-llama-3-8b-4bits", "special_tokens_map.json", "models/OmostPrompt/omost-llama-3-8b-4bits"), ], - # Translator "opus-mt-zh-en": [ ("Helsinki-NLP/opus-mt-zh-en", "config.json", "models/translator/opus-mt-zh-en"), @@ -346,6 +345,24 @@ preset_models_on_modelscope = { ("AI-ModelScope/controlnet-union-sdxl-1.0", "diffusion_pytorch_model_promax.safetensors", "models/ControlNet/controlnet_union"), ("sd_lora/Annotators", "dpt_hybrid-midas-501f0c75.pt", "models/Annotators") ], + "Annotators:Depth": [ + ("lllyasviel/Annotators", "dpt_hybrid-midas-501f0c75.pt", "models/Annotators"), + ], + "Annotators:Softedge": [ + ("lllyasviel/Annotators", "ControlNetHED.pth", "models/Annotators"), + ], + "Annotators:Lineart": [ + ("lllyasviel/Annotators", "sk_model.pth", "models/Annotators"), + ("lllyasviel/Annotators", "sk_model2.pth", "models/Annotators"), + ], + "Annotators:Normal": [ + ("lllyasviel/Annotators", "scannet.pt", "models/Annotators"), + ], + "Annotators:Openpose": [ + ("lllyasviel/Annotators", "body_pose_model.pth", "models/Annotators"), + ("lllyasviel/Annotators", "facenet.pth", "models/Annotators"), + ("lllyasviel/Annotators", "hand_pose_model.pth", "models/Annotators"), + ], # AnimateDiff "AnimateDiff_v2": [ ("Shanghai_AI_Laboratory/animatediff", "mm_sd_v15_v2.ckpt", "models/AnimateDiff"), @@ -487,6 +504,30 @@ preset_models_on_modelscope = { "models/FLUX/FLUX.1-schnell/flux1-schnell.safetensors" ], }, + "InstantX/FLUX.1-dev-Controlnet-Union-alpha": [ + ("InstantX/FLUX.1-dev-Controlnet-Union-alpha", "diffusion_pytorch_model.safetensors", "models/ControlNet/InstantX/FLUX.1-dev-Controlnet-Union-alpha"), + ], + "jasperai/Flux.1-dev-Controlnet-Depth": [ + ("jasperai/Flux.1-dev-Controlnet-Depth", "diffusion_pytorch_model.safetensors", "models/ControlNet/jasperai/Flux.1-dev-Controlnet-Depth"), + ], + "jasperai/Flux.1-dev-Controlnet-Surface-Normals": [ + ("jasperai/Flux.1-dev-Controlnet-Surface-Normals", "diffusion_pytorch_model.safetensors", "models/ControlNet/jasperai/Flux.1-dev-Controlnet-Surface-Normals"), + ], + "jasperai/Flux.1-dev-Controlnet-Upscaler": [ + ("jasperai/Flux.1-dev-Controlnet-Upscaler", "diffusion_pytorch_model.safetensors", "models/ControlNet/jasperai/Flux.1-dev-Controlnet-Upscaler"), + ], + "alimama-creative/FLUX.1-dev-Controlnet-Inpainting-Alpha": [ + ("alimama-creative/FLUX.1-dev-Controlnet-Inpainting-Alpha", "diffusion_pytorch_model.safetensors", "models/ControlNet/alimama-creative/FLUX.1-dev-Controlnet-Inpainting-Alpha"), + ], + "alimama-creative/FLUX.1-dev-Controlnet-Inpainting-Beta": [ + ("alimama-creative/FLUX.1-dev-Controlnet-Inpainting-Beta", "diffusion_pytorch_model.safetensors", "models/ControlNet/alimama-creative/FLUX.1-dev-Controlnet-Inpainting-Beta"), + ], + "Shakker-Labs/FLUX.1-dev-ControlNet-Depth": [ + ("Shakker-Labs/FLUX.1-dev-ControlNet-Depth", "diffusion_pytorch_model.safetensors", "models/ControlNet/Shakker-Labs/FLUX.1-dev-ControlNet-Depth"), + ], + "Shakker-Labs/FLUX.1-dev-ControlNet-Union-Pro": [ + ("Shakker-Labs/FLUX.1-dev-ControlNet-Union-Pro", "diffusion_pytorch_model.safetensors", "models/ControlNet/Shakker-Labs/FLUX.1-dev-ControlNet-Union-Pro"), + ], # ESRGAN "ESRGAN_x4": [ ("AI-ModelScope/Real-ESRGAN", "RealESRGAN_x4.pth", "models/ESRGAN"), @@ -546,10 +587,23 @@ Preset_model_id: TypeAlias = Literal[ "ControlNet_union_sdxl_promax", "FLUX.1-dev", "FLUX.1-schnell", + "InstantX/FLUX.1-dev-Controlnet-Union-alpha", + "jasperai/Flux.1-dev-Controlnet-Depth", + "jasperai/Flux.1-dev-Controlnet-Surface-Normals", + "jasperai/Flux.1-dev-Controlnet-Upscaler", + "alimama-creative/FLUX.1-dev-Controlnet-Inpainting-Alpha", + "alimama-creative/FLUX.1-dev-Controlnet-Inpainting-Beta", + "Shakker-Labs/FLUX.1-dev-ControlNet-Depth", + "Shakker-Labs/FLUX.1-dev-ControlNet-Union-Pro", "SDXL_lora_zyd232_ChineseInkStyle_SDXL_v1_0", "QwenPrompt", "OmostPrompt", "ESRGAN_x4", "RIFE", "CogVideoX-5B", + "Annotators:Depth", + "Annotators:Softedge", + "Annotators:Lineart", + "Annotators:Normal", + "Annotators:Openpose", ] diff --git a/diffsynth/models/tiler.py b/diffsynth/models/tiler.py index 6f36cdf..77c443b 100644 --- a/diffsynth/models/tiler.py +++ b/diffsynth/models/tiler.py @@ -107,6 +107,60 @@ class TileWorker: +class FastTileWorker: + def __init__(self): + pass + + + def build_mask(self, data, is_bound): + _, _, H, W = data.shape + h = repeat(torch.arange(H), "H -> H W", H=H, W=W) + w = repeat(torch.arange(W), "W -> H W", H=H, W=W) + border_width = (H + W) // 4 + pad = torch.ones_like(h) * border_width + mask = torch.stack([ + pad if is_bound[0] else h + 1, + pad if is_bound[1] else H - h, + pad if is_bound[2] else w + 1, + pad if is_bound[3] else W - w + ]).min(dim=0).values + mask = mask.clip(1, border_width) + mask = (mask / border_width).to(dtype=data.dtype, device=data.device) + mask = rearrange(mask, "H W -> 1 H W") + return mask + + + def tiled_forward(self, forward_fn, model_input, tile_size, tile_stride, tile_device="cpu", tile_dtype=torch.float32, border_width=None): + # Prepare + B, C, H, W = model_input.shape + border_width = int(tile_stride*0.5) if border_width is None else border_width + weight = torch.zeros((1, 1, H, W), dtype=tile_dtype, device=tile_device) + values = torch.zeros((B, C, H, W), dtype=tile_dtype, device=tile_device) + + # Split tasks + tasks = [] + for h in range(0, H, tile_stride): + for w in range(0, W, tile_stride): + if (h-tile_stride >= 0 and h-tile_stride+tile_size >= H) or (w-tile_stride >= 0 and w-tile_stride+tile_size >= W): + continue + h_, w_ = h + tile_size, w + tile_size + if h_ > H: h, h_ = H - tile_size, H + if w_ > W: w, w_ = W - tile_size, W + tasks.append((h, h_, w, w_)) + + # Run + for hl, hr, wl, wr in tasks: + # Forward + hidden_states_batch = forward_fn(hl, hr, wl, wr).to(dtype=tile_dtype, device=tile_device) + + mask = self.build_mask(hidden_states_batch, is_bound=(hl==0, hr>=H, wl==0, wr>=W)) + values[:, :, hl:hr, wl:wr] += hidden_states_batch * mask + weight[:, :, hl:hr, wl:wr] += mask + values /= weight + return values + + + class TileWorker2Dto3D: """ Process 3D tensors, but only enable TileWorker on 2D. diff --git a/diffsynth/pipelines/base.py b/diffsynth/pipelines/base.py index 55cfc14..f8f7178 100644 --- a/diffsynth/pipelines/base.py +++ b/diffsynth/pipelines/base.py @@ -47,9 +47,12 @@ class BasePipeline(torch.nn.Module): return value - def control_noise_via_local_prompts(self, prompt_emb_global, prompt_emb_locals, masks, mask_scales, inference_callback): - noise_pred_global = inference_callback(prompt_emb_global) - noise_pred_locals = [inference_callback(prompt_emb_local) for prompt_emb_local in prompt_emb_locals] + def control_noise_via_local_prompts(self, prompt_emb_global, prompt_emb_locals, masks, mask_scales, inference_callback, special_kwargs={}, special_local_kwargs_list=None): + noise_pred_global = inference_callback(prompt_emb_global, special_kwargs) + if special_local_kwargs_list is None: + noise_pred_locals = [inference_callback(prompt_emb_local) for prompt_emb_local in prompt_emb_locals] + else: + noise_pred_locals = [inference_callback(prompt_emb_local, special_kwargs) for prompt_emb_local, special_kwargs in zip(prompt_emb_locals, special_local_kwargs_list)] noise_pred = self.merge_latents(noise_pred_global, noise_pred_locals, masks, mask_scales) return noise_pred diff --git a/diffsynth/pipelines/flux_image.py b/diffsynth/pipelines/flux_image.py index 176651e..89d730f 100644 --- a/diffsynth/pipelines/flux_image.py +++ b/diffsynth/pipelines/flux_image.py @@ -8,6 +8,7 @@ import torch from tqdm import tqdm import numpy as np from PIL import Image +from ..models.tiler import FastTileWorker @@ -142,6 +143,7 @@ class FluxImagePipeline(BasePipeline): input_image=None, controlnet_image=None, controlnet_inpaint_mask=None, + enable_controlnet_on_negative=False, denoising_strength=1.0, height=1024, width=1024, @@ -186,8 +188,13 @@ class FluxImagePipeline(BasePipeline): # Prepare ControlNets if controlnet_image is not None: controlnet_kwargs = {"controlnet_frames": self.prepare_controlnet_input(controlnet_image, controlnet_inpaint_mask, tiler_kwargs)} + if len(masks) > 0 and controlnet_inpaint_mask is not None: + print("The controlnet_inpaint_mask will be overridden by masks.") + local_controlnet_kwargs = [{"controlnet_frames": self.prepare_controlnet_input(controlnet_image, mask, tiler_kwargs)} for mask in masks] + else: + local_controlnet_kwargs = None else: - controlnet_kwargs = {"controlnet_frames": None} + controlnet_kwargs, local_controlnet_kwargs = {"controlnet_frames": None}, [{}] * len(masks) # Denoise self.load_models_to_device(['dit', 'controlnet']) @@ -195,17 +202,21 @@ class FluxImagePipeline(BasePipeline): timestep = timestep.unsqueeze(0).to(self.device) # Classifier-free guidance - inference_callback = lambda prompt_emb_posi: lets_dance_flux( + inference_callback = lambda prompt_emb_posi, controlnet_kwargs: lets_dance_flux( dit=self.dit, controlnet=self.controlnet, hidden_states=latents, timestep=timestep, **prompt_emb_posi, **tiler_kwargs, **extra_input, **controlnet_kwargs ) - noise_pred_posi = self.control_noise_via_local_prompts(prompt_emb_posi, prompt_emb_locals, masks, mask_scales, inference_callback) + noise_pred_posi = self.control_noise_via_local_prompts( + prompt_emb_posi, prompt_emb_locals, masks, mask_scales, inference_callback, + special_kwargs=controlnet_kwargs, special_local_kwargs_list=local_controlnet_kwargs + ) if cfg_scale != 1.0: + negative_controlnet_kwargs = controlnet_kwargs if enable_controlnet_on_negative else {} noise_pred_nega = lets_dance_flux( dit=self.dit, controlnet=self.controlnet, hidden_states=latents, timestep=timestep, - **prompt_emb_nega, **tiler_kwargs, **extra_input, **controlnet_kwargs + **prompt_emb_nega, **tiler_kwargs, **extra_input, **negative_controlnet_kwargs, ) noise_pred = noise_pred_nega + cfg_scale * (noise_pred_posi - noise_pred_nega) else: @@ -244,6 +255,32 @@ def lets_dance_flux( tile_stride=64, **kwargs ): + if tiled: + def flux_forward_fn(hl, hr, wl, wr): + return lets_dance_flux( + dit=dit, + controlnet=controlnet, + hidden_states=hidden_states[:, :, hl: hr, wl: wr], + timestep=timestep, + prompt_emb=prompt_emb, + pooled_prompt_emb=pooled_prompt_emb, + guidance=guidance, + text_ids=text_ids, + image_ids=None, + controlnet_frames=[f[:, :, hl: hr, wl: wr] for f in controlnet_frames], + tiled=False, + **kwargs + ) + return FastTileWorker().tiled_forward( + flux_forward_fn, + hidden_states, + tile_size=tile_size, + tile_stride=tile_stride, + tile_device=hidden_states.device, + tile_dtype=hidden_states.dtype + ) + + # ControlNet if controlnet is not None and controlnet_frames is not None: controlnet_extra_kwargs = { diff --git a/examples/ControlNet/flux_controlnet.py b/examples/ControlNet/flux_controlnet.py new file mode 100644 index 0000000..6fc7526 --- /dev/null +++ b/examples/ControlNet/flux_controlnet.py @@ -0,0 +1,299 @@ +from diffsynth import ModelManager, FluxImagePipeline, ControlNetConfigUnit, download_models, download_customized_models +import torch +from PIL import Image +import numpy as np + + + +def example_1(): + model_manager = ModelManager(torch_dtype=torch.bfloat16, model_id_list=["FLUX.1-dev", "jasperai/Flux.1-dev-Controlnet-Upscaler"]) + pipe = FluxImagePipeline.from_model_manager(model_manager, controlnet_config_units=[ + ControlNetConfigUnit( + processor_id="tile", + model_path="models/ControlNet/jasperai/Flux.1-dev-Controlnet-Upscaler/diffusion_pytorch_model.safetensors", + scale=0.7 + ), + ]) + + image_1 = pipe( + prompt="a photo of a cat, highly detailed", + height=768, width=768, + seed=0 + ) + image_1.save("image_1.png") + + image_2 = pipe( + prompt="a photo of a cat, highly detailed", + controlnet_image=image_1.resize((2048, 2048)), + input_image=image_1.resize((2048, 2048)), denoising_strength=0.99, + height=2048, width=2048, tiled=True, + seed=1 + ) + image_2.save("image_2.png") + + + +def example_2(): + model_manager = ModelManager(torch_dtype=torch.bfloat16, model_id_list=["FLUX.1-dev", "jasperai/Flux.1-dev-Controlnet-Upscaler"]) + pipe = FluxImagePipeline.from_model_manager(model_manager, controlnet_config_units=[ + ControlNetConfigUnit( + processor_id="tile", + model_path="models/ControlNet/jasperai/Flux.1-dev-Controlnet-Upscaler/diffusion_pytorch_model.safetensors", + scale=0.7 + ), + ]) + + image_1 = pipe( + prompt="a beautiful Chinese girl, delicate skin texture", + height=768, width=768, + seed=2 + ) + image_1.save("image_3.png") + + image_2 = pipe( + prompt="a beautiful Chinese girl, delicate skin texture", + controlnet_image=image_1.resize((2048, 2048)), + input_image=image_1.resize((2048, 2048)), denoising_strength=0.99, + height=2048, width=2048, tiled=True, + seed=3 + ) + image_2.save("image_4.png") + + +def example_3(): + model_manager = ModelManager(torch_dtype=torch.bfloat16, model_id_list=["FLUX.1-dev", "InstantX/FLUX.1-dev-Controlnet-Union-alpha"]) + pipe = FluxImagePipeline.from_model_manager(model_manager, controlnet_config_units=[ + ControlNetConfigUnit( + processor_id="canny", + model_path="models/ControlNet/InstantX/FLUX.1-dev-Controlnet-Union-alpha/diffusion_pytorch_model.safetensors", + scale=0.3 + ), + ControlNetConfigUnit( + processor_id="depth", + model_path="models/ControlNet/InstantX/FLUX.1-dev-Controlnet-Union-alpha/diffusion_pytorch_model.safetensors", + scale=0.3 + ), + ]) + + image_1 = pipe( + prompt="a cat is running", + height=1024, width=1024, + seed=4 + ) + image_1.save("image_5.png") + + image_2 = pipe( + prompt="sunshine, a cat is running", + controlnet_image=image_1, + height=1024, width=1024, + seed=5 + ) + image_2.save("image_6.png") + + +def example_4(): + model_manager = ModelManager(torch_dtype=torch.bfloat16, model_id_list=["FLUX.1-dev", "InstantX/FLUX.1-dev-Controlnet-Union-alpha"]) + pipe = FluxImagePipeline.from_model_manager(model_manager, controlnet_config_units=[ + ControlNetConfigUnit( + processor_id="canny", + model_path="models/ControlNet/InstantX/FLUX.1-dev-Controlnet-Union-alpha/diffusion_pytorch_model.safetensors", + scale=0.3 + ), + ControlNetConfigUnit( + processor_id="depth", + model_path="models/ControlNet/InstantX/FLUX.1-dev-Controlnet-Union-alpha/diffusion_pytorch_model.safetensors", + scale=0.3 + ), + ]) + + image_1 = pipe( + prompt="a beautiful Asian girl, full body, red dress, summer", + height=1024, width=1024, + seed=6 + ) + image_1.save("image_7.png") + + image_2 = pipe( + prompt="a beautiful Asian girl, full body, red dress, winter", + controlnet_image=image_1, + height=1024, width=1024, + seed=7 + ) + image_2.save("image_8.png") + + + +def example_5(): + model_manager = ModelManager(torch_dtype=torch.bfloat16, model_id_list=["FLUX.1-dev", "alimama-creative/FLUX.1-dev-Controlnet-Inpainting-Beta"]) + pipe = FluxImagePipeline.from_model_manager(model_manager, controlnet_config_units=[ + ControlNetConfigUnit( + processor_id="inpaint", + model_path="models/ControlNet/alimama-creative/FLUX.1-dev-Controlnet-Inpainting-Beta/diffusion_pytorch_model.safetensors", + scale=0.9 + ), + ]) + + image_1 = pipe( + prompt="a cat sitting on a chair", + height=1024, width=1024, + seed=8 + ) + image_1.save("image_9.png") + + mask = np.zeros((1024, 1024, 3), dtype=np.uint8) + mask[100:350, 350: -300] = 255 + mask = Image.fromarray(mask) + mask.save("mask_9.png") + + image_2 = pipe( + prompt="a cat sitting on a chair, wearing sunglasses", + controlnet_image=image_1, controlnet_inpaint_mask=mask, + height=1024, width=1024, + seed=9 + ) + image_2.save("image_10.png") + + + +def example_6(): + model_manager = ModelManager(torch_dtype=torch.bfloat16, model_id_list=[ + "FLUX.1-dev", + "jasperai/Flux.1-dev-Controlnet-Surface-Normals", + "alimama-creative/FLUX.1-dev-Controlnet-Inpainting-Beta" + ]) + pipe = FluxImagePipeline.from_model_manager(model_manager, controlnet_config_units=[ + ControlNetConfigUnit( + processor_id="inpaint", + model_path="models/ControlNet/alimama-creative/FLUX.1-dev-Controlnet-Inpainting-Beta/diffusion_pytorch_model.safetensors", + scale=0.9 + ), + ControlNetConfigUnit( + processor_id="normal", + model_path="models/ControlNet/jasperai/Flux.1-dev-Controlnet-Surface-Normals/diffusion_pytorch_model.safetensors", + scale=0.6 + ), + ]) + + image_1 = pipe( + prompt="a beautiful Asian woman looking at the sky, wearing a blue t-shirt.", + height=1024, width=1024, + seed=10 + ) + image_1.save("image_11.png") + + mask = np.zeros((1024, 1024, 3), dtype=np.uint8) + mask[-400:, 10:-40] = 255 + mask = Image.fromarray(mask) + mask.save("mask_11.png") + + image_2 = pipe( + prompt="a beautiful Asian woman looking at the sky, wearing a yellow t-shirt.", + controlnet_image=image_1, controlnet_inpaint_mask=mask, + height=1024, width=1024, + seed=11 + ) + image_2.save("image_12.png") + + +def example_7(): + model_manager = ModelManager(torch_dtype=torch.bfloat16, model_id_list=[ + "FLUX.1-dev", + "InstantX/FLUX.1-dev-Controlnet-Union-alpha", + "alimama-creative/FLUX.1-dev-Controlnet-Inpainting-Beta", + "jasperai/Flux.1-dev-Controlnet-Upscaler", + ]) + pipe = FluxImagePipeline.from_model_manager(model_manager, controlnet_config_units=[ + ControlNetConfigUnit( + processor_id="inpaint", + model_path="models/ControlNet/alimama-creative/FLUX.1-dev-Controlnet-Inpainting-Beta/diffusion_pytorch_model.safetensors", + scale=0.9 + ), + ControlNetConfigUnit( + processor_id="canny", + model_path="models/ControlNet/InstantX/FLUX.1-dev-Controlnet-Union-alpha/diffusion_pytorch_model.safetensors", + scale=0.5 + ), + ]) + + image_1 = pipe( + prompt="a beautiful Asian woman and a cat on a bed. The woman wears a dress.", + height=1024, width=1024, + seed=100 + ) + image_1.save("image_13.png") + + mask_global = np.zeros((1024, 1024, 3), dtype=np.uint8) + mask_global = Image.fromarray(mask_global) + mask_global.save("mask_13_global.png") + + mask_1 = np.zeros((1024, 1024, 3), dtype=np.uint8) + mask_1[300:-100, 30: 450] = 255 + mask_1 = Image.fromarray(mask_1) + mask_1.save("mask_13_1.png") + + mask_2 = np.zeros((1024, 1024, 3), dtype=np.uint8) + mask_2[500:-100, -400:] = 255 + mask_2[-200:-100, -500:-400] = 255 + mask_2 = Image.fromarray(mask_2) + mask_2.save("mask_13_2.png") + + image_2 = pipe( + prompt="a beautiful Asian woman and a cat on a bed. The woman wears a dress.", + controlnet_image=image_1, controlnet_inpaint_mask=mask_global, + local_prompts=["an orange cat, highly detailed", "a girl wearing a red camisole"], masks=[mask_1, mask_2], mask_scales=[10.0, 10.0], + height=1024, width=1024, + seed=101 + ) + image_2.save("image_14.png") + + model_manager.load_lora("models/lora/FLUX-dev-lora-AntiBlur.safetensors", lora_alpha=2) + image_3 = pipe( + prompt="a beautiful Asian woman wearing a red camisole and an orange cat on a bed. clear background.", + negative_prompt="blur, blurry", + input_image=image_2, denoising_strength=0.7, + height=1024, width=1024, + cfg_scale=2.0, num_inference_steps=50, + seed=102 + ) + image_3.save("image_15.png") + + pipe = FluxImagePipeline.from_model_manager(model_manager, controlnet_config_units=[ + ControlNetConfigUnit( + processor_id="tile", + model_path="models/ControlNet/jasperai/Flux.1-dev-Controlnet-Upscaler/diffusion_pytorch_model.safetensors", + scale=0.7 + ), + ]) + image_4 = pipe( + prompt="a beautiful Asian woman wearing a red camisole and an orange cat on a bed. highly detailed, delicate skin texture, clear background.", + controlnet_image=image_3.resize((2048, 2048)), + input_image=image_3.resize((2048, 2048)), denoising_strength=0.99, + height=2048, width=2048, tiled=True, + seed=103 + ) + image_4.save("image_16.png") + + image_5 = pipe( + prompt="a beautiful Asian woman wearing a red camisole and an orange cat on a bed. highly detailed, delicate skin texture, clear background.", + controlnet_image=image_4.resize((4096, 4096)), + input_image=image_4.resize((4096, 4096)), denoising_strength=0.99, + height=4096, width=4096, tiled=True, + seed=104 + ) + image_5.save("image_17.png") + + + +download_models(["Annotators:Depth", "Annotators:Normal"]) +download_customized_models( + model_id="LiblibAI/FLUX.1-dev-LoRA-AntiBlur", + origin_file_path="FLUX-dev-lora-AntiBlur.safetensors", + local_dir="models/lora" +) +example_1() +example_2() +example_3() +example_4() +example_5() +example_6() +example_7() diff --git a/examples/image_synthesis/flux_controlnet.py b/examples/image_synthesis/flux_controlnet.py deleted file mode 100644 index 80be320..0000000 --- a/examples/image_synthesis/flux_controlnet.py +++ /dev/null @@ -1,44 +0,0 @@ -from diffsynth.models.flux_controlnet import FluxControlNet -from diffsynth import load_state_dict, ModelManager, FluxImagePipeline, hash_state_dict_keys, ControlNetConfigUnit -import torch -from PIL import Image -import numpy as np - - -model_manager = ModelManager(torch_dtype=torch.bfloat16, model_id_list=["FLUX.1-dev"]) -model_manager.load_models([ - "models/ControlNet/InstantX/FLUX___1-dev-Controlnet-Union-alpha/diffusion_pytorch_model.safetensors", - "models/ControlNet/jasperai/Flux___1-dev-Controlnet-Depth/diffusion_pytorch_model.safetensors", - "models/ControlNet/jasperai/Flux___1-dev-Controlnet-Surface-Normals/diffusion_pytorch_model.safetensors", - "models/ControlNet/jasperai/Flux___1-dev-Controlnet-Upscaler/diffusion_pytorch_model.safetensors", - "models/ControlNet/alimama-creative/FLUX___1-dev-Controlnet-Inpainting-Alpha/diffusion_pytorch_model.safetensors", - "models/ControlNet/alimama-creative/FLUX___1-dev-Controlnet-Inpainting-Beta/diffusion_pytorch_model.safetensors", - "models/ControlNet/Shakker-Labs/FLUX___1-dev-ControlNet-Depth/diffusion_pytorch_model.safetensors", - "models/ControlNet/Shakker-Labs/FLUX___1-dev-ControlNet-Union-Pro/diffusion_pytorch_model.safetensors" -]) -pipe = FluxImagePipeline.from_model_manager(model_manager, controlnet_config_units=[ - ControlNetConfigUnit(processor_id="canny", model_path="models/ControlNet/InstantX/FLUX___1-dev-Controlnet-Union-alpha/diffusion_pytorch_model.safetensors", scale=0.3), - ControlNetConfigUnit(processor_id="depth", model_path="models/ControlNet/jasperai/Flux___1-dev-Controlnet-Depth/diffusion_pytorch_model.safetensors", scale=0.1), - ControlNetConfigUnit(processor_id="normal", model_path="models/ControlNet/jasperai/Flux___1-dev-Controlnet-Surface-Normals/diffusion_pytorch_model.safetensors", scale=0.1), - ControlNetConfigUnit(processor_id="tile", model_path="models/ControlNet/jasperai/Flux___1-dev-Controlnet-Upscaler/diffusion_pytorch_model.safetensors", scale=0.05), - ControlNetConfigUnit(processor_id="inpaint", model_path="models/ControlNet/alimama-creative/FLUX___1-dev-Controlnet-Inpainting-Alpha/diffusion_pytorch_model.safetensors", scale=0.01), - ControlNetConfigUnit(processor_id="inpaint", model_path="models/ControlNet/alimama-creative/FLUX___1-dev-Controlnet-Inpainting-Beta/diffusion_pytorch_model.safetensors", scale=0.01), - ControlNetConfigUnit(processor_id="depth", model_path="models/ControlNet/Shakker-Labs/FLUX___1-dev-ControlNet-Depth/diffusion_pytorch_model.safetensors", scale=0.05), - ControlNetConfigUnit(processor_id="canny", model_path="models/ControlNet/Shakker-Labs/FLUX___1-dev-ControlNet-Union-Pro/diffusion_pytorch_model.safetensors", scale=0.3), -]) - -torch.manual_seed(0) - -control_image = Image.open("controlnet_input.jpeg").resize((768, 1024)) -control_mask = Image.open("controlnet_mask.jpg").resize((768, 1024)) - -prompt = "masterpiece, best quality, a beautiful girl, CG, blue sky, long red hair, black clothes" -negative_prompt = "oil painting, worst quality, low quality, monochrome, zombie, interlocked fingers, Aissist, cleavage, nsfw," - -image = pipe( - prompt=prompt, negative_prompt=negative_prompt, - embedded_guidance=3.5, num_inference_steps=50, - height=1024, width=768, - controlnet_image=control_image, controlnet_inpaint_mask=control_mask, -) -image.save("image.jpg")