compatibility update

examples/sd_text_to_image.py

@@ -0,0 +1,75 @@
+from diffsynth import ModelManager, SDImagePipeline, ControlNetConfigUnit
+import torch
+
+
+# Download models
+# `models/stable_diffusion/aingdiffusion_v12.safetensors`: [link](https://civitai.com/api/download/models/229575?type=Model&format=SafeTensor&size=full&fp=fp16)
+# `models/ControlNet/control_v11p_sd15_lineart.pth`: [link](https://huggingface.co/lllyasviel/ControlNet-v1-1/resolve/main/control_v11p_sd15_lineart.pth)
+# `models/ControlNet/control_v11f1e_sd15_tile.pth`: [link](https://huggingface.co/lllyasviel/ControlNet-v1-1/resolve/main/control_v11f1e_sd15_tile.pth)
+# `models/Annotators/sk_model.pth`: [link](https://huggingface.co/lllyasviel/Annotators/resolve/main/sk_model.pth)
+# `models/Annotators/sk_model2.pth`: [link](https://huggingface.co/lllyasviel/Annotators/resolve/main/sk_model2.pth)
+
+
+# Load models
+model_manager = ModelManager(torch_dtype=torch.float16, device="cuda")
+model_manager.load_textual_inversions("models/textual_inversion")
+model_manager.load_models([
+    "models/stable_diffusion/aingdiffusion_v12.safetensors",
+    "models/ControlNet/control_v11f1e_sd15_tile.pth",
+    "models/ControlNet/control_v11p_sd15_lineart.pth"
+])
+pipe = SDImagePipeline.from_model_manager(
+    model_manager,
+    [
+        ControlNetConfigUnit(
+            processor_id="tile",
+            model_path=rf"models/ControlNet/control_v11f1e_sd15_tile.pth",
+            scale=0.5
+        ),
+        ControlNetConfigUnit(
+            processor_id="lineart",
+            model_path=rf"models/ControlNet/control_v11p_sd15_lineart.pth",
+            scale=0.7
+        ),
+    ]
+)
+
+prompt = "masterpiece, best quality, solo, long hair, wavy hair, silver hair, blue eyes, blue dress, medium breasts, dress, underwater, air bubble, floating hair, refraction, portrait,"
+negative_prompt = "worst quality, low quality, monochrome, zombie, interlocked fingers, Aissist, cleavage, nsfw,",
+
+torch.manual_seed(0)
+image = pipe(
+    prompt=prompt,
+    negative_prompt=negative_prompt,
+    cfg_scale=7.5, clip_skip=1,
+    height=512, width=512, num_inference_steps=80,
+)
+image.save("512.jpg")
+
+image = pipe(
+    prompt=prompt,
+    negative_prompt=negative_prompt,
+    cfg_scale=7.5, clip_skip=1,
+    input_image=image.resize((1024, 1024)), controlnet_image=image.resize((1024, 1024)),
+    height=1024, width=1024, num_inference_steps=40, denoising_strength=0.7,
+)
+image.save("1024.jpg")
+
+image = pipe(
+    prompt=prompt,
+    negative_prompt=negative_prompt,
+    cfg_scale=7.5, clip_skip=1,
+    input_image=image.resize((2048, 2048)), controlnet_image=image.resize((2048, 2048)),
+    height=2048, width=2048, num_inference_steps=20, denoising_strength=0.7,
+)
+image.save("2048.jpg")
+
+image = pipe(
+    prompt=prompt,
+    negative_prompt=negative_prompt,
+    cfg_scale=7.5, clip_skip=1,
+    input_image=image.resize((4096, 4096)), controlnet_image=image.resize((4096, 4096)),
+    height=4096, width=4096, num_inference_steps=10, denoising_strength=0.5,
+    tiled=True, tile_size=128, tile_stride=64
+)
+image.save("4096.jpg")

examples/sd_toon_shading.py

@@ -0,0 +1,56 @@
+from diffsynth import ModelManager, SDVideoPipeline, ControlNetConfigUnit, VideoData, save_video, save_frames
+import torch
+
+
+# Download models
+# `models/stable_diffusion/flat2DAnimerge_v45Sharp.safetensors`: [link](https://civitai.com/api/download/models/266360?type=Model&format=SafeTensor&size=pruned&fp=fp16)
+# `models/AnimateDiff/mm_sd_v15_v2.ckpt`: [link](https://huggingface.co/guoyww/animatediff/resolve/main/mm_sd_v15_v2.ckpt)
+# `models/ControlNet/control_v11p_sd15_lineart.pth`: [link](https://huggingface.co/lllyasviel/ControlNet-v1-1/resolve/main/control_v11p_sd15_lineart.pth)
+# `models/ControlNet/control_v11f1e_sd15_tile.pth`: [link](https://huggingface.co/lllyasviel/ControlNet-v1-1/resolve/main/control_v11f1e_sd15_tile.pth)
+# `models/Annotators/sk_model.pth`: [link](https://huggingface.co/lllyasviel/Annotators/resolve/main/sk_model.pth)
+# `models/Annotators/sk_model2.pth`: [link](https://huggingface.co/lllyasviel/Annotators/resolve/main/sk_model2.pth)
+
+
+# Load models
+model_manager = ModelManager(torch_dtype=torch.float16, device="cuda")
+model_manager.load_textual_inversions("models/textual_inversion")
+model_manager.load_models([
+    "models/stable_diffusion/flat2DAnimerge_v45Sharp.safetensors",
+    "models/AnimateDiff/mm_sd_v15_v2.ckpt",
+    "models/ControlNet/control_v11p_sd15_lineart.pth",
+    "models/ControlNet/control_v11f1e_sd15_tile.pth",
+])
+pipe = SDVideoPipeline.from_model_manager(
+    model_manager,
+    [
+        ControlNetConfigUnit(
+            processor_id="lineart",
+            model_path="models/ControlNet/control_v11p_sd15_lineart.pth",
+            scale=1.0
+        ),
+        ControlNetConfigUnit(
+            processor_id="tile",
+            model_path="models/ControlNet/control_v11f1e_sd15_tile.pth",
+            scale=0.5
+        ),
+    ]
+)
+
+# Load video (we only use 16 frames in this example for testing)
+video = VideoData(video_file="input_video.mp4", height=1536, width=1536)
+input_video = [video[i] for i in range(16)]
+
+# Toon shading
+torch.manual_seed(0)
+output_video = pipe(
+    prompt="best quality, perfect anime illustration, light, a girl is dancing, smile, solo",
+    negative_prompt="verybadimagenegative_v1.3",
+    cfg_scale=5, clip_skip=2,
+    controlnet_frames=input_video, num_frames=len(input_video),
+    num_inference_steps=10, height=1536, width=1536,
+    vram_limit_level=0,
+)
+
+# Save images and video
+save_frames(output_video, "output_frames")
+save_video(output_video, "output_video.mp4", fps=16)

examples/sdxl_text_to_image.py

@@ -0,0 +1,34 @@
+from diffsynth import ModelManager, SDXLImagePipeline
+import torch
+
+
+# Download models
+# `models/stable_diffusion_xl/bluePencilXL_v200.safetensors`: [link](https://civitai.com/api/download/models/245614?type=Model&format=SafeTensor&size=pruned&fp=fp16)
+
+
+# Load models
+model_manager = ModelManager(torch_dtype=torch.float16, device="cuda")
+model_manager.load_models(["models/stable_diffusion_xl/bluePencilXL_v200.safetensors"])
+pipe = SDXLImagePipeline.from_model_manager(model_manager)
+
+prompt = "masterpiece, best quality, solo, long hair, wavy hair, silver hair, blue eyes, blue dress, medium breasts, dress, underwater, air bubble, floating hair, refraction, portrait,"
+negative_prompt = "worst quality, low quality, monochrome, zombie, interlocked fingers, Aissist, cleavage, nsfw,",
+
+torch.manual_seed(0)
+image = pipe(
+    prompt=prompt,
+    negative_prompt=negative_prompt,
+    cfg_scale=6,
+    height=1024, width=1024, num_inference_steps=60,
+)
+image.save("1024.jpg")
+
+image = pipe(
+    prompt=prompt,
+    negative_prompt=negative_prompt,
+    cfg_scale=6,
+    input_image=image.resize((2048, 2048)),
+    height=2048, width=2048, num_inference_steps=60, denoising_strength=0.5
+)
+image.save("2048.jpg")
+

examples/sdxl_turbo.py

@@ -0,0 +1,31 @@
+from diffsynth import ModelManager, SDXLImagePipeline
+import torch
+
+
+# Download models
+# `models/stable_diffusion_xl_turbo/sd_xl_turbo_1.0_fp16.safetensors`: [link](https://huggingface.co/stabilityai/sdxl-turbo/resolve/main/sd_xl_turbo_1.0_fp16.safetensors)
+
+
+# Load models
+model_manager = ModelManager(torch_dtype=torch.float16, device="cuda")
+model_manager.load_models(["models/stable_diffusion_xl_turbo/sd_xl_turbo_1.0_fp16.safetensors"])
+pipe = SDXLImagePipeline.from_model_manager(model_manager)
+
+# Text to image
+torch.manual_seed(0)
+image = pipe(
+    prompt="black car",
+    # Do not modify the following parameters!
+    cfg_scale=1, height=512, width=512, num_inference_steps=1, progress_bar_cmd=lambda x:x
+)
+image.save(f"black_car.jpg")
+
+# Image to image
+torch.manual_seed(0)
+image = pipe(
+    prompt="red car",
+    input_image=image, denoising_strength=0.7,
+    # Do not modify the following parameters!
+    cfg_scale=1, height=512, width=512, num_inference_steps=1, progress_bar_cmd=lambda x:x
+)
+image.save(f"black_car_to_red_car.jpg")