rearrange examples

examples/diffsynth/README.md

@@ -0,0 +1,7 @@
+# DiffSynth
+
+DiffSynth is the initial version of our video synthesis framework. In this framework, you can apply video deflickering algorithms to the latent space of diffusion models. You can refer to the [original repo](https://github.com/alibaba/EasyNLP/tree/master/diffusion/DiffSynth) for more details.
+
+We provide an example for video stylization. In this pipeline, the rendered video is completely different from the original video, thus we need a powerful deflickering algorithm. We use FastBlend to implement the deflickering module. Please see [`sd_video_rerender.py`](./sd_video_rerender.py).
+
+https://github.com/Artiprocher/DiffSynth-Studio/assets/35051019/59fb2f7b-8de0-4481-b79f-0c3a7361a1ea

examples/diffsynth/sd_video_rerender.py

@@ -0,0 +1,57 @@
+from diffsynth import ModelManager, SDVideoPipeline, ControlNetConfigUnit, VideoData, save_video
+from diffsynth.processors.FastBlend import FastBlendSmoother
+from diffsynth.processors.PILEditor import ContrastEditor, SharpnessEditor
+from diffsynth.processors.sequencial_processor import SequencialProcessor
+import torch
+
+
+# Download models
+# `models/stable_diffusion/dreamshaper_8.safetensors`: [link](https://civitai.com/api/download/models/128713?type=Model&format=SafeTensor&size=pruned&fp=fp16)
+# `models/ControlNet/control_v11f1p_sd15_depth.pth`: [link](https://huggingface.co/lllyasviel/ControlNet-v1-1/resolve/main/control_v11f1p_sd15_depth.pth)
+# `models/ControlNet/control_v11p_sd15_softedge.pth`: [link](https://huggingface.co/lllyasviel/ControlNet-v1-1/resolve/main/control_v11p_sd15_softedge.pth)
+# `models/Annotators/dpt_hybrid-midas-501f0c75.pt`: [link](https://huggingface.co/lllyasviel/Annotators/resolve/main/dpt_hybrid-midas-501f0c75.pt)
+# `models/Annotators/ControlNetHED.pth`: [link](https://huggingface.co/lllyasviel/Annotators/resolve/main/ControlNetHED.pth)
+
+# Load models
+model_manager = ModelManager(torch_dtype=torch.float16, device="cuda")
+model_manager.load_models([
+    "models/stable_diffusion/dreamshaper_8.safetensors",
+    "models/ControlNet/control_v11f1p_sd15_depth.pth",
+    "models/ControlNet/control_v11p_sd15_softedge.pth"
+])
+pipe = SDVideoPipeline.from_model_manager(
+    model_manager,
+    [
+        ControlNetConfigUnit(
+            processor_id="depth",
+            model_path=rf"models/ControlNet/control_v11f1p_sd15_depth.pth",
+            scale=0.5
+        ),
+        ControlNetConfigUnit(
+            processor_id="softedge",
+            model_path=rf"models/ControlNet/control_v11p_sd15_softedge.pth",
+            scale=0.5
+        )
+    ]
+)
+smoother = SequencialProcessor([FastBlendSmoother(), ContrastEditor(rate=1.1), SharpnessEditor(rate=1.1)])
+
+# Load video
+# Original video: https://pixabay.com/videos/flow-rocks-water-fluent-stones-159627/
+video = VideoData(video_file="data/pixabay100/159627 (1080p).mp4", height=512, width=768)
+input_video = [video[i] for i in range(128)]
+
+# Rerender
+torch.manual_seed(0)
+output_video = pipe(
+    prompt="winter, ice, snow, water, river",
+    negative_prompt="", cfg_scale=7,
+    input_frames=input_video, controlnet_frames=input_video, num_frames=len(input_video),
+    num_inference_steps=20, height=512, width=768,
+    animatediff_batch_size=8, animatediff_stride=4, unet_batch_size=8,
+    cross_frame_attention=True,
+    smoother=smoother, smoother_progress_ids=[4, 9, 14, 19]
+)
+
+# Save images and video
+save_video(output_video, "output_video.mp4", fps=30)