theluyuan/DiffSynth-Studio
1
0
Fork 0
mirror of https://github.com/modelscope/DiffSynth-Studio.git synced 2026-03-18 22:08:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

update smoothers

Browse Source
This commit is contained in:
Artiprocher
2024-02-05 01:43:55 +08:00
parent e1763d11ac
commit eab2dcbf7b
7 changed files with 276 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
examples/sd_video_rerender.py
View File

@@ -1,5 +1,7 @@
from diffsynth import ModelManager, SDVideoPipeline, ControlNetConfigUnit, VideoData, save_video
from diffsynth.extensions.FastBlend import FastBlendSmoother
from diffsynth.processors.FastBlend import FastBlendSmoother
from diffsynth.processors.PILEditor import ContrastEditor, SharpnessEditor
from diffsynth.processors.sequencial_processor import SequencialProcessor
import torch
@@ -9,16 +11,13 @@ import torch
# `models/ControlNet/control_v11p_sd15_softedge.pth`: [link](https://huggingface.co/lllyasviel/ControlNet-v1-1/resolve/main/control_v11p_sd15_softedge.pth)
# `models/Annotators/dpt_hybrid-midas-501f0c75.pt`: [link](https://huggingface.co/lllyasviel/Annotators/resolve/main/dpt_hybrid-midas-501f0c75.pt)
# `models/Annotators/ControlNetHED.pth`: [link](https://huggingface.co/lllyasviel/Annotators/resolve/main/ControlNetHED.pth)
# `models/RIFE/flownet.pkl`: [link](https://drive.google.com/file/d/1APIzVeI-4ZZCEuIRE1m6WYfSCaOsi_7_/view?usp=sharing)
# Load models
model_manager = ModelManager(torch_dtype=torch.float16, device="cuda")
model_manager.load_models([
"models/stable_diffusion/dreamshaper_8.safetensors",
"models/ControlNet/control_v11f1p_sd15_depth.pth",
"models/ControlNet/control_v11p_sd15_softedge.pth",
"models/RIFE/flownet.pkl"
"models/ControlNet/control_v11p_sd15_softedge.pth"
])
pipe = SDVideoPipeline.from_model_manager(
model_manager,
@@ -35,7 +34,7 @@ pipe = SDVideoPipeline.from_model_manager(
)
]
)
smoother = FastBlendSmoother.from_model_manager(model_manager)
smoother = SequencialProcessor([FastBlendSmoother(), ContrastEditor(rate=1.1), SharpnessEditor(rate=1.1)])
# Load video
# Original video: https://pixabay.com/videos/flow-rocks-water-fluent-stones-159627/
@@ -48,10 +47,10 @@ output_video = pipe(
prompt="winter, ice, snow, water, river",
negative_prompt="", cfg_scale=7,
input_frames=input_video, controlnet_frames=input_video, num_frames=len(input_video),
num_inference_steps=10, height=512, width=768,
animatediff_batch_size=32, animatediff_stride=16, unet_batch_size=4,
num_inference_steps=20, height=512, width=768,
animatediff_batch_size=8, animatediff_stride=4, unet_batch_size=8,
cross_frame_attention=True,
smoother=smoother, smoother_progress_ids=[4, 9]
smoother=smoother, smoother_progress_ids=[4, 9, 14, 19]
)
# Save images and video