theluyuan/DiffSynth-Studio
1
0
Fork 0
mirror of https://github.com/modelscope/DiffSynth-Studio.git synced 2026-03-24 10:18:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

add wans2v example

Browse Source
This commit is contained in:
mi804
2025-08-29 15:11:30 +08:00
parent ff71720297
commit a0bae07825
1 changed files with 39 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
examples/wanvideo/model_inference/Wan2.2-S2V-14B.py
View File

@@ -1,8 +1,9 @@
import torch import torch
from PIL import Image from PIL import Image
import librosa import librosa
from diffsynth import save_video, VideoData, save_video_with_audio from diffsynth import VideoData, save_video_with_audio
from diffsynth.pipelines.wan_video_new import WanVideoPipeline, ModelConfig from diffsynth.pipelines.wan_video_new import WanVideoPipeline, ModelConfig
from modelscope import dataset_snapshot_download
pipe = WanVideoPipeline.from_pretrained( pipe = WanVideoPipeline.from_pretrained(
torch_dtype=torch.bfloat16, torch_dtype=torch.bfloat16,
@@ -15,24 +16,54 @@ pipe = WanVideoPipeline.from_pretrained(
], ],
audio_processor_config=ModelConfig(model_id="Wan-AI/Wan2.2-S2V-14B", origin_file_pattern="wav2vec2-large-xlsr-53-english/"), audio_processor_config=ModelConfig(model_id="Wan-AI/Wan2.2-S2V-14B", origin_file_pattern="wav2vec2-large-xlsr-53-english/"),
) )
dataset_snapshot_download(
dataset_id="DiffSynth-Studio/example_video_dataset",
local_dir="./data/example_video_dataset",
allow_file_pattern=f"wans2v/*"
)
num_frames = 81 # 4n+1
height = 448
width = 832
prompt = "a person is singing" prompt = "a person is singing"
input_image = Image.open("/mnt/nas1/zhanghong/project/aigc/Wan2.2_s2v/examples/pose.png").convert("RGB").resize((width, height)) negative_prompt = "画面模糊，最差质量，画面模糊，细节模糊不清，情绪激动剧烈，手快速抖动，字幕，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走"
input_image = Image.open("data/example_video_dataset/wans2v/pose.png").convert("RGB").resize((width, height))
# s2v audio input, recommend 16kHz sampling rate # s2v audio input, recommend 16kHz sampling rate
audio_path = '/mnt/nas1/zhanghong/project/aigc/Wan2.2_s2v/examples/sing.MP3' audio_path = 'data/example_video_dataset/wans2v/sing.MP3'
input_audio, sample_rate = librosa.load(audio_path, sr=16000) input_audio, sample_rate = librosa.load(audio_path, sr=16000)
# Speech-to-video # Speech-to-video
video = pipe( video = pipe(
prompt=prompt, prompt=prompt,
input_image=input_image, input_image=input_image,
negative_prompt="", negative_prompt=negative_prompt,
seed=0, seed=0,
num_frames=81, num_frames=num_frames,
height=1280, height=height,
width=720, width=width,
audio_sample_rate=sample_rate, audio_sample_rate=sample_rate,
input_audio=input_audio, input_audio=input_audio,
num_inference_steps=40, num_inference_steps=40,
) )
save_video_with_audio(video, "video_with_audio.mp4", audio_path, fps=16, quality=5) save_video_with_audio(video[1:], "video_with_audio.mp4", audio_path, fps=16, quality=5)
# s2v will use the first (num_frames) frames as reference. height and width must be the same as input_image. And fps should be 16, the same as output video fps.
pose_video_path = 'data/example_video_dataset/wans2v/pose.mp4'
pose_video = VideoData(pose_video_path, height=height, width=width)
# Speech-to-video with pose
video = pipe(
prompt=prompt,
input_image=input_image,
negative_prompt=negative_prompt,
seed=0,
num_frames=num_frames,
height=height,
width=width,
audio_sample_rate=sample_rate,
input_audio=input_audio,
s2v_pose_video=pose_video,
num_inference_steps=40,
)
save_video_with_audio(video[1:], "video_pose_with_audio.mp4", audio_path, fps=16, quality=5)