wans2v usp

examples/wanvideo/model_inference/Wan2.1-S2V-14B.py

@@ -1,8 +1,9 @@
 import torch
 from PIL import Image
 import librosa
-from diffsynth import save_video, VideoData, save_video_with_audio
+from diffsynth import VideoData, save_video_with_audio
 from diffsynth.pipelines.wan_video_new import WanVideoPipeline, ModelConfig
+from modelscope import dataset_snapshot_download
 
 pipe = WanVideoPipeline.from_pretrained(
     torch_dtype=torch.bfloat16,
@@ -15,21 +16,28 @@ pipe = WanVideoPipeline.from_pretrained(
     ],
     audio_processor_config=ModelConfig(model_id="Wan-AI/Wan2.2-S2V-14B", origin_file_pattern="wav2vec2-large-xlsr-53-english/"),
 )
+dataset_snapshot_download(
+    dataset_id="DiffSynth-Studio/example_video_dataset",
+    local_dir="./data/example_video_dataset",
+    allow_file_pattern=f"wans2v/*"
+)
+
 num_frames = 81 # 4n+1
 height = 448
 width = 832
 
 prompt = "a person is singing"
-input_image = Image.open("/mnt/nas1/zhanghong/project/aigc/Wan2.2_s2v/examples/pose.png").convert("RGB").resize((width, height))
+negative_prompt = "画面模糊，最差质量，画面模糊，细节模糊不清，情绪激动剧烈，手快速抖动，字幕，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走"
+input_image = Image.open("data/example_video_dataset/wans2v/pose.png").convert("RGB").resize((width, height))
 # s2v audio input, recommend 16kHz sampling rate
-audio_path = '/mnt/nas1/zhanghong/project/aigc/Wan2.2_s2v/examples/sing.MP3'
+audio_path = 'data/example_video_dataset/wans2v/sing.MP3'
 input_audio, sample_rate = librosa.load(audio_path, sr=16000)
 
 # Speech-to-video
 video = pipe(
     prompt=prompt,
     input_image=input_image,
-    negative_prompt="",
+    negative_prompt=negative_prompt,
     seed=0,
     num_frames=num_frames,
     height=height,
@@ -38,18 +46,17 @@ video = pipe(
     input_audio=input_audio,
     num_inference_steps=40,
 )
-save_video_with_audio(video, "video_with_audio.mp4", audio_path, fps=16, quality=5)
+save_video_with_audio(video[1:], "video_with_audio.mp4", audio_path, fps=16, quality=5)
 
 # s2v will use the first (num_frames) frames as reference. height and width must be the same as input_image. And fps should be 16, the same as output video fps.
-pose_video_path = '/mnt/nas1/zhanghong/project/aigc/Wan2.2_s2v/examples/pose.mp4'
+pose_video_path = 'data/example_video_dataset/wans2v/pose.mp4'
 pose_video = VideoData(pose_video_path, height=height, width=width)
-pose_video.set_length(num_frames)
 
 # Speech-to-video with pose
 video = pipe(
     prompt=prompt,
     input_image=input_image,
-    negative_prompt="",
+    negative_prompt=negative_prompt,
     seed=0,
     num_frames=num_frames,
     height=height,
@@ -59,5 +66,4 @@ video = pipe(
     s2v_pose_video=pose_video,
     num_inference_steps=40,
 )
-save_video_with_audio(video, "video_pose_with_audio.mp4", audio_path, fps=16, quality=5)
-save_video(pose_video, "video_pose_input.mp4", fps=16, quality=5)
+save_video_with_audio(video[1:], "video_pose_with_audio.mp4", audio_path, fps=16, quality=5)