theluyuan/DiffSynth-Studio
1
0
Fork 0
mirror of https://github.com/modelscope/DiffSynth-Studio.git synced 2026-03-18 22:08:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Mova (#1337)

Browse Source 
* support mova inference

* mova media_io

* add unified audio_video api & fix bug of mono audio input for ltx

* support mova train

* mova docs

* fix bug
This commit is contained in:
Hong Zhang
2026-03-13 13:06:07 +08:00
committed by GitHub
parent 4741542523
commit 681df93a85
37 changed files with 3102 additions and 181 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
examples/ltx2/model_inference/LTX-2.3-A2V-TwoStage.py
View File

@@ -1,6 +1,7 @@
import torch
from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig
from diffsynth.utils.data.media_io_ltx2 import read_audio_with_torchaudio, write_video_audio_ltx2
from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2
from diffsynth.utils.data.audio import read_audio
from modelscope import dataset_snapshot_download
vram_config = {
@@ -42,7 +43,7 @@ negative_prompt = (
)
height, width, num_frames, frame_rate = 512 * 2, 768 * 2, 121, 24
duration = num_frames / frame_rate
audio, audio_sample_rate = read_audio_with_torchaudio("data/example_video_dataset/ltx2/sing.MP3", start_time=1, duration=duration)
audio, audio_sample_rate = read_audio("data/example_video_dataset/ltx2/sing.MP3", start_time=1, duration=duration)
video, audio = pipe(
prompt=prompt,
negative_prompt=negative_prompt,

5
examples/ltx2/model_inference/LTX-2.3-T2AV-TwoStage-Retake.py
View File

@@ -1,6 +1,7 @@
import torch
from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig
from diffsynth.utils.data.media_io_ltx2 import read_audio_with_torchaudio, write_video_audio_ltx2
from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2
from diffsynth.utils.data.audio import read_audio
from modelscope import dataset_snapshot_download
from diffsynth.utils.data import VideoData
@@ -47,7 +48,7 @@ path = "data/example_video_dataset/ltx2/video2.mp4"
video = VideoData(path, height=height, width=width).raw_data()[:num_frames]
assert len(video) == num_frames, f"Input video has {len(video)} frames, but expected {num_frames} frames based on the specified num_frames argument."
duration = num_frames / frame_rate
audio, audio_sample_rate = read_audio_with_torchaudio(path)
audio, audio_sample_rate = read_audio(path)
# Regenerate the video within time regions. You can specify different time regions for video frames and audio retake.
# retake regions are in seconds, and the example below retakes video frames in the time regions of [1s, 2s] and [3s, 4s], and retakes audio in the time regions of [0s, 1s] and [4s, 5s].

5
examples/ltx2/model_inference_low_vram/LTX-2.3-A2V-TwoStage.py
View File

@@ -1,6 +1,7 @@
import torch
from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig
from diffsynth.utils.data.media_io_ltx2 import read_audio_with_torchaudio, write_video_audio_ltx2
from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2
from diffsynth.utils.data.audio import read_audio
from modelscope import dataset_snapshot_download
vram_config = {
@@ -43,7 +44,7 @@ negative_prompt = (
)
height, width, num_frames, frame_rate = 512 * 2, 768 * 2, 121, 24
duration = num_frames / frame_rate
audio, audio_sample_rate = read_audio_with_torchaudio("data/example_video_dataset/ltx2/sing.MP3", start_time=1, duration=duration)
audio, audio_sample_rate = read_audio("data/example_video_dataset/ltx2/sing.MP3", start_time=1, duration=duration)
video, audio = pipe(
prompt=prompt,
negative_prompt=negative_prompt,

5
examples/ltx2/model_inference_low_vram/LTX-2.3-T2AV-TwoStage-Retake.py
View File

@@ -1,6 +1,7 @@
import torch
from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig
from diffsynth.utils.data.media_io_ltx2 import read_audio_with_torchaudio, write_video_audio_ltx2
from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2
from diffsynth.utils.data.audio import read_audio
from modelscope import dataset_snapshot_download
from diffsynth.utils.data import VideoData
@@ -48,7 +49,7 @@ path = "data/example_video_dataset/ltx2/video2.mp4"
video = VideoData(path, height=height, width=width).raw_data()[:num_frames]
assert len(video) == num_frames, f"Input video has {len(video)} frames, but expected {num_frames} frames based on the specified num_frames argument."
duration = num_frames / frame_rate
audio, audio_sample_rate = read_audio_with_torchaudio(path)
audio, audio_sample_rate = read_audio(path)
# Regenerate the video within time regions. You can specify different time regions for video frames and audio retake.
# retake regions are in seconds, and the example below retakes video frames in the time regions of [1s, 2s] and [3s, 4s], and retakes audio in the time regions of [0s, 1s] and [4s, 5s].