mirror of
https://github.com/modelscope/DiffSynth-Studio.git
synced 2026-03-18 22:08:13 +00:00
681df93a85
* support mova inference * mova media_io * add unified audio_video api & fix bug of mono audio input for ltx * support mova train * mova docs * fix bug
135 lines
5.0 KiB
Python
135 lines
5.0 KiB
Python
import av
|
|
from fractions import Fraction
|
|
import torch
|
|
from PIL import Image
|
|
from tqdm import tqdm
|
|
from .audio import convert_to_stereo
|
|
|
|
|
|
def _resample_audio(
|
|
container: av.container.Container, audio_stream: av.audio.AudioStream, frame_in: av.AudioFrame
|
|
) -> None:
|
|
cc = audio_stream.codec_context
|
|
|
|
# Use the encoder's format/layout/rate as the *target*
|
|
target_format = cc.format or "fltp" # AAC → usually fltp
|
|
target_layout = cc.layout or "stereo"
|
|
target_rate = cc.sample_rate or frame_in.sample_rate
|
|
|
|
audio_resampler = av.audio.resampler.AudioResampler(
|
|
format=target_format,
|
|
layout=target_layout,
|
|
rate=target_rate,
|
|
)
|
|
|
|
audio_next_pts = 0
|
|
for rframe in audio_resampler.resample(frame_in):
|
|
if rframe.pts is None:
|
|
rframe.pts = audio_next_pts
|
|
audio_next_pts += rframe.samples
|
|
rframe.sample_rate = frame_in.sample_rate
|
|
container.mux(audio_stream.encode(rframe))
|
|
|
|
# flush audio encoder
|
|
for packet in audio_stream.encode():
|
|
container.mux(packet)
|
|
|
|
|
|
def _write_audio(
|
|
container: av.container.Container, audio_stream: av.audio.AudioStream, samples: torch.Tensor, audio_sample_rate: int
|
|
) -> None:
|
|
if samples.ndim == 1:
|
|
samples = samples.unsqueeze(0)
|
|
samples = convert_to_stereo(samples)
|
|
assert samples.ndim == 2 and samples.shape[0] == 2, "audio samples must be [C, S] or [S], C must be 1 or 2"
|
|
samples = samples.T
|
|
# Convert to int16 packed for ingestion; resampler converts to encoder fmt.
|
|
if samples.dtype != torch.int16:
|
|
samples = torch.clip(samples, -1.0, 1.0)
|
|
samples = (samples * 32767.0).to(torch.int16)
|
|
|
|
frame_in = av.AudioFrame.from_ndarray(
|
|
samples.contiguous().reshape(1, -1).cpu().numpy(),
|
|
format="s16",
|
|
layout="stereo",
|
|
)
|
|
frame_in.sample_rate = audio_sample_rate
|
|
|
|
_resample_audio(container, audio_stream, frame_in)
|
|
|
|
|
|
def _prepare_audio_stream(container: av.container.Container, audio_sample_rate: int) -> av.audio.AudioStream:
|
|
"""
|
|
Prepare the audio stream for writing.
|
|
"""
|
|
audio_stream = container.add_stream("aac")
|
|
supported_sample_rates = audio_stream.codec_context.codec.audio_rates
|
|
if supported_sample_rates:
|
|
best_rate = min(supported_sample_rates, key=lambda x: abs(x - audio_sample_rate))
|
|
if best_rate != audio_sample_rate:
|
|
print(f"Using closest supported audio sample rate: {best_rate}")
|
|
else:
|
|
best_rate = audio_sample_rate
|
|
audio_stream.codec_context.sample_rate = best_rate
|
|
audio_stream.codec_context.layout = "stereo"
|
|
audio_stream.codec_context.time_base = Fraction(1, best_rate)
|
|
return audio_stream
|
|
|
|
|
|
def write_video_audio(
|
|
video: list[Image.Image],
|
|
audio: torch.Tensor | None,
|
|
output_path: str,
|
|
fps: int = 24,
|
|
audio_sample_rate: int | None = None,
|
|
) -> None:
|
|
"""
|
|
Writes a sequence of images and an audio tensor to a video file.
|
|
|
|
This function utilizes PyAV (or a similar multimedia library) to encode a list of PIL images into a video stream
|
|
and multiplex a PyTorch tensor as the audio stream into the output container.
|
|
|
|
Args:
|
|
video (list[Image.Image]): A list of PIL Image objects representing the video frames.
|
|
The length of this list determines the total duration of the video based on the FPS.
|
|
audio (torch.Tensor | None): The audio data as a PyTorch tensor.
|
|
The shape is typically (channels, samples). If no audio is required, pass None.
|
|
channels can be 1 or 2. 1 for mono, 2 for stereo.
|
|
output_path (str): The file path (including extension) where the output video will be saved.
|
|
fps (int, optional): The frame rate (frames per second) for the video. Defaults to 24.
|
|
audio_sample_rate (int | None, optional): The sample rate (e.g., 44100, 48000) for the audio.
|
|
If the audio tensor is provided and this is None, the function attempts to infer the rate
|
|
based on the audio tensor's length and the video duration.
|
|
Raises:
|
|
ValueError: If an audio tensor is provided but the sample rate cannot be determined.
|
|
"""
|
|
duration = len(video) / fps
|
|
if audio_sample_rate is None:
|
|
audio_sample_rate = int(audio.shape[-1] / duration)
|
|
|
|
width, height = video[0].size
|
|
container = av.open(output_path, mode="w")
|
|
stream = container.add_stream("libx264", rate=int(fps))
|
|
stream.width = width
|
|
stream.height = height
|
|
stream.pix_fmt = "yuv420p"
|
|
|
|
if audio is not None:
|
|
if audio_sample_rate is None:
|
|
raise ValueError("audio_sample_rate is required when audio is provided")
|
|
audio_stream = _prepare_audio_stream(container, audio_sample_rate)
|
|
|
|
for frame in tqdm(video, total=len(video)):
|
|
frame = av.VideoFrame.from_image(frame)
|
|
for packet in stream.encode(frame):
|
|
container.mux(packet)
|
|
|
|
# Flush encoder
|
|
for packet in stream.encode():
|
|
container.mux(packet)
|
|
|
|
if audio is not None:
|
|
_write_audio(container, audio_stream, audio, audio_sample_rate)
|
|
|
|
container.close()
|