theluyuan/DiffSynth-Studio
1
0
Fork 0
mirror of https://github.com/modelscope/DiffSynth-Studio.git synced 2026-03-24 10:18:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

hunyuanvideo step_processor

Browse Source
This commit is contained in:
Artiprocher
2024-12-26 10:13:46 +08:00
parent e38ccf4c2f
commit 6a999e1127
2 changed files with 21 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
diffsynth/pipelines/hunyuan_video.py
View File

@@ -96,6 +96,7 @@ class HunyuanVideoPipeline(BasePipeline):
num_inference_steps=30, num_inference_steps=30,
tile_size=(17, 30, 30), tile_size=(17, 30, 30),
tile_stride=(12, 20, 20), tile_stride=(12, 20, 20),
step_processor=None,
progress_bar_cmd=lambda x: x, progress_bar_cmd=lambda x: x,
progress_bar_st=None, progress_bar_st=None,
): ):
@@ -140,6 +141,20 @@ class HunyuanVideoPipeline(BasePipeline):
else: else:
noise_pred = noise_pred_posi noise_pred = noise_pred_posi
# (Experimental feature, may be removed in the future)
if step_processor is not None:
self.load_models_to_device(['vae_decoder'])
rendered_frames = self.scheduler.step(noise_pred, self.scheduler.timesteps[progress_id], latents, to_final=True)
rendered_frames = self.vae_decoder.decode_video(rendered_frames, **tiler_kwargs)
rendered_frames = self.tensor2video(rendered_frames[0])
rendered_frames = step_processor(rendered_frames, original_frames=input_video)
self.load_models_to_device(['vae_encoder'])
rendered_frames = self.preprocess_images(rendered_frames)
rendered_frames = torch.stack(rendered_frames, dim=2)
target_latents = self.encode_video(rendered_frames).to(dtype=self.torch_dtype, device=self.device)
noise_pred = self.scheduler.return_to_timestep(self.scheduler.timesteps[progress_id], latents, target_latents)
self.load_models_to_device([] if self.vram_management else ["dit"])
# Scheduler # Scheduler
latents = self.scheduler.step(noise_pred, self.scheduler.timesteps[progress_id], latents) latents = self.scheduler.step(noise_pred, self.scheduler.timesteps[progress_id], latents)

7
diffsynth/schedulers/flow_match.py
View File

@@ -47,7 +47,12 @@ class FlowMatchScheduler():
def return_to_timestep(self, timestep, sample, sample_stablized): def return_to_timestep(self, timestep, sample, sample_stablized):
# This scheduler doesn't support this function. # This scheduler doesn't support this function.
pass if isinstance(timestep, torch.Tensor):
timestep = timestep.cpu()
timestep_id = torch.argmin((self.timesteps - timestep).abs())
sigma = self.sigmas[timestep_id]
model_output = (sample - sample_stablized) / sigma
return model_output
def add_noise(self, original_samples, noise, timestep): def add_noise(self, original_samples, noise, timestep):