theluyuan/DiffSynth-Studio
1
0
Fork 0
mirror of https://github.com/modelscope/DiffSynth-Studio.git synced 2026-03-18 22:08:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix bugs

Browse Source
This commit is contained in:
Artiprocher
2024-07-26 17:51:03 +08:00
parent e3f8a576cf
commit 9ed54c188e
4 changed files with 7 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
examples/ExVideo/ExVideo_svd_train.py
View File

@@ -4,7 +4,7 @@ import numpy as np
from einops import rearrange, repeat
import lightning as pl
from diffsynth import ModelManager, SVDImageEncoder, SVDUNet, SVDVAEEncoder, ContinuousODEScheduler, load_state_dict
from diffsynth.pipelines.stable_video_diffusion import SVDCLIPImageProcessor
from diffsynth.pipelines.svd_video import SVDCLIPImageProcessor
from diffsynth.models.svd_unet import TemporalAttentionBlock
@@ -131,14 +131,14 @@ class LightningModel(pl.LightningModule):
self.image_encoder.requires_grad_(False)
self.unet = SVDUNet(add_positional_conv=add_positional_conv).to(dtype=torch.float16, device=self.device)
self.unet.load_state_dict(SVDUNet.state_dict_converter().from_civitai(state_dict), strict=False)
self.unet.load_state_dict(SVDUNet.state_dict_converter().from_civitai(state_dict, add_positional_conv=add_positional_conv), strict=False)
self.unet.train()
self.unet.requires_grad_(False)
for block in self.unet.blocks:
if isinstance(block, TemporalAttentionBlock):
block.requires_grad_(True)
self.vae_encoder = SVDVAEEncoder.to(dtype=torch.float16, device=self.device)
self.vae_encoder = SVDVAEEncoder().to(dtype=torch.float16, device=self.device)
self.vae_encoder.load_state_dict(SVDVAEEncoder.state_dict_converter().from_civitai(state_dict))
self.vae_encoder.eval()
self.vae_encoder.requires_grad_(False)

32
examples/image_synthesis/sd3_text_to_image_textual_inversion.py
View File

@@ -1,32 +0,0 @@
from diffsynth import ModelManager, SD3ImagePipeline, download_models, load_state_dict
import torch
# Download models (automatically)
# `models/stable_diffusion_3/sd3_medium_incl_clips.safetensors`: [link](https://huggingface.co/stabilityai/stable-diffusion-3-medium/resolve/main/sd3_medium_incl_clips.safetensors)
# `models/textual_inversion/verybadimagenegative_v1.3.pt`: [link](https://civitai.com/api/download/models/25820?type=Model&format=PickleTensor&size=full&fp=fp16)
download_models(["StableDiffusion3_without_T5", "TextualInversion_VeryBadImageNegative_v1.3"])
model_manager = ModelManager(torch_dtype=torch.float16, device="cuda")
model_manager.load_textual_inversions("models/textual_inversion")
model_manager.load_models(["models/stable_diffusion_3/sd3_medium_incl_clips.safetensors"])
pipe = SD3ImagePipeline.from_model_manager(model_manager)
for seed in range(4):
torch.manual_seed(seed)
image = pipe(
prompt="a girl, highly detailed, absurd res, perfect image",
negative_prompt="verybadimagenegative_v1.3",
cfg_scale=4.5,
num_inference_steps=50, width=1024, height=1024,
)
image.save(f"image_with_textual_inversion_{seed}.jpg")
torch.manual_seed(seed)
image = pipe(
prompt="a girl, highly detailed, absurd res, perfect image",
negative_prompt="",
cfg_scale=4.5,
num_inference_steps=50, width=1024, height=1024,
)
image.save(f"image_without_textual_inversion_{seed}.jpg")