from ..models import ModelManager, SDTextEncoder, SDUNet, SDVAEDecoder, SDVAEEncoder from ..controlnets.controlnet_unit import MultiControlNetManager from ..prompts import SDPrompter from ..schedulers import EnhancedDDIMScheduler import torch from tqdm import tqdm from PIL import Image import numpy as np class SDPipeline(torch.nn.Module): def __init__(self, device="cuda", torch_dtype=torch.float16): super().__init__() self.scheduler = EnhancedDDIMScheduler() self.prompter = SDPrompter() self.device = device self.torch_dtype = torch_dtype # models self.text_encoder: SDTextEncoder = None self.unet: SDUNet = None self.vae_decoder: SDVAEDecoder = None self.vae_encoder: SDVAEEncoder = None self.controlnet: MultiControlNetManager = None def fetch_main_models(self, model_manager: ModelManager): self.text_encoder = model_manager.text_encoder self.unet = model_manager.unet self.vae_decoder = model_manager.vae_decoder self.vae_encoder = model_manager.vae_encoder # load textual inversion self.prompter.load_textual_inversion(model_manager.textual_inversion_dict) def fetch_controlnet_models(self, controlnet_units=[]): self.controlnet = MultiControlNetManager(controlnet_units) def preprocess_image(self, image): image = torch.Tensor(np.array(image, dtype=np.float32) * (2 / 255) - 1).permute(2, 0, 1).unsqueeze(0) return image @torch.no_grad() def __call__( self, prompt, negative_prompt="", cfg_scale=7.5, clip_skip=1, init_image=None, controlnet_image=None, denoising_strength=1.0, height=512, width=512, num_inference_steps=20, tiled=False, tile_size=64, tile_stride=32, progress_bar_cmd=tqdm, progress_bar_st=None, ): # Encode prompts prompt_emb_posi = self.prompter.encode_prompt(self.text_encoder, prompt, clip_skip=clip_skip, device=self.device) prompt_emb_nega = self.prompter.encode_prompt(self.text_encoder, negative_prompt, clip_skip=clip_skip, device=self.device) # Prepare scheduler self.scheduler.set_timesteps(num_inference_steps, denoising_strength) # Prepare latent tensors if init_image is not None: image = self.preprocess_image(init_image).to(device=self.device, dtype=self.torch_dtype) latents = self.vae_encoder(image, tiled=tiled, tile_size=tile_size, tile_stride=tile_stride) noise = torch.randn((1, 4, height//8, width//8), device=self.device, dtype=self.torch_dtype) latents = self.scheduler.add_noise(latents, noise, timestep=self.scheduler.timesteps[0]) else: latents = torch.randn((1, 4, height//8, width//8), device=self.device, dtype=self.torch_dtype) # Prepare ControlNets if controlnet_image is not None: controlnet_image = self.controlnet.process_image(controlnet_image).to(device=self.device, dtype=self.torch_dtype) # Denoise for progress_id, timestep in enumerate(progress_bar_cmd(self.scheduler.timesteps)): timestep = torch.IntTensor((timestep,))[0].to(self.device) # ControlNet if controlnet_image is not None: additional_res_stack_posi = self.controlnet(latents, timestep, prompt_emb_posi, controlnet_image) additional_res_stack_nega = self.controlnet(latents, timestep, prompt_emb_nega, controlnet_image) else: additional_res_stack_posi = None additional_res_stack_nega = None # Classifier-free guidance noise_pred_posi = self.unet( latents, timestep, prompt_emb_posi, additional_res_stack=additional_res_stack_posi, tiled=tiled, tile_size=tile_size, tile_stride=tile_stride ) noise_pred_nega = self.unet( latents, timestep, prompt_emb_nega, additional_res_stack=additional_res_stack_nega, tiled=tiled, tile_size=tile_size, tile_stride=tile_stride ) noise_pred = noise_pred_nega + cfg_scale * (noise_pred_posi - noise_pred_nega) # DDIM latents = self.scheduler.step(noise_pred, timestep, latents) # UI if progress_bar_st is not None: progress_bar_st.progress(progress_id / len(self.scheduler.timesteps)) # Decode image image = self.vae_decoder(latents, tiled=tiled, tile_size=tile_size, tile_stride=tile_stride)[0] image = image.cpu().permute(1, 2, 0).numpy() image = Image.fromarray(((image / 2 + 0.5).clip(0, 1) * 255).astype("uint8")) return image