update UI

diffsynth/pipelines/base.py

@@ -31,4 +31,23 @@ class BasePipeline(torch.nn.Module):
         video = vae_output.cpu().permute(1, 2, 0).numpy()
         video = [Image.fromarray(((image / 2 + 0.5).clip(0, 1) * 255).astype("uint8")) for image in video]
         return video
+
+    
+    def merge_latents(self, value, latents, masks, scales):
+        height, width = value.shape[-2:]
+        weight = torch.ones_like(value)
+        for latent, mask, scale in zip(latents, masks, scales):
+            mask = self.preprocess_image(mask.resize((height, width))).mean(dim=1, keepdim=True) > 0
+            mask = mask.repeat(1, latent.shape[1], 1, 1)
+            value[mask] += latent[mask] * scale
+            weight[mask] += scale
+        value /= weight
+        return value
+
+
+    def control_noise_via_local_prompts(self, prompt_emb_global, prompt_emb_locals, masks, mask_scales, inference_callback):
+        noise_pred_global = inference_callback(prompt_emb_global)
+        noise_pred_locals = [inference_callback(prompt_emb_local) for prompt_emb_local in prompt_emb_locals]
+        noise_pred = self.merge_latents(noise_pred_global, noise_pred_locals, masks, mask_scales)
+        return noise_pred