mirror of
https://github.com/modelscope/DiffSynth-Studio.git
synced 2026-03-20 07:18:14 +00:00
Delete docs directory
This commit is contained in:
yrk111222
GitHub
@@ -1,133 +0,0 @@
|
||||
# ControlNet、LoRA、IP-Adapter——精准控制技术
|
||||
|
||||
在文生图模型的基础上，还可以使用各种 Adapter 架构的模型对生成过程进行控制。
|
||||
|
||||
接下来的例子会用到很多模型，我们先把它们下载好。
|
||||
|
||||
* 一个广受好评的 Stable Diffusion XL 架构动漫风格模型
|
||||
* 一个支持多种控制模式的 ControlNet 模型
|
||||
* 一个 Stable Diffusion XL 模型的 LoRA 模型
|
||||
* 一个 IP-Adapter 模型及其对应的图像编码器
|
||||
|
||||
```python
|
||||
from diffsynth import download_models
|
||||
|
||||
download_models([
|
||||
"BluePencilXL_v200",
|
||||
"ControlNet_union_sdxl_promax",
|
||||
"SDXL_lora_zyd232_ChineseInkStyle_SDXL_v1_0",
|
||||
"IP-Adapter-SDXL"
|
||||
])
|
||||
```
|
||||
|
||||
用基础文生图功能生成一张图
|
||||
|
||||
```python
|
||||
from diffsynth import ModelManager, SDXLImagePipeline
|
||||
import torch
|
||||
|
||||
model_manager = ModelManager(torch_dtype=torch.float16, device="cuda")
|
||||
model_manager.load_models(["models/stable_diffusion_xl/bluePencilXL_v200.safetensors"])
|
||||
pipe = SDXLImagePipeline.from_model_manager(model_manager)
|
||||
torch.manual_seed(1)
|
||||
image = pipe(
|
||||
prompt="masterpiece, best quality, solo, long hair, wavy hair, silver hair, blue eyes, blue dress, medium breasts, dress, underwater, air bubble, floating hair, refraction, portrait,",
|
||||
negative_prompt="worst quality, low quality, monochrome, zombie, interlocked fingers, Aissist, cleavage, nsfw,",
|
||||
cfg_scale=6, num_inference_steps=60,
|
||||
)
|
||||
image.save("image.jpg")
|
||||
```
|
||||
|
||||
|
||||
|
||||
接下来，我们让这位水下翩翩起舞的少女变成火系魔法师！启用 ControlNet 保持画面结构的同时，修改提示词。
|
||||
|
||||
```python
|
||||
from diffsynth import ModelManager, SDXLImagePipeline, ControlNetConfigUnit
|
||||
import torch
|
||||
from PIL import Image
|
||||
|
||||
model_manager = ModelManager(torch_dtype=torch.float16, device="cuda")
|
||||
model_manager.load_models([
|
||||
"models/stable_diffusion_xl/bluePencilXL_v200.safetensors",
|
||||
"models/ControlNet/controlnet_union/diffusion_pytorch_model_promax.safetensors"
|
||||
])
|
||||
pipe = SDXLImagePipeline.from_model_manager(model_manager, controlnet_config_units=[
|
||||
ControlNetConfigUnit("depth", "models/ControlNet/controlnet_union/diffusion_pytorch_model_promax.safetensors", scale=1)
|
||||
])
|
||||
torch.manual_seed(2)
|
||||
image = pipe(
|
||||
prompt="masterpiece, best quality, solo, long hair, wavy hair, pink hair, red eyes, red dress, medium breasts, dress, fire ball, fire background, floating hair, refraction, portrait,",
|
||||
negative_prompt="worst quality, low quality, monochrome, zombie, interlocked fingers, Aissist, cleavage, nsfw, white background",
|
||||
cfg_scale=6, num_inference_steps=60,
|
||||
controlnet_image=Image.open("image.jpg")
|
||||
)
|
||||
image.save("image_controlnet.jpg")
|
||||
```
|
||||
|
||||
|
||||
|
||||
很酷对不对？还有更酷的，加个 LoRA，让画面更贴近手绘漫画的扁平风格。这个 LoRA 需要一定的触发词才能生效，这在原作者的模型页面有提到，记得在提示词的开头加上触发词哦。
|
||||
|
||||
```python
|
||||
from diffsynth import ModelManager, SDXLImagePipeline, ControlNetConfigUnit
|
||||
import torch
|
||||
from PIL import Image
|
||||
|
||||
model_manager = ModelManager(torch_dtype=torch.float16, device="cuda")
|
||||
model_manager.load_models([
|
||||
"models/stable_diffusion_xl/bluePencilXL_v200.safetensors",
|
||||
"models/ControlNet/controlnet_union/diffusion_pytorch_model_promax.safetensors"
|
||||
])
|
||||
model_manager.load_lora("models/lora/zyd232_ChineseInkStyle_SDXL_v1_0.safetensors", lora_alpha=1.0)
|
||||
pipe = SDXLImagePipeline.from_model_manager(model_manager, controlnet_config_units=[
|
||||
ControlNetConfigUnit("depth", "models/ControlNet/controlnet_union/diffusion_pytorch_model_promax.safetensors", scale=1.0)
|
||||
])
|
||||
torch.manual_seed(3)
|
||||
image = pipe(
|
||||
prompt="zydink, ink sketch, flat anime, masterpiece, best quality, solo, long hair, wavy hair, pink hair, red eyes, red dress, medium breasts, dress, fire ball, fire background, floating hair, refraction, portrait,",
|
||||
negative_prompt="worst quality, low quality, monochrome, zombie, interlocked fingers, Aissist, cleavage, nsfw, white background",
|
||||
cfg_scale=6, num_inference_steps=60,
|
||||
controlnet_image=Image.open("image.jpg")
|
||||
)
|
||||
image.save("image_lora.jpg")
|
||||
```
|
||||
|
||||
|
||||
|
||||
还没结束呢！找一张水墨风的中国画作为风格引导，启动 IP-Adapter，让古典艺术和现代美学碰撞！
|
||||
|
||||
|就用这张图作为风格引导吧||
|
||||
|-|-|
|
||||
|
||||
```python
|
||||
from diffsynth import ModelManager, SDXLImagePipeline, ControlNetConfigUnit
|
||||
import torch
|
||||
from PIL import Image
|
||||
|
||||
model_manager = ModelManager(torch_dtype=torch.float16, device="cuda")
|
||||
model_manager.load_models([
|
||||
"models/stable_diffusion_xl/bluePencilXL_v200.safetensors",
|
||||
"models/ControlNet/controlnet_union/diffusion_pytorch_model_promax.safetensors",
|
||||
"models/IpAdapter/stable_diffusion_xl/ip-adapter_sdxl.bin",
|
||||
"models/IpAdapter/stable_diffusion_xl/image_encoder/model.safetensors",
|
||||
])
|
||||
model_manager.load_lora("models/lora/zyd232_ChineseInkStyle_SDXL_v1_0.safetensors", lora_alpha=1.0)
|
||||
pipe = SDXLImagePipeline.from_model_manager(model_manager, controlnet_config_units=[
|
||||
ControlNetConfigUnit("depth", "models/ControlNet/controlnet_union/diffusion_pytorch_model_promax.safetensors", scale=1.0)
|
||||
])
|
||||
torch.manual_seed(2)
|
||||
image = pipe(
|
||||
prompt="zydink, ink sketch, flat anime, masterpiece, best quality, solo, long hair, wavy hair, pink hair, red eyes, red dress, medium breasts, dress, fire ball, fire background, floating hair, refraction, portrait,",
|
||||
negative_prompt="worst quality, low quality, monochrome, zombie, interlocked fingers, Aissist, cleavage, nsfw, white background",
|
||||
cfg_scale=6, num_inference_steps=60,
|
||||
controlnet_image=Image.open("image.jpg"),
|
||||
ipadapter_images=[Image.open("ink_style.jpg")],
|
||||
ipadapter_use_instant_style=True, ipadapter_scale=0.5
|
||||
)
|
||||
image.save("image_ipadapter.jpg")
|
||||
```
|
||||
|
||||
|
||||
|
||||
用 Diffusion 生成图像的乐趣在于，各种生态模型的组合，可以实现各种奇思妙想。
|
||||
@@ -1,65 +0,0 @@
|
||||
# 文生图、图生图、高分辨率修复——初识绚丽的 Diffusion
|
||||
|
||||
加载文生图模型，这里我们使用一个 Civiai 上一个动漫风格的模型作为例子。
|
||||
|
||||
```python
|
||||
import torch
|
||||
from diffsynth import ModelManager, SDImagePipeline, download_models
|
||||
|
||||
download_models(["AingDiffusion_v12"])
|
||||
model_manager = ModelManager(torch_dtype=torch.float16, device="cuda")
|
||||
model_manager.load_models(["models/stable_diffusion/aingdiffusion_v12.safetensors"])
|
||||
pipe = SDImagePipeline.from_model_manager(model_manager)
|
||||
```
|
||||
|
||||
生成一张图小试身手。
|
||||
|
||||
```python
|
||||
torch.manual_seed(0)
|
||||
image = pipe(
|
||||
prompt="masterpiece, best quality, a girl with long silver hair",
|
||||
negative_prompt="worst quality, low quality, monochrome, zombie, interlocked fingers, Aissist, cleavage, nsfw,",
|
||||
height=512, width=512, num_inference_steps=80,
|
||||
)
|
||||
image.save("image.jpg")
|
||||
```
|
||||
|
||||
嗯，一个可爱的小姐姐。
|
||||
|
||||
|
||||
|
||||
用图生图功能把她的头发变成红色，只需要添加 `input_image` 和 `denoising_strength` 两个参数。其中 `denoising_strength` 用于控制加噪声的强度，为 0 时生成的图与输入的图完全一致，为 1 时完全随机生成图。
|
||||
|
||||
```python
|
||||
torch.manual_seed(1)
|
||||
image_edited = pipe(
|
||||
prompt="masterpiece, best quality, a girl with long red hair",
|
||||
negative_prompt="worst quality, low quality, monochrome, zombie, interlocked fingers, Aissist, cleavage, nsfw,",
|
||||
height=512, width=512, num_inference_steps=80,
|
||||
input_image=image, denoising_strength=0.6,
|
||||
)
|
||||
image_edited.save("image_edited.jpg")
|
||||
```
|
||||
|
||||
嗯，一个红色头发的可爱小姐姐。
|
||||
|
||||
|
||||
|
||||
由于模型本身是在 512*512 分辨率下训练的，所以图片看起来有点模糊，不过我们可以利用模型自身的能力润色这张图，为其填充细节。具体来说，就是提高分辨率后进行图生图。
|
||||
|
||||
```python
|
||||
torch.manual_seed(2)
|
||||
image_highres = pipe(
|
||||
prompt="masterpiece, best quality, a girl with long red hair",
|
||||
negative_prompt="worst quality, low quality, monochrome, zombie, interlocked fingers, Aissist, cleavage, nsfw,",
|
||||
height=1024, width=1024, num_inference_steps=80,
|
||||
input_image=image_edited.resize((1024, 1024)), denoising_strength=0.6,
|
||||
)
|
||||
image_highres.save("image_highres.jpg")
|
||||
```
|
||||
|
||||
嗯，一个清晰的红色头发可爱小姐姐。
|
||||
|
||||
|
||||
|
||||
值得注意的是，图生图和高分辨率修复功能是全局支持的，目前我们所有的图像生成流水线都可以这样使用。
|
||||
@@ -1,78 +0,0 @@
|
||||
# 翻译、润色——提示词的魔法
|
||||
|
||||
在生成图像时，我们需要编写提示词，用来描述图像的内容。提示词会直接影响生成的效果，但提示词的编写也是一门学问，好的提示词可以生成具有高度美感的图像，我们提供了一系列模型来帮助用户处理提示词。
|
||||
|
||||
## 翻译
|
||||
|
||||
目前大多数文生图模型都是只支持英文提示词的，对于非英文母语的用户，使用起来有些困难，我们可以使用开源的翻译模型把提示词翻译成英文。在下面这个例子中，我们以“一个女孩”为提示词，使用模型 opus-mt-zh-en（可在 [HuggingFace](https://huggingface.co/Helsinki-NLP/opus-mt-zh-en) 或 [ModelScope](https://modelscope.cn/models/moxying/opus-mt-zh-en) 下载）进行翻译。
|
||||
|
||||
```python
|
||||
from diffsynth import ModelManager, SDXLImagePipeline, Translator
|
||||
import torch
|
||||
|
||||
model_manager = ModelManager(
|
||||
torch_dtype=torch.float16, device="cuda",
|
||||
model_id_list=["BluePencilXL_v200", "opus-mt-zh-en"]
|
||||
)
|
||||
pipe = SDXLImagePipeline.from_model_manager(model_manager, prompt_refiner_classes=[Translator])
|
||||
|
||||
torch.manual_seed(0)
|
||||
prompt = "一个女孩"
|
||||
image = pipe(
|
||||
prompt=prompt, negative_prompt="",
|
||||
height=1024, width=1024, num_inference_steps=30
|
||||
)
|
||||
image.save("image_1.jpg")
|
||||
```
|
||||
|
||||
|
||||
|
||||
## 润色
|
||||
|
||||
详细的提示词可以生成细节更丰富的图像，我们可以使用提示词润色模型 BeautifulPrompt（可在 [HuggingFace](https://huggingface.co/alibaba-pai/pai-bloom-1b1-text2prompt-sd) 或 [ModelScope](https://modelscope.cn/models/AI-ModelScope/pai-bloom-1b1-text2prompt-sd) 下载）对简单的提示词进行润色，这个模型能够让整体画面风格更加华丽。
|
||||
|
||||
这个模块可以和翻译模块同时启用，但请注意顺序，先翻译，后润色。
|
||||
|
||||
```python
|
||||
from diffsynth import ModelManager, SDXLImagePipeline, Translator, BeautifulPrompt
|
||||
import torch
|
||||
|
||||
model_manager = ModelManager(
|
||||
torch_dtype=torch.float16, device="cuda",
|
||||
model_id_list=["BluePencilXL_v200", "opus-mt-zh-en", "BeautifulPrompt"]
|
||||
)
|
||||
pipe = SDXLImagePipeline.from_model_manager(model_manager, prompt_refiner_classes=[Translator, BeautifulPrompt])
|
||||
|
||||
torch.manual_seed(0)
|
||||
prompt = "一个女孩"
|
||||
image = pipe(
|
||||
prompt=prompt, negative_prompt="",
|
||||
height=1024, width=1024, num_inference_steps=30
|
||||
)
|
||||
image.save("image_2.jpg")
|
||||
```
|
||||
|
||||
|
||||
|
||||
我们还内置了一个通义千问模型，这个模型可以一步到位地完成提示词的翻译和润色工作。
|
||||
|
||||
```python
|
||||
from diffsynth import ModelManager, SDXLImagePipeline, QwenPrompt
|
||||
import torch
|
||||
|
||||
model_manager = ModelManager(
|
||||
torch_dtype=torch.float16, device="cuda",
|
||||
model_id_list=["BluePencilXL_v200", "QwenPrompt"]
|
||||
)
|
||||
pipe = SDXLImagePipeline.from_model_manager(model_manager, prompt_refiner_classes=[QwenPrompt])
|
||||
|
||||
torch.manual_seed(0)
|
||||
prompt = "一个女孩"
|
||||
image = pipe(
|
||||
prompt=prompt, negative_prompt="",
|
||||
height=1024, width=1024, num_inference_steps=30
|
||||
)
|
||||
image.save("image_3.jpg")
|
||||
```
|
||||
|
||||
|
||||
@@ -1,95 +0,0 @@
|
||||
# 当图像模型遇见 AnimateDiff——模型组合技术
|
||||
|
||||
我们已经领略到了 Stable Diffusion 模型及其生态模型的强大图像生成能力，现在我们引入一个新的模块：AnimateDiff，这样一来就可以把图像模型的能力迁移到视频中。在本篇文章中，我们为您展示基于 DiffSynth-Studio 搭建的动漫风格视频渲染方案：Diffutoon。
|
||||
|
||||
## 下载模型
|
||||
|
||||
接下来的例子会用到很多模型，我们先把它们下载好。
|
||||
|
||||
* 一个动漫风格的 Stable Diffusion 架构模型
|
||||
* 两个 ControlNet 模型
|
||||
* 一个 Textual Inversion 模型
|
||||
* 一个 AnimateDiff 模型
|
||||
|
||||
```python
|
||||
from diffsynth import download_models
|
||||
|
||||
download_models([
|
||||
"AingDiffusion_v12",
|
||||
"AnimateDiff_v2",
|
||||
"ControlNet_v11p_sd15_lineart",
|
||||
"ControlNet_v11f1e_sd15_tile",
|
||||
"TextualInversion_VeryBadImageNegative_v1.3"
|
||||
])
|
||||
```
|
||||
|
||||
## 下载视频
|
||||
|
||||
你可以随意选择任何你喜欢的视频，我们使用[这个视频](https://www.bilibili.com/video/BV1iG411a7sQ)作为演示，你可以通过以下命令下载这个视频文件，但请注意，在没有获得视频原作者的商用版权时，请不要将其用作商业用途。
|
||||
|
||||
```
|
||||
modelscope download --dataset Artiprocher/examples_in_diffsynth data/examples/diffutoon/input_video.mp4 --local_dir ./
|
||||
```
|
||||
|
||||
## 生成动漫
|
||||
|
||||
```python
|
||||
from diffsynth import ModelManager, SDVideoPipeline, ControlNetConfigUnit, VideoData, save_video
|
||||
import torch
|
||||
|
||||
# Load models
|
||||
model_manager = ModelManager(torch_dtype=torch.float16, device="cuda")
|
||||
model_manager.load_models([
|
||||
"models/stable_diffusion/aingdiffusion_v12.safetensors",
|
||||
"models/AnimateDiff/mm_sd_v15_v2.ckpt",
|
||||
"models/ControlNet/control_v11p_sd15_lineart.pth",
|
||||
"models/ControlNet/control_v11f1e_sd15_tile.pth",
|
||||
])
|
||||
|
||||
# Build pipeline
|
||||
pipe = SDVideoPipeline.from_model_manager(
|
||||
model_manager,
|
||||
[
|
||||
ControlNetConfigUnit(
|
||||
processor_id="tile",
|
||||
model_path="models/ControlNet/control_v11f1e_sd15_tile.pth",
|
||||
scale=0.5
|
||||
),
|
||||
ControlNetConfigUnit(
|
||||
processor_id="lineart",
|
||||
model_path="models/ControlNet/control_v11p_sd15_lineart.pth",
|
||||
scale=0.5
|
||||
)
|
||||
]
|
||||
)
|
||||
pipe.prompter.load_textual_inversions(["models/textual_inversion/verybadimagenegative_v1.3.pt"])
|
||||
|
||||
# Load video
|
||||
video = VideoData(
|
||||
video_file="data/examples/diffutoon/input_video.mp4",
|
||||
height=1536, width=1536
|
||||
)
|
||||
input_video = [video[i] for i in range(30)]
|
||||
|
||||
# Generate
|
||||
torch.manual_seed(0)
|
||||
output_video = pipe(
|
||||
prompt="best quality, perfect anime illustration, light, a girl is dancing, smile, solo",
|
||||
negative_prompt="verybadimagenegative_v1.3",
|
||||
cfg_scale=7, clip_skip=2,
|
||||
input_frames=input_video, denoising_strength=1.0,
|
||||
controlnet_frames=input_video, num_frames=len(input_video),
|
||||
num_inference_steps=10, height=1536, width=1536,
|
||||
animatediff_batch_size=16, animatediff_stride=8,
|
||||
)
|
||||
|
||||
# Save video
|
||||
save_video(output_video, "output_video.mp4", fps=30)
|
||||
```
|
||||
|
||||
## 效果展示
|
||||
|
||||
<video width="512" height="256" controls>
|
||||
<source src="https://github.com/Artiprocher/DiffSynth-Studio/assets/35051019/b54c05c5-d747-4709-be5e-b39af82404dd" type="video/mp4">
|
||||
您的浏览器不支持Video标签。
|
||||
</video>
|
||||
Reference in New Issue
Block a user