From d242af8e22460a639d2db9ff7482f3fb44d0f5a8 Mon Sep 17 00:00:00 2001 From: Artiprocher Date: Tue, 10 Sep 2024 17:36:35 +0800 Subject: [PATCH] update examples --- examples/image_synthesis/README.md | 16 --------- .../image_synthesis/qwen_prompt_refining.py | 25 -------------- examples/prompt_magic/README.md | 34 +++++++++++++++++++ .../bf_prompt_refining.py} | 0 .../omost_flux_text_to_image.py | 0 examples/prompt_magic/qwen_prompt_refining.py | 27 +++++++++++++++ 6 files changed, 61 insertions(+), 41 deletions(-) delete mode 100644 examples/image_synthesis/qwen_prompt_refining.py create mode 100644 examples/prompt_magic/README.md rename examples/{image_synthesis/sd_prompt_refining.py => prompt_magic/bf_prompt_refining.py} (100%) rename examples/{image_synthesis => prompt_magic}/omost_flux_text_to_image.py (100%) create mode 100644 examples/prompt_magic/qwen_prompt_refining.py diff --git a/examples/image_synthesis/README.md b/examples/image_synthesis/README.md index 04ec0e7..10f6efb 100644 --- a/examples/image_synthesis/README.md +++ b/examples/image_synthesis/README.md @@ -85,19 +85,3 @@ We highly recommend you to use this model in the WebUI. |"black car"|"red car"| |-|-| |![black_car](https://github.com/Artiprocher/DiffSynth-Studio/assets/35051019/7fbfd803-68d4-44f3-8713-8c925fec47d0)|![black_car_to_red_car](https://github.com/Artiprocher/DiffSynth-Studio/assets/35051019/aaf886e4-c33c-4fd8-98e2-29eef117ba00)| - -### Example: Prompt Processing - -If you are not native English user, we provide translation service for you. Our prompter can translate other language to English and refine it using "BeautifulPrompt" models. Please see [`sd_prompt_refining.py`](./sd_prompt_refining.py) for more details. - -Prompt: "一个漂亮的女孩". The [translation model](https://huggingface.co/Helsinki-NLP/opus-mt-en-zh) will translate it to English. - -|seed=0|seed=1|seed=2|seed=3| -|-|-|-|-| -|![0_](https://github.com/Artiprocher/DiffSynth-Studio/assets/35051019/ebb25ca8-7ce1-4d9e-8081-59a867c70c4d)|![1_](https://github.com/Artiprocher/DiffSynth-Studio/assets/35051019/a7e79853-3c1a-471a-9c58-c209ec4b76dd)|![2_](https://github.com/Artiprocher/DiffSynth-Studio/assets/35051019/a292b959-a121-481f-b79c-61cc3346f810)|![3_](https://github.com/Artiprocher/DiffSynth-Studio/assets/35051019/1c19b54e-5a6f-4d48-960b-a7b2b149bb4c)| - -Prompt: "一个漂亮的女孩". The [translation model](https://huggingface.co/Helsinki-NLP/opus-mt-en-zh) will translate it to English. Then the [refining model](https://huggingface.co/alibaba-pai/pai-bloom-1b1-text2prompt-sd) will refine the translated prompt for better visual quality. - -|seed=0|seed=1|seed=2|seed=3| -|-|-|-|-| -|![0](https://github.com/Artiprocher/DiffSynth-Studio/assets/35051019/778b1bd9-44e0-46ac-a99c-712b3fc9aaa4)|![1](https://github.com/Artiprocher/DiffSynth-Studio/assets/35051019/c03479b8-2082-4c6e-8e1c-3582b98686f6)|![2](https://github.com/Artiprocher/DiffSynth-Studio/assets/35051019/edb33d21-3288-4a55-96ca-a4bfe1b50b00)|![3](https://github.com/Artiprocher/DiffSynth-Studio/assets/35051019/7848cfc1-cad5-4848-8373-41d24e98e584)| diff --git a/examples/image_synthesis/qwen_prompt_refining.py b/examples/image_synthesis/qwen_prompt_refining.py deleted file mode 100644 index 1cf1239..0000000 --- a/examples/image_synthesis/qwen_prompt_refining.py +++ /dev/null @@ -1,25 +0,0 @@ -from diffsynth import ModelManager, SDXLImagePipeline, download_models, QwenPrompt -import torch - - -download_models(["StableDiffusionXL_v1", "QwenPrompt"]) - -# Load models -model_manager = ModelManager(torch_dtype=torch.float16, device="cuda") -model_manager.load_models([ - "models/stable_diffusion_xl/sd_xl_base_1.0.safetensors", - "models/QwenPrompt/qwen2-1.5b-instruct", -]) -pipe = SDXLImagePipeline.from_model_manager(model_manager, prompt_refiner_classes=[QwenPrompt]) - -prompt = "一个漂亮的女孩" -negative_prompt = "" - -for seed in range(4): - torch.manual_seed(seed) - image = pipe( - prompt=prompt, negative_prompt=negative_prompt, - height=1024, width=1024, - num_inference_steps=30 - ) - image.save(f"{seed}.jpg") diff --git a/examples/prompt_magic/README.md b/examples/prompt_magic/README.md new file mode 100644 index 0000000..cbb920e --- /dev/null +++ b/examples/prompt_magic/README.md @@ -0,0 +1,34 @@ +# Prompt Refining + +Prompt refining is supported in DiffSynth-Studio. + +### Example: Qwen + +If you are not native English user, we provide LLM-based translation for you. Qwen is a typical example. See [`qwen_prompt_refining.py`](./qwen_prompt_refining.py). + +Prompt: "鹰". This prompt will be refined: + +* A majestic eagle soaring high above a vast expanse of open sky, its wings spread wide and its eyes fixed on the horizon. +* A majestic eagle soaring high above the horizon, its wingspan stretching out towards the endless sky. Its sharp beak and powerful talons stand out against the azure sky, highlighting its strength and freedom. The eagle's silhouette is silhouetted against the fading sun, casting long shadows behind it. +* A majestic eagle soaring high above a vast, open landscape, its wings spread wide and its beak pointed towards the sky. The sun casts long shadows across the ground, adding depth and texture to the image. The bird's feathers shimmer in the light, creating a sense of movement and power. +* A majestic eagle soaring high above a vast, open landscape, its sharp talons gripping a fish effortlessly in its beak. The sun casts a warm golden glow behind it, casting long shadows across the barren earth below. The eagle's wingspan stretches out towards infinity, its feathers glistening in the light. Its eyes fixate on the distant horizon, as if sensing something important about to unfold. + +|seed=0|seed=1|seed=2|seed=3| +|-|-|-|-| +|![0](https://github.com/user-attachments/assets/6f509b0d-204c-4ca9-b3f6-04925fb4b40d)|![1](https://github.com/user-attachments/assets/824f2713-134a-4cae-a155-25224d4afb9a)|![2](https://github.com/user-attachments/assets/747be52a-0b46-45fc-b0e8-a0c83f7e716b)|![3](https://github.com/user-attachments/assets/490564e6-d247-40c9-8361-5db72058c337)| + +### Example: OPUS + BeautifulPrompt + +Our prompter can translate other language to English and refine it using "BeautifulPrompt" models. Please see [`bf_prompt_refining.py`](./bf_prompt_refining.py) for more details. + +Prompt: "一个漂亮的女孩". The [translation model](https://huggingface.co/Helsinki-NLP/opus-mt-en-zh) will translate it to English. + +|seed=0|seed=1|seed=2|seed=3| +|-|-|-|-| +|![0_](https://github.com/Artiprocher/DiffSynth-Studio/assets/35051019/ebb25ca8-7ce1-4d9e-8081-59a867c70c4d)|![1_](https://github.com/Artiprocher/DiffSynth-Studio/assets/35051019/a7e79853-3c1a-471a-9c58-c209ec4b76dd)|![2_](https://github.com/Artiprocher/DiffSynth-Studio/assets/35051019/a292b959-a121-481f-b79c-61cc3346f810)|![3_](https://github.com/Artiprocher/DiffSynth-Studio/assets/35051019/1c19b54e-5a6f-4d48-960b-a7b2b149bb4c)| + +Prompt: "一个漂亮的女孩". The [translation model](https://huggingface.co/Helsinki-NLP/opus-mt-en-zh) will translate it to English. Then the [refining model](https://huggingface.co/alibaba-pai/pai-bloom-1b1-text2prompt-sd) will refine the translated prompt for better visual quality. + +|seed=0|seed=1|seed=2|seed=3| +|-|-|-|-| +|![0](https://github.com/Artiprocher/DiffSynth-Studio/assets/35051019/778b1bd9-44e0-46ac-a99c-712b3fc9aaa4)|![1](https://github.com/Artiprocher/DiffSynth-Studio/assets/35051019/c03479b8-2082-4c6e-8e1c-3582b98686f6)|![2](https://github.com/Artiprocher/DiffSynth-Studio/assets/35051019/edb33d21-3288-4a55-96ca-a4bfe1b50b00)|![3](https://github.com/Artiprocher/DiffSynth-Studio/assets/35051019/7848cfc1-cad5-4848-8373-41d24e98e584)| diff --git a/examples/image_synthesis/sd_prompt_refining.py b/examples/prompt_magic/bf_prompt_refining.py similarity index 100% rename from examples/image_synthesis/sd_prompt_refining.py rename to examples/prompt_magic/bf_prompt_refining.py diff --git a/examples/image_synthesis/omost_flux_text_to_image.py b/examples/prompt_magic/omost_flux_text_to_image.py similarity index 100% rename from examples/image_synthesis/omost_flux_text_to_image.py rename to examples/prompt_magic/omost_flux_text_to_image.py diff --git a/examples/prompt_magic/qwen_prompt_refining.py b/examples/prompt_magic/qwen_prompt_refining.py new file mode 100644 index 0000000..511d506 --- /dev/null +++ b/examples/prompt_magic/qwen_prompt_refining.py @@ -0,0 +1,27 @@ +from diffsynth import ModelManager, FluxImagePipeline, download_models, QwenPrompt +import torch + + +download_models(["FLUX.1-dev", "QwenPrompt"]) + +model_manager = ModelManager(torch_dtype=torch.bfloat16, device="cuda") +model_manager.load_models([ + "models/FLUX/FLUX.1-dev/text_encoder/model.safetensors", + "models/FLUX/FLUX.1-dev/text_encoder_2", + "models/FLUX/FLUX.1-dev/ae.safetensors", + "models/FLUX/FLUX.1-dev/flux1-dev.safetensors", + "models/QwenPrompt/qwen2-1.5b-instruct", +]) +pipe = FluxImagePipeline.from_model_manager(model_manager, prompt_refiner_classes=[QwenPrompt]) + +prompt = "鹰" +negative_prompt = "" + +for seed in range(4): + torch.manual_seed(seed) + image = pipe( + prompt=prompt, negative_prompt=negative_prompt, + height=1024, width=1024, + num_inference_steps=30 + ) + image.save(f"{seed}.jpg")