update preference models

examples/QualityMetric/README.md

@@ -1,49 +0,0 @@
-# Image Quality Metric
-
-The image quality assessment functionality has now been integrated into Diffsynth.
-
-## Usage
-
-### Step 1: Download pretrained reward models 
-
-```
-modelscope download --model 'DiffSynth-Studio/QualityMetric_reward_pretrained' 
-```
-
-The file directory is shown below.
-
-```
-DiffSynth-Studio/
-└── models/
-    └── QualityMetric/
-        ├── HPS_v2/
-        │   ├── HPS_v2_compressed.safetensors
-        │   ├── HPS_v2.1_compressed.safetensors
-        └── ...
-```
-
-### Step 2: Test image quality metric
-
-Prompt: "a painting of an ocean with clouds and birds, day time, low depth field effect"
-
-|1.webp|2.webp|3.webp|4.webp|
-|-|-|-|-|
-|![0](images/1.webp)|![1](images/2.webp)|![2](images/3.webp)|![3](images/4.webp)|
-
-
-
-```
-CUDA_VISIBLE_DEVICES=0 python testreward.py
-```
-
-### Output:
-
-```
-ImageReward: [0.5811904668807983, 0.2745198607444763, -1.4158903360366821, -2.032487154006958]
-Aesthetic [5.900862693786621, 5.776571273803711, 5.799864292144775, 5.05204963684082]
-PickScore: [0.20737126469612122, 0.20443597435951233, 0.20660750567913055, 0.19426065683364868]
-CLIPScore: [0.3894640803337097, 0.3544551134109497, 0.33861416578292847, 0.32878392934799194]
-HPScorev2: [0.2672519087791443, 0.25495243072509766, 0.24888549745082855, 0.24302822351455688]
-HPScorev21: [0.2321144938468933, 0.20233657956123352, 0.1978294551372528, 0.19230154156684875]
-MPS_score: [10.921875, 10.71875, 10.578125, 9.25]
-```

examples/QualityMetric/testreward.py

@@ -1,80 +0,0 @@
-import os
-import torch
-from PIL import Image
-from diffsynth.extensions.QualityMetric.imagereward import ImageRewardScore
-from diffsynth.extensions.QualityMetric.pickscore import PickScore
-from diffsynth.extensions.QualityMetric.aesthetic import AestheticScore
-from diffsynth.extensions.QualityMetric.clip import CLIPScore
-from diffsynth.extensions.QualityMetric.hps import HPScore_v2
-from diffsynth.extensions.QualityMetric.mps import MPScore
-
-# download model from modelscope
-from modelscope.hub.snapshot_download import snapshot_download
-
-current_dir = os.path.dirname(os.path.abspath(__file__))
-project_root = os.path.abspath(os.path.join(current_dir, '../../'))
-model_folder = os.path.join(project_root, 'models', 'QualityMetric')
-
-# download HPS_v2 to your folder
-# model_id = "DiffSynth-Studio/QualityMetric_reward_pretrained"
-# downloaded_path = snapshot_download(
-#     model_id,
-#     cache_dir=os.path.join(model_folder, 'HPS_v2'), 
-#     allow_patterns=["HPS_v2/*"], 
-# )
-
-device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-
-def get_model_path(model_folder, model_name):
-    return os.path.join(model_folder, model_name)
-
-# your model path
-model_path = {
-    "aesthetic_predictor": get_model_path(model_folder, "aesthetic-predictor/sac+logos+ava1-l14-linearMSE.safetensors"),
-    "open_clip": get_model_path(model_folder, "CLIP-ViT-H-14-laion2B-s32B-b79K/open_clip_pytorch_model.bin"),
-    "hpsv2": get_model_path(model_folder, "HPS_v2/HPS_v2_compressed.safetensors"),
-    "hpsv2.1": get_model_path(model_folder, "HPS_v2/HPS_v2.1_compressed.safetensors"),
-    "imagereward": get_model_path(model_folder, "ImageReward/ImageReward.safetensors"),
-    "med_config": get_model_path(model_folder, "ImageReward/med_config.json"),
-    "clip": get_model_path(model_folder, "CLIP-ViT-H-14-laion2B-s32B-b79K"),
-    "clip-large": get_model_path(model_folder, "clip-vit-large-patch14"),
-    "mps": get_model_path(model_folder, "MPS_overall_checkpoint/MPS_overall_checkpoint_diffsynth.safetensors"),
-    "pickscore": get_model_path(model_folder, "PickScore_v1")
-}
-
-# load reward models
-mps_score = MPScore(device,path = model_path)
-image_reward = ImageRewardScore(device, path = model_path)
-aesthetic_score = AestheticScore(device, path = model_path)
-pick_score = PickScore(device, path = model_path)
-clip_score = CLIPScore(device, path = model_path)
-hps_score = HPScore_v2(device, path = model_path, model_version = 'v2')
-hps2_score = HPScore_v2(device, path = model_path, model_version = 'v21')
-
-prompt = "a painting of an ocean with clouds and birds, day time, low depth field effect"
-img_prefix = "images"
-generations = [f"{pic_id}.webp" for pic_id in range(1, 5)]
-
-img_list = [Image.open(os.path.join(img_prefix, img)) for img in generations]
-#img_list = [os.path.join(img_prefix, img) for img in generations]
-
-imre_scores = image_reward.score(img_list, prompt)
-print("ImageReward:", imre_scores)
-
-aes_scores = aesthetic_score.score(img_list)
-print("Aesthetic", aes_scores)
-
-p_scores = pick_score.score(img_list, prompt)
-print("PickScore:", p_scores)
-
-c_scores = clip_score.score(img_list, prompt)
-print("CLIPScore:", c_scores)
-
-h_scores = hps_score.score(img_list,prompt)
-print("HPScorev2:", h_scores)
-
-h2_scores = hps2_score.score(img_list,prompt)
-print("HPScorev21:", h2_scores)
-
-m_scores = mps_score.score(img_list, prompt)
-print("MPS_score:", m_scores)

examples/image_quality_metric/README.md

@@ -0,0 +1,15 @@
+# Image Quality Metric
+
+The image quality assessment functionality has been integrated into Diffsynth. We support the following models:
+
+* [ImageReward](https://github.com/THUDM/ImageReward)
+* [Aesthetic](https://github.com/christophschuhmann/improved-aesthetic-predictor)
+* [PickScore](https://github.com/yuvalkirstain/pickscore)
+* [CLIP](https://github.com/openai/CLIP)
+* [HPSv2](https://github.com/tgxs002/HPSv2)
+* [HPSv2.1](https://github.com/tgxs002/HPSv2)
+* [MPS](https://github.com/Kwai-Kolors/MPS)
+
+## Usage
+
+See [`./image_quality_evaluation.py`](./image_quality_evaluation.py) for more details.

examples/image_quality_metric/image_quality_evaluation.py

@@ -0,0 +1,23 @@
+from diffsynth.extensions.ImageQualityMetric import download_preference_model, load_preference_model
+from modelscope import dataset_snapshot_download
+from PIL import Image
+
+
+# Download example image
+dataset_snapshot_download(
+    dataset_id="DiffSynth-Studio/examples_in_diffsynth",
+    allow_file_pattern="data/examples/ImageQualityMetric/image.jpg",
+    local_dir="./"
+)
+
+# Parameters
+prompt = "an orange cat"
+image = Image.open("data\examples\ImageQualityMetric\image.jpg")
+device = "cuda"
+cache_dir = "./models"
+
+# Run preference models
+for model_name in ["ImageReward", "Aesthetic", "PickScore", "CLIP", "HPSv2", "HPSv2.1", "MPS"]:
+    path = download_preference_model(model_name, cache_dir=cache_dir)
+    preference_model = load_preference_model(model_name, device=device, path=path)
+    print(model_name, preference_model.score(image, prompt))