theluyuan/DiffSynth-Studio
1
0
Fork 0
mirror of https://github.com/modelscope/DiffSynth-Studio.git synced 2026-03-25 10:48:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #657 from modelscope/dev-dzj

Browse Source 
support json dataset
This commit is contained in:
Zhongjie Duan
2025-07-02 20:08:13 +08:00
committed by GitHub
parent 789dade026 1363a0559f
commit 22d28665fe
1 changed files with 13 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
diffsynth/trainers/utils.py
View File

@@ -1,4 +1,4 @@
import imageio, os, torch, warnings, torchvision, argparse import imageio, os, torch, warnings, torchvision, argparse, json
from peft import LoraConfig, inject_adapter_in_model from peft import LoraConfig, inject_adapter_in_model
from PIL import Image from PIL import Image
import pandas as pd import pandas as pd
@@ -48,6 +48,11 @@ class ImageDataset(torch.utils.data.Dataset):
print("No metadata. Trying to generate it.") print("No metadata. Trying to generate it.")
metadata = self.generate_metadata(base_path) metadata = self.generate_metadata(base_path)
print(f"{len(metadata)} lines in metadata.") print(f"{len(metadata)} lines in metadata.")
self.data = [metadata.iloc[i].to_dict() for i in range(len(metadata))]
elif metadata_path.endswith(".json"):
with open(metadata_path, "r") as f:
metadata = json.load(f)
self.data = metadata
else: else:
metadata = pd.read_csv(metadata_path) metadata = pd.read_csv(metadata_path)
self.data = [metadata.iloc[i].to_dict() for i in range(len(metadata))] self.data = [metadata.iloc[i].to_dict() for i in range(len(metadata))]
@@ -177,6 +182,11 @@ class VideoDataset(torch.utils.data.Dataset):
print("No metadata. Trying to generate it.") print("No metadata. Trying to generate it.")
metadata = self.generate_metadata(base_path) metadata = self.generate_metadata(base_path)
print(f"{len(metadata)} lines in metadata.") print(f"{len(metadata)} lines in metadata.")
self.data = [metadata.iloc[i].to_dict() for i in range(len(metadata))]
elif metadata_path.endswith(".json"):
with open(metadata_path, "r") as f:
metadata = json.load(f)
self.data = metadata
else: else:
metadata = pd.read_csv(metadata_path) metadata = pd.read_csv(metadata_path)
self.data = [metadata.iloc[i].to_dict() for i in range(len(metadata))] self.data = [metadata.iloc[i].to_dict() for i in range(len(metadata))]