diff --git a/.msc b/.msc new file mode 100644 index 0000000..eb82657 Binary files /dev/null and b/.msc differ diff --git a/.mv b/.mv new file mode 100644 index 0000000..8b25206 --- /dev/null +++ b/.mv @@ -0,0 +1 @@ +master \ No newline at end of file diff --git a/dchen/7.png b/dchen/7.png new file mode 100644 index 0000000..9a107af Binary files /dev/null and b/dchen/7.png differ diff --git a/dchen/__pycache__/camera_adapter.cpython-310.pyc b/dchen/__pycache__/camera_adapter.cpython-310.pyc new file mode 100644 index 0000000..c0af964 Binary files /dev/null and b/dchen/__pycache__/camera_adapter.cpython-310.pyc differ diff --git a/dchen/__pycache__/camera_compute.cpython-310.pyc b/dchen/__pycache__/camera_compute.cpython-310.pyc new file mode 100644 index 0000000..53d6145 Binary files /dev/null and b/dchen/__pycache__/camera_compute.cpython-310.pyc differ diff --git a/dchen/camera_adapter.py b/dchen/camera_adapter.py new file mode 100644 index 0000000..a22c1a9 --- /dev/null +++ b/dchen/camera_adapter.py @@ -0,0 +1,62 @@ +import torch +import torch.nn as nn + +class SimpleAdapter(nn.Module): + def __init__(self, in_dim, out_dim, kernel_size, stride, num_residual_blocks=1): + super(SimpleAdapter, self).__init__() + + # Pixel Unshuffle: reduce spatial dimensions by a factor of 8 + self.pixel_unshuffle = nn.PixelUnshuffle(downscale_factor=8) + + # Convolution: reduce spatial dimensions by a factor + # of 2 (without overlap) + self.conv = nn.Conv2d(in_dim * 64, out_dim, kernel_size=kernel_size, stride=stride, padding=0) + + # Residual blocks for feature extraction + self.residual_blocks = nn.Sequential( + *[ResidualBlock(out_dim) for _ in range(num_residual_blocks)] + ) + + def forward(self, x): + # Reshape to merge the frame dimension into batch + bs, c, f, h, w = x.size() + x = x.permute(0, 2, 1, 3, 4).contiguous().view(bs * f, c, h, w) + + # Pixel Unshuffle operation + x_unshuffled = self.pixel_unshuffle(x) + + # Convolution operation + x_conv = self.conv(x_unshuffled) + + # Feature extraction with residual blocks + out = self.residual_blocks(x_conv) + + # Reshape to restore original bf dimension + out = out.view(bs, f, out.size(1), out.size(2), out.size(3)) + + # Permute dimensions to reorder (if needed), e.g., swap channels and feature frames + out = out.permute(0, 2, 1, 3, 4) + + return out + +class ResidualBlock(nn.Module): + def __init__(self, dim): + super(ResidualBlock, self).__init__() + self.conv1 = nn.Conv2d(dim, dim, kernel_size=3, padding=1) + self.relu = nn.ReLU(inplace=True) + self.conv2 = nn.Conv2d(dim, dim, kernel_size=3, padding=1) + + def forward(self, x): + residual = x + out = self.relu(self.conv1(x)) + out = self.conv2(out) + out += residual + return out + +# Example usage +# in_dim = 3 +# out_dim = 64 +# adapter = SimpleAdapterWithReshape(in_dim, out_dim) +# x = torch.randn(1, in_dim, 4, 64, 64) # e.g., batch size = 1, channels = 3, frames/features = 4 +# output = adapter(x) +# print(output.shape) # Should reflect transformed dimensions diff --git a/dchen/camera_compute.py b/dchen/camera_compute.py new file mode 100644 index 0000000..cec1830 --- /dev/null +++ b/dchen/camera_compute.py @@ -0,0 +1,174 @@ +import csv +import gc +import io +import json +import math +import os +import random +from random import shuffle + +import albumentations +import cv2 +import numpy as np +import torch +import torch.nn.functional as F +import torchvision.transforms as transforms +from decord import VideoReader +from einops import rearrange +from packaging import version as pver +from PIL import Image +from torch.utils.data import BatchSampler, Sampler +from torch.utils.data.dataset import Dataset + +class Camera(object): + """Copied from https://github.com/hehao13/CameraCtrl/blob/main/inference.py + """ + def __init__(self, entry): + fx, fy, cx, cy = entry[1:5] + self.fx = fx + self.fy = fy + self.cx = cx + self.cy = cy + w2c_mat = np.array(entry[7:]).reshape(3, 4) + w2c_mat_4x4 = np.eye(4) + w2c_mat_4x4[:3, :] = w2c_mat + self.w2c_mat = w2c_mat_4x4 + self.c2w_mat = np.linalg.inv(w2c_mat_4x4) + +def get_relative_pose(cam_params): + """Copied from https://github.com/hehao13/CameraCtrl/blob/main/inference.py + """ + abs_w2cs = [cam_param.w2c_mat for cam_param in cam_params] + abs_c2ws = [cam_param.c2w_mat for cam_param in cam_params] + cam_to_origin = 0 + target_cam_c2w = np.array([ + [1, 0, 0, 0], + [0, 1, 0, -cam_to_origin], + [0, 0, 1, 0], + [0, 0, 0, 1] + ]) + abs2rel = target_cam_c2w @ abs_w2cs[0] + ret_poses = [target_cam_c2w, ] + [abs2rel @ abs_c2w for abs_c2w in abs_c2ws[1:]] + ret_poses = np.array(ret_poses, dtype=np.float32) + return ret_poses + +def ray_condition(K, c2w, H, W, device): + """Copied from https://github.com/hehao13/CameraCtrl/blob/main/inference.py + """ + # c2w: B, V, 4, 4 + # K: B, V, 4 + + B = K.shape[0] + + j, i = custom_meshgrid( + torch.linspace(0, H - 1, H, device=device, dtype=c2w.dtype), + torch.linspace(0, W - 1, W, device=device, dtype=c2w.dtype), + ) + i = i.reshape([1, 1, H * W]).expand([B, 1, H * W]) + 0.5 # [B, HxW] + j = j.reshape([1, 1, H * W]).expand([B, 1, H * W]) + 0.5 # [B, HxW] + + fx, fy, cx, cy = K.chunk(4, dim=-1) # B,V, 1 + + zs = torch.ones_like(i) # [B, HxW] + xs = (i - cx) / fx * zs + ys = (j - cy) / fy * zs + zs = zs.expand_as(ys) + + directions = torch.stack((xs, ys, zs), dim=-1) # B, V, HW, 3 + directions = directions / directions.norm(dim=-1, keepdim=True) # B, V, HW, 3 + + rays_d = directions @ c2w[..., :3, :3].transpose(-1, -2) # B, V, 3, HW + rays_o = c2w[..., :3, 3] # B, V, 3 + rays_o = rays_o[:, :, None].expand_as(rays_d) # B, V, 3, HW + # c2w @ dirctions + rays_dxo = torch.cross(rays_o, rays_d) + plucker = torch.cat([rays_dxo, rays_d], dim=-1) + plucker = plucker.reshape(B, c2w.shape[1], H, W, 6) # B, V, H, W, 6 + # plucker = plucker.permute(0, 1, 4, 2, 3) + return plucker + +def custom_meshgrid(*args): + """Copied from https://github.com/hehao13/CameraCtrl/blob/main/inference.py + """ + # ref: https://pytorch.org/docs/stable/generated/torch.meshgrid.html?highlight=meshgrid#torch.meshgrid + if pver.parse(torch.__version__) < pver.parse('1.10'): + return torch.meshgrid(*args) + else: + return torch.meshgrid(*args, indexing='ij') + + +def process_pose_file(pose_file_path, width=672, height=384, original_pose_width=1280, original_pose_height=720, device='cpu', return_poses=False): + """Modified from https://github.com/hehao13/CameraCtrl/blob/main/inference.py + """ + with open(pose_file_path, 'r') as f: + poses = f.readlines() + + poses = [pose.strip().split(' ') for pose in poses[1:]] + cam_params = [[float(x) for x in pose] for pose in poses] + if return_poses: + return cam_params + else: + cam_params = [Camera(cam_param) for cam_param in cam_params] + + sample_wh_ratio = width / height + pose_wh_ratio = original_pose_width / original_pose_height # Assuming placeholder ratios, change as needed + + if pose_wh_ratio > sample_wh_ratio: + resized_ori_w = height * pose_wh_ratio + for cam_param in cam_params: + cam_param.fx = resized_ori_w * cam_param.fx / width + else: + resized_ori_h = width / pose_wh_ratio + for cam_param in cam_params: + cam_param.fy = resized_ori_h * cam_param.fy / height + + intrinsic = np.asarray([[cam_param.fx * width, + cam_param.fy * height, + cam_param.cx * width, + cam_param.cy * height] + for cam_param in cam_params], dtype=np.float32) + + K = torch.as_tensor(intrinsic)[None] # [1, 1, 4] + c2ws = get_relative_pose(cam_params) # Assuming this function is defined elsewhere + c2ws = torch.as_tensor(c2ws)[None] # [1, n_frame, 4, 4] + plucker_embedding = ray_condition(K, c2ws, height, width, device=device)[0].permute(0, 3, 1, 2).contiguous() # V, 6, H, W + plucker_embedding = plucker_embedding[None] + plucker_embedding = rearrange(plucker_embedding, "b f c h w -> b f h w c")[0] + return plucker_embedding + + + + + + + +def process_pose_params(cam_params, width=672, height=384, original_pose_width=1280, original_pose_height=720, device='cpu'): + """Modified from https://github.com/hehao13/CameraCtrl/blob/main/inference.py + """ + cam_params = [Camera(cam_param) for cam_param in cam_params] + + sample_wh_ratio = width / height + pose_wh_ratio = original_pose_width / original_pose_height # Assuming placeholder ratios, change as needed + + if pose_wh_ratio > sample_wh_ratio: + resized_ori_w = height * pose_wh_ratio + for cam_param in cam_params: + cam_param.fx = resized_ori_w * cam_param.fx / width + else: + resized_ori_h = width / pose_wh_ratio + for cam_param in cam_params: + cam_param.fy = resized_ori_h * cam_param.fy / height + + intrinsic = np.asarray([[cam_param.fx * width, + cam_param.fy * height, + cam_param.cx * width, + cam_param.cy * height] + for cam_param in cam_params], dtype=np.float32) + + K = torch.as_tensor(intrinsic)[None] # [1, 1, 4] + c2ws = get_relative_pose(cam_params) # Assuming this function is defined elsewhere + c2ws = torch.as_tensor(c2ws)[None] # [1, n_frame, 4, 4] + plucker_embedding = ray_condition(K, c2ws, height, width, device=device)[0].permute(0, 3, 1, 2).contiguous() # V, 6, H, W + plucker_embedding = plucker_embedding[None] + plucker_embedding = rearrange(plucker_embedding, "b f c h w -> b f h w c")[0] + return plucker_embedding \ No newline at end of file diff --git a/dchen/camera_information.txt b/dchen/camera_information.txt new file mode 100644 index 0000000..3e277a9 --- /dev/null +++ b/dchen/camera_information.txt @@ -0,0 +1,82 @@ + +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 0.018518518518518517 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 0.037037037037037035 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 0.05555555555555555 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 0.07407407407407407 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 0.09259259259259259 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 0.1111111111111111 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 0.12962962962962962 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 0.14814814814814814 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 0.16666666666666666 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 0.18518518518518517 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 0.2037037037037037 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 0.2222222222222222 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 0.24074074074074073 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 0.25925925925925924 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 0.2777777777777778 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 0.2962962962962963 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 0.31481481481481477 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 0.3333333333333333 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 0.35185185185185186 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 0.37037037037037035 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 0.38888888888888884 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 0.4074074074074074 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 0.42592592592592593 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 0.4444444444444444 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 0.4629629629629629 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 0.48148148148148145 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 0.5 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 0.5185185185185185 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 0.537037037037037 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 0.5555555555555556 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 0.5740740740740741 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 0.5925925925925926 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 0.611111111111111 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 0.6296296296296295 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 0.6481481481481481 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 0.6666666666666666 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 0.6851851851851851 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 0.7037037037037037 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 0.7222222222222222 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 0.7407407407407407 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 0.7592592592592593 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 0.7777777777777777 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 0.7962962962962963 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 0.8148148148148148 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 0.8333333333333334 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 0.8518518518518519 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 0.8703703703703705 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 0.8888888888888888 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 0.9074074074074074 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 0.9259259259259258 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 0.9444444444444444 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 0.9629629629629629 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 0.9814814814814815 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 1.0 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 1.0185185185185186 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 1.037037037037037 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 1.0555555555555556 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 1.074074074074074 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 1.0925925925925926 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 1.1111111111111112 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 1.1296296296296298 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 1.1481481481481481 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 1.1666666666666667 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 1.1851851851851851 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 1.2037037037037037 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 1.222222222222222 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 1.2407407407407407 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 1.259259259259259 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 1.2777777777777777 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 1.2962962962962963 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 1.3148148148148149 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 1.3333333333333333 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 1.3518518518518519 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 1.3703703703703702 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 1.3888888888888888 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 1.4074074074074074 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 1.425925925925926 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 1.4444444444444444 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 1.462962962962963 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 +0 0.532139961 0.946026558 0.5 0.5 0 0 1.0 0.0 0.0 1.4814814814814814 0.0 1.0 0.0 0.0 0.0 0.0 1.0 0.0 diff --git a/diffsynth.egg-info/PKG-INFO b/diffsynth.egg-info/PKG-INFO new file mode 100644 index 0000000..9e1c1c4 --- /dev/null +++ b/diffsynth.egg-info/PKG-INFO @@ -0,0 +1,33 @@ +Metadata-Version: 2.2 +Name: diffsynth +Version: 1.1.7 +Summary: Enjoy the magic of Diffusion models! +Author: Artiprocher +License: UNKNOWN +Platform: UNKNOWN +Classifier: Programming Language :: Python :: 3 +Classifier: License :: OSI Approved :: Apache Software License +Classifier: Operating System :: OS Independent +Requires-Python: >=3.6 +License-File: LICENSE +Requires-Dist: torch>=2.0.0 +Requires-Dist: torchvision +Requires-Dist: cupy-cuda12x +Requires-Dist: transformers +Requires-Dist: controlnet-aux==0.0.7 +Requires-Dist: imageio +Requires-Dist: imageio[ffmpeg] +Requires-Dist: safetensors +Requires-Dist: einops +Requires-Dist: sentencepiece +Requires-Dist: protobuf +Requires-Dist: modelscope +Requires-Dist: ftfy +Requires-Dist: pynvml +Dynamic: author +Dynamic: classifier +Dynamic: requires-dist +Dynamic: requires-python +Dynamic: summary + +UNKNOWN diff --git a/diffsynth.egg-info/SOURCES.txt b/diffsynth.egg-info/SOURCES.txt new file mode 100644 index 0000000..9d308cc --- /dev/null +++ b/diffsynth.egg-info/SOURCES.txt @@ -0,0 +1,226 @@ +LICENSE +README.md +setup.py +diffsynth/__init__.py +diffsynth.egg-info/PKG-INFO +diffsynth.egg-info/SOURCES.txt +diffsynth.egg-info/dependency_links.txt +diffsynth.egg-info/requires.txt +diffsynth.egg-info/top_level.txt +diffsynth/configs/__init__.py +diffsynth/configs/model_config.py +diffsynth/controlnets/__init__.py +diffsynth/controlnets/controlnet_unit.py +diffsynth/controlnets/processors.py +diffsynth/data/__init__.py +diffsynth/data/simple_text_image.py +diffsynth/data/video.py +diffsynth/distributed/__init__.py +diffsynth/distributed/xdit_context_parallel.py +diffsynth/extensions/__init__.py +diffsynth/extensions/ESRGAN/__init__.py +diffsynth/extensions/FastBlend/__init__.py +diffsynth/extensions/FastBlend/api.py +diffsynth/extensions/FastBlend/cupy_kernels.py +diffsynth/extensions/FastBlend/data.py +diffsynth/extensions/FastBlend/patch_match.py +diffsynth/extensions/FastBlend/runners/__init__.py +diffsynth/extensions/FastBlend/runners/accurate.py +diffsynth/extensions/FastBlend/runners/balanced.py +diffsynth/extensions/FastBlend/runners/fast.py +diffsynth/extensions/FastBlend/runners/interpolation.py +diffsynth/extensions/ImageQualityMetric/__init__.py +diffsynth/extensions/ImageQualityMetric/aesthetic.py +diffsynth/extensions/ImageQualityMetric/clip.py +diffsynth/extensions/ImageQualityMetric/config.py +diffsynth/extensions/ImageQualityMetric/hps.py +diffsynth/extensions/ImageQualityMetric/imagereward.py +diffsynth/extensions/ImageQualityMetric/mps.py +diffsynth/extensions/ImageQualityMetric/pickscore.py +diffsynth/extensions/ImageQualityMetric/BLIP/__init__.py +diffsynth/extensions/ImageQualityMetric/BLIP/blip.py +diffsynth/extensions/ImageQualityMetric/BLIP/blip_pretrain.py +diffsynth/extensions/ImageQualityMetric/BLIP/med.py +diffsynth/extensions/ImageQualityMetric/BLIP/vit.py +diffsynth/extensions/ImageQualityMetric/open_clip/__init__.py +diffsynth/extensions/ImageQualityMetric/open_clip/coca_model.py +diffsynth/extensions/ImageQualityMetric/open_clip/constants.py +diffsynth/extensions/ImageQualityMetric/open_clip/factory.py +diffsynth/extensions/ImageQualityMetric/open_clip/generation_utils.py +diffsynth/extensions/ImageQualityMetric/open_clip/hf_configs.py +diffsynth/extensions/ImageQualityMetric/open_clip/hf_model.py +diffsynth/extensions/ImageQualityMetric/open_clip/loss.py +diffsynth/extensions/ImageQualityMetric/open_clip/model.py +diffsynth/extensions/ImageQualityMetric/open_clip/modified_resnet.py +diffsynth/extensions/ImageQualityMetric/open_clip/openai.py +diffsynth/extensions/ImageQualityMetric/open_clip/pretrained.py +diffsynth/extensions/ImageQualityMetric/open_clip/push_to_hf_hub.py +diffsynth/extensions/ImageQualityMetric/open_clip/timm_model.py +diffsynth/extensions/ImageQualityMetric/open_clip/tokenizer.py +diffsynth/extensions/ImageQualityMetric/open_clip/transform.py +diffsynth/extensions/ImageQualityMetric/open_clip/transformer.py +diffsynth/extensions/ImageQualityMetric/open_clip/utils.py +diffsynth/extensions/ImageQualityMetric/open_clip/version.py +diffsynth/extensions/ImageQualityMetric/trainer/__init__.py +diffsynth/extensions/ImageQualityMetric/trainer/models/__init__.py +diffsynth/extensions/ImageQualityMetric/trainer/models/base_model.py +diffsynth/extensions/ImageQualityMetric/trainer/models/clip_model.py +diffsynth/extensions/ImageQualityMetric/trainer/models/cross_modeling.py +diffsynth/extensions/RIFE/__init__.py +diffsynth/lora/__init__.py +diffsynth/models/__init__.py +diffsynth/models/attention.py +diffsynth/models/cog_dit.py +diffsynth/models/cog_vae.py +diffsynth/models/downloader.py +diffsynth/models/flux_controlnet.py +diffsynth/models/flux_dit.py +diffsynth/models/flux_infiniteyou.py +diffsynth/models/flux_ipadapter.py +diffsynth/models/flux_text_encoder.py +diffsynth/models/flux_vae.py +diffsynth/models/hunyuan_dit.py +diffsynth/models/hunyuan_dit_text_encoder.py +diffsynth/models/hunyuan_video_dit.py +diffsynth/models/hunyuan_video_text_encoder.py +diffsynth/models/hunyuan_video_vae_decoder.py +diffsynth/models/hunyuan_video_vae_encoder.py +diffsynth/models/kolors_text_encoder.py +diffsynth/models/lora.py +diffsynth/models/model_manager.py +diffsynth/models/omnigen.py +diffsynth/models/qwenvl.py +diffsynth/models/sd3_dit.py +diffsynth/models/sd3_text_encoder.py +diffsynth/models/sd3_vae_decoder.py +diffsynth/models/sd3_vae_encoder.py +diffsynth/models/sd_controlnet.py +diffsynth/models/sd_ipadapter.py +diffsynth/models/sd_motion.py +diffsynth/models/sd_text_encoder.py +diffsynth/models/sd_unet.py +diffsynth/models/sd_vae_decoder.py +diffsynth/models/sd_vae_encoder.py +diffsynth/models/sdxl_controlnet.py +diffsynth/models/sdxl_ipadapter.py +diffsynth/models/sdxl_motion.py +diffsynth/models/sdxl_text_encoder.py +diffsynth/models/sdxl_unet.py +diffsynth/models/sdxl_vae_decoder.py +diffsynth/models/sdxl_vae_encoder.py +diffsynth/models/step1x_connector.py +diffsynth/models/stepvideo_dit.py +diffsynth/models/stepvideo_text_encoder.py +diffsynth/models/stepvideo_vae.py +diffsynth/models/svd_image_encoder.py +diffsynth/models/svd_unet.py +diffsynth/models/svd_vae_decoder.py +diffsynth/models/svd_vae_encoder.py +diffsynth/models/tiler.py +diffsynth/models/utils.py +diffsynth/models/wan_video_dit.py +diffsynth/models/wan_video_image_encoder.py +diffsynth/models/wan_video_motion_controller.py +diffsynth/models/wan_video_text_encoder.py +diffsynth/models/wan_video_vace.py +diffsynth/models/wan_video_vae.py +diffsynth/pipelines/__init__.py +diffsynth/pipelines/base.py +diffsynth/pipelines/cog_video.py +diffsynth/pipelines/dancer.py +diffsynth/pipelines/flux_image.py +diffsynth/pipelines/hunyuan_image.py +diffsynth/pipelines/hunyuan_video.py +diffsynth/pipelines/omnigen_image.py +diffsynth/pipelines/pipeline_runner.py +diffsynth/pipelines/sd3_image.py +diffsynth/pipelines/sd_image.py +diffsynth/pipelines/sd_video.py +diffsynth/pipelines/sdxl_image.py +diffsynth/pipelines/sdxl_video.py +diffsynth/pipelines/step_video.py +diffsynth/pipelines/svd_video.py +diffsynth/pipelines/wan_video.py +diffsynth/pipelines/wan_video_new.py +diffsynth/processors/FastBlend.py +diffsynth/processors/PILEditor.py +diffsynth/processors/RIFE.py +diffsynth/processors/__init__.py +diffsynth/processors/base.py +diffsynth/processors/sequencial_processor.py +diffsynth/prompters/__init__.py +diffsynth/prompters/base_prompter.py +diffsynth/prompters/cog_prompter.py +diffsynth/prompters/flux_prompter.py +diffsynth/prompters/hunyuan_dit_prompter.py +diffsynth/prompters/hunyuan_video_prompter.py +diffsynth/prompters/kolors_prompter.py +diffsynth/prompters/omnigen_prompter.py +diffsynth/prompters/omost.py +diffsynth/prompters/prompt_refiners.py +diffsynth/prompters/sd3_prompter.py +diffsynth/prompters/sd_prompter.py +diffsynth/prompters/sdxl_prompter.py +diffsynth/prompters/stepvideo_prompter.py +diffsynth/prompters/wan_prompter.py +diffsynth/schedulers/__init__.py +diffsynth/schedulers/continuous_ode.py +diffsynth/schedulers/ddim.py +diffsynth/schedulers/flow_match.py +diffsynth/tokenizer_configs/__init__.py +diffsynth/tokenizer_configs/cog/tokenizer/added_tokens.json +diffsynth/tokenizer_configs/cog/tokenizer/special_tokens_map.json +diffsynth/tokenizer_configs/cog/tokenizer/spiece.model +diffsynth/tokenizer_configs/cog/tokenizer/tokenizer_config.json +diffsynth/tokenizer_configs/flux/tokenizer_1/merges.txt +diffsynth/tokenizer_configs/flux/tokenizer_1/special_tokens_map.json +diffsynth/tokenizer_configs/flux/tokenizer_1/tokenizer_config.json +diffsynth/tokenizer_configs/flux/tokenizer_1/vocab.json +diffsynth/tokenizer_configs/flux/tokenizer_2/special_tokens_map.json +diffsynth/tokenizer_configs/flux/tokenizer_2/spiece.model +diffsynth/tokenizer_configs/flux/tokenizer_2/tokenizer.json +diffsynth/tokenizer_configs/flux/tokenizer_2/tokenizer_config.json +diffsynth/tokenizer_configs/hunyuan_dit/tokenizer/special_tokens_map.json +diffsynth/tokenizer_configs/hunyuan_dit/tokenizer/tokenizer_config.json +diffsynth/tokenizer_configs/hunyuan_dit/tokenizer/vocab.txt +diffsynth/tokenizer_configs/hunyuan_dit/tokenizer/vocab_org.txt +diffsynth/tokenizer_configs/hunyuan_dit/tokenizer_t5/config.json +diffsynth/tokenizer_configs/hunyuan_dit/tokenizer_t5/special_tokens_map.json +diffsynth/tokenizer_configs/hunyuan_dit/tokenizer_t5/spiece.model +diffsynth/tokenizer_configs/hunyuan_dit/tokenizer_t5/tokenizer_config.json +diffsynth/tokenizer_configs/hunyuan_video/tokenizer_1/merges.txt +diffsynth/tokenizer_configs/hunyuan_video/tokenizer_1/special_tokens_map.json +diffsynth/tokenizer_configs/hunyuan_video/tokenizer_1/tokenizer_config.json +diffsynth/tokenizer_configs/hunyuan_video/tokenizer_1/vocab.json +diffsynth/tokenizer_configs/hunyuan_video/tokenizer_2/preprocessor_config.json +diffsynth/tokenizer_configs/hunyuan_video/tokenizer_2/special_tokens_map.json +diffsynth/tokenizer_configs/hunyuan_video/tokenizer_2/tokenizer.json +diffsynth/tokenizer_configs/hunyuan_video/tokenizer_2/tokenizer_config.json +diffsynth/tokenizer_configs/kolors/tokenizer/tokenizer.model +diffsynth/tokenizer_configs/kolors/tokenizer/tokenizer_config.json +diffsynth/tokenizer_configs/kolors/tokenizer/vocab.txt +diffsynth/tokenizer_configs/stable_diffusion/tokenizer/merges.txt +diffsynth/tokenizer_configs/stable_diffusion/tokenizer/special_tokens_map.json +diffsynth/tokenizer_configs/stable_diffusion/tokenizer/tokenizer_config.json +diffsynth/tokenizer_configs/stable_diffusion/tokenizer/vocab.json +diffsynth/tokenizer_configs/stable_diffusion_3/tokenizer_1/merges.txt +diffsynth/tokenizer_configs/stable_diffusion_3/tokenizer_1/special_tokens_map.json +diffsynth/tokenizer_configs/stable_diffusion_3/tokenizer_1/tokenizer_config.json +diffsynth/tokenizer_configs/stable_diffusion_3/tokenizer_1/vocab.json +diffsynth/tokenizer_configs/stable_diffusion_3/tokenizer_2/merges.txt +diffsynth/tokenizer_configs/stable_diffusion_3/tokenizer_2/special_tokens_map.json +diffsynth/tokenizer_configs/stable_diffusion_3/tokenizer_2/tokenizer_config.json +diffsynth/tokenizer_configs/stable_diffusion_3/tokenizer_2/vocab.json +diffsynth/tokenizer_configs/stable_diffusion_3/tokenizer_3/special_tokens_map.json +diffsynth/tokenizer_configs/stable_diffusion_3/tokenizer_3/spiece.model +diffsynth/tokenizer_configs/stable_diffusion_3/tokenizer_3/tokenizer.json +diffsynth/tokenizer_configs/stable_diffusion_3/tokenizer_3/tokenizer_config.json +diffsynth/tokenizer_configs/stable_diffusion_xl/tokenizer_2/merges.txt +diffsynth/tokenizer_configs/stable_diffusion_xl/tokenizer_2/special_tokens_map.json +diffsynth/tokenizer_configs/stable_diffusion_xl/tokenizer_2/tokenizer_config.json +diffsynth/tokenizer_configs/stable_diffusion_xl/tokenizer_2/vocab.json +diffsynth/trainers/__init__.py +diffsynth/trainers/text_to_image.py +diffsynth/trainers/utils.py +diffsynth/vram_management/__init__.py +diffsynth/vram_management/layers.py \ No newline at end of file diff --git a/diffsynth.egg-info/dependency_links.txt b/diffsynth.egg-info/dependency_links.txt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/diffsynth.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/diffsynth.egg-info/requires.txt b/diffsynth.egg-info/requires.txt new file mode 100644 index 0000000..92d8b48 --- /dev/null +++ b/diffsynth.egg-info/requires.txt @@ -0,0 +1,14 @@ +torch>=2.0.0 +torchvision +cupy-cuda12x +transformers +controlnet-aux==0.0.7 +imageio +imageio[ffmpeg] +safetensors +einops +sentencepiece +protobuf +modelscope +ftfy +pynvml diff --git a/diffsynth.egg-info/top_level.txt b/diffsynth.egg-info/top_level.txt new file mode 100644 index 0000000..a4c845e --- /dev/null +++ b/diffsynth.egg-info/top_level.txt @@ -0,0 +1 @@ +diffsynth diff --git a/diffsynth/__pycache__/__init__.cpython-310.pyc b/diffsynth/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000..aaeba36 Binary files /dev/null and b/diffsynth/__pycache__/__init__.cpython-310.pyc differ diff --git a/diffsynth/configs/__pycache__/__init__.cpython-310.pyc b/diffsynth/configs/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000..9100616 Binary files /dev/null and b/diffsynth/configs/__pycache__/__init__.cpython-310.pyc differ diff --git a/diffsynth/configs/__pycache__/model_config.cpython-310.pyc b/diffsynth/configs/__pycache__/model_config.cpython-310.pyc new file mode 100644 index 0000000..d2e9340 Binary files /dev/null and b/diffsynth/configs/__pycache__/model_config.cpython-310.pyc differ diff --git a/diffsynth/configs/model_config.py b/diffsynth/configs/model_config.py index dc4a1ff..ff485f0 100644 --- a/diffsynth/configs/model_config.py +++ b/diffsynth/configs/model_config.py @@ -133,6 +133,8 @@ model_loader_configs = [ (None, "3ef3b1f8e1dab83d5b71fd7b617f859f", ["wan_video_dit"], [WanModel], "civitai"), (None, "70ddad9d3a133785da5ea371aae09504", ["wan_video_dit"], [WanModel], "civitai"), (None, "26bde73488a92e64cc20b0a7485b9e5b", ["wan_video_dit"], [WanModel], "civitai"), + (None, "ac6a5aa74f4a0aab6f64eb9a72f19901", ["wan_video_dit"], [WanModel], "civitai"), + (None, "b61c605c2adbd23124d152ed28e049ae", ["wan_video_dit"], [WanModel], "civitai"), (None, "a61453409b67cd3246cf0c3bebad47ba", ["wan_video_dit", "wan_video_vace"], [WanModel, VaceWanModel], "civitai"), (None, "cb104773c6c2cb6df4f9529ad5c60d0b", ["wan_video_dit"], [WanModel], "diffusers"), (None, "9c8818c2cbea55eca56c7b447df170da", ["wan_video_text_encoder"], [WanTextEncoder], "civitai"), diff --git a/diffsynth/controlnets/__pycache__/__init__.cpython-310.pyc b/diffsynth/controlnets/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000..9ae16d4 Binary files /dev/null and b/diffsynth/controlnets/__pycache__/__init__.cpython-310.pyc differ diff --git a/diffsynth/controlnets/__pycache__/controlnet_unit.cpython-310.pyc b/diffsynth/controlnets/__pycache__/controlnet_unit.cpython-310.pyc new file mode 100644 index 0000000..4468d93 Binary files /dev/null and b/diffsynth/controlnets/__pycache__/controlnet_unit.cpython-310.pyc differ diff --git a/diffsynth/controlnets/__pycache__/processors.cpython-310.pyc b/diffsynth/controlnets/__pycache__/processors.cpython-310.pyc new file mode 100644 index 0000000..6a2babc Binary files /dev/null and b/diffsynth/controlnets/__pycache__/processors.cpython-310.pyc differ diff --git a/diffsynth/data/__pycache__/__init__.cpython-310.pyc b/diffsynth/data/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000..a73cba5 Binary files /dev/null and b/diffsynth/data/__pycache__/__init__.cpython-310.pyc differ diff --git a/diffsynth/data/__pycache__/video.cpython-310.pyc b/diffsynth/data/__pycache__/video.cpython-310.pyc new file mode 100644 index 0000000..d77c90a Binary files /dev/null and b/diffsynth/data/__pycache__/video.cpython-310.pyc differ diff --git a/diffsynth/extensions/ESRGAN/__pycache__/__init__.cpython-310.pyc b/diffsynth/extensions/ESRGAN/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000..4a2585d Binary files /dev/null and b/diffsynth/extensions/ESRGAN/__pycache__/__init__.cpython-310.pyc differ diff --git a/diffsynth/extensions/RIFE/__pycache__/__init__.cpython-310.pyc b/diffsynth/extensions/RIFE/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000..caea266 Binary files /dev/null and b/diffsynth/extensions/RIFE/__pycache__/__init__.cpython-310.pyc differ diff --git a/diffsynth/extensions/__pycache__/__init__.cpython-310.pyc b/diffsynth/extensions/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000..b8d8be0 Binary files /dev/null and b/diffsynth/extensions/__pycache__/__init__.cpython-310.pyc differ diff --git a/diffsynth/lora/__pycache__/__init__.cpython-310.pyc b/diffsynth/lora/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000..76d6821 Binary files /dev/null and b/diffsynth/lora/__pycache__/__init__.cpython-310.pyc differ diff --git a/diffsynth/models/__pycache__/__init__.cpython-310.pyc b/diffsynth/models/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000..a785d62 Binary files /dev/null and b/diffsynth/models/__pycache__/__init__.cpython-310.pyc differ diff --git a/diffsynth/models/__pycache__/attention.cpython-310.pyc b/diffsynth/models/__pycache__/attention.cpython-310.pyc new file mode 100644 index 0000000..7f32e01 Binary files /dev/null and b/diffsynth/models/__pycache__/attention.cpython-310.pyc differ diff --git a/diffsynth/models/__pycache__/cog_dit.cpython-310.pyc b/diffsynth/models/__pycache__/cog_dit.cpython-310.pyc new file mode 100644 index 0000000..014f774 Binary files /dev/null and b/diffsynth/models/__pycache__/cog_dit.cpython-310.pyc differ diff --git a/diffsynth/models/__pycache__/cog_vae.cpython-310.pyc b/diffsynth/models/__pycache__/cog_vae.cpython-310.pyc new file mode 100644 index 0000000..afdb288 Binary files /dev/null and b/diffsynth/models/__pycache__/cog_vae.cpython-310.pyc differ diff --git a/diffsynth/models/__pycache__/downloader.cpython-310.pyc b/diffsynth/models/__pycache__/downloader.cpython-310.pyc new file mode 100644 index 0000000..63a8f91 Binary files /dev/null and b/diffsynth/models/__pycache__/downloader.cpython-310.pyc differ diff --git a/diffsynth/models/__pycache__/flux_controlnet.cpython-310.pyc b/diffsynth/models/__pycache__/flux_controlnet.cpython-310.pyc new file mode 100644 index 0000000..3b8a12b Binary files /dev/null and b/diffsynth/models/__pycache__/flux_controlnet.cpython-310.pyc differ diff --git a/diffsynth/models/__pycache__/flux_dit.cpython-310.pyc b/diffsynth/models/__pycache__/flux_dit.cpython-310.pyc new file mode 100644 index 0000000..453f82a Binary files /dev/null and b/diffsynth/models/__pycache__/flux_dit.cpython-310.pyc differ diff --git a/diffsynth/models/__pycache__/flux_infiniteyou.cpython-310.pyc b/diffsynth/models/__pycache__/flux_infiniteyou.cpython-310.pyc new file mode 100644 index 0000000..b70580c Binary files /dev/null and b/diffsynth/models/__pycache__/flux_infiniteyou.cpython-310.pyc differ diff --git a/diffsynth/models/__pycache__/flux_ipadapter.cpython-310.pyc b/diffsynth/models/__pycache__/flux_ipadapter.cpython-310.pyc new file mode 100644 index 0000000..fa2ca0e Binary files /dev/null and b/diffsynth/models/__pycache__/flux_ipadapter.cpython-310.pyc differ diff --git a/diffsynth/models/__pycache__/flux_text_encoder.cpython-310.pyc b/diffsynth/models/__pycache__/flux_text_encoder.cpython-310.pyc new file mode 100644 index 0000000..7c4c9fd Binary files /dev/null and b/diffsynth/models/__pycache__/flux_text_encoder.cpython-310.pyc differ diff --git a/diffsynth/models/__pycache__/flux_vae.cpython-310.pyc b/diffsynth/models/__pycache__/flux_vae.cpython-310.pyc new file mode 100644 index 0000000..8ef63de Binary files /dev/null and b/diffsynth/models/__pycache__/flux_vae.cpython-310.pyc differ diff --git a/diffsynth/models/__pycache__/hunyuan_dit.cpython-310.pyc b/diffsynth/models/__pycache__/hunyuan_dit.cpython-310.pyc new file mode 100644 index 0000000..bd7c96e Binary files /dev/null and b/diffsynth/models/__pycache__/hunyuan_dit.cpython-310.pyc differ diff --git a/diffsynth/models/__pycache__/hunyuan_dit_text_encoder.cpython-310.pyc b/diffsynth/models/__pycache__/hunyuan_dit_text_encoder.cpython-310.pyc new file mode 100644 index 0000000..91f1f7f Binary files /dev/null and b/diffsynth/models/__pycache__/hunyuan_dit_text_encoder.cpython-310.pyc differ diff --git a/diffsynth/models/__pycache__/hunyuan_video_dit.cpython-310.pyc b/diffsynth/models/__pycache__/hunyuan_video_dit.cpython-310.pyc new file mode 100644 index 0000000..a4c5b63 Binary files /dev/null and b/diffsynth/models/__pycache__/hunyuan_video_dit.cpython-310.pyc differ diff --git a/diffsynth/models/__pycache__/hunyuan_video_text_encoder.cpython-310.pyc b/diffsynth/models/__pycache__/hunyuan_video_text_encoder.cpython-310.pyc new file mode 100644 index 0000000..2ec0513 Binary files /dev/null and b/diffsynth/models/__pycache__/hunyuan_video_text_encoder.cpython-310.pyc differ diff --git a/diffsynth/models/__pycache__/hunyuan_video_vae_decoder.cpython-310.pyc b/diffsynth/models/__pycache__/hunyuan_video_vae_decoder.cpython-310.pyc new file mode 100644 index 0000000..a23f94c Binary files /dev/null and b/diffsynth/models/__pycache__/hunyuan_video_vae_decoder.cpython-310.pyc differ diff --git a/diffsynth/models/__pycache__/hunyuan_video_vae_encoder.cpython-310.pyc b/diffsynth/models/__pycache__/hunyuan_video_vae_encoder.cpython-310.pyc new file mode 100644 index 0000000..5870a79 Binary files /dev/null and b/diffsynth/models/__pycache__/hunyuan_video_vae_encoder.cpython-310.pyc differ diff --git a/diffsynth/models/__pycache__/kolors_text_encoder.cpython-310.pyc b/diffsynth/models/__pycache__/kolors_text_encoder.cpython-310.pyc new file mode 100644 index 0000000..207a660 Binary files /dev/null and b/diffsynth/models/__pycache__/kolors_text_encoder.cpython-310.pyc differ diff --git a/diffsynth/models/__pycache__/lora.cpython-310.pyc b/diffsynth/models/__pycache__/lora.cpython-310.pyc new file mode 100644 index 0000000..8ccec57 Binary files /dev/null and b/diffsynth/models/__pycache__/lora.cpython-310.pyc differ diff --git a/diffsynth/models/__pycache__/model_manager.cpython-310.pyc b/diffsynth/models/__pycache__/model_manager.cpython-310.pyc new file mode 100644 index 0000000..dee3901 Binary files /dev/null and b/diffsynth/models/__pycache__/model_manager.cpython-310.pyc differ diff --git a/diffsynth/models/__pycache__/omnigen.cpython-310.pyc b/diffsynth/models/__pycache__/omnigen.cpython-310.pyc new file mode 100644 index 0000000..9d3ec4c Binary files /dev/null and b/diffsynth/models/__pycache__/omnigen.cpython-310.pyc differ diff --git a/diffsynth/models/__pycache__/sd3_dit.cpython-310.pyc b/diffsynth/models/__pycache__/sd3_dit.cpython-310.pyc new file mode 100644 index 0000000..51cfbdc Binary files /dev/null and b/diffsynth/models/__pycache__/sd3_dit.cpython-310.pyc differ diff --git a/diffsynth/models/__pycache__/sd3_text_encoder.cpython-310.pyc b/diffsynth/models/__pycache__/sd3_text_encoder.cpython-310.pyc new file mode 100644 index 0000000..88687fb Binary files /dev/null and b/diffsynth/models/__pycache__/sd3_text_encoder.cpython-310.pyc differ diff --git a/diffsynth/models/__pycache__/sd3_vae_decoder.cpython-310.pyc b/diffsynth/models/__pycache__/sd3_vae_decoder.cpython-310.pyc new file mode 100644 index 0000000..1e9b70b Binary files /dev/null and b/diffsynth/models/__pycache__/sd3_vae_decoder.cpython-310.pyc differ diff --git a/diffsynth/models/__pycache__/sd3_vae_encoder.cpython-310.pyc b/diffsynth/models/__pycache__/sd3_vae_encoder.cpython-310.pyc new file mode 100644 index 0000000..14d53bf Binary files /dev/null and b/diffsynth/models/__pycache__/sd3_vae_encoder.cpython-310.pyc differ diff --git a/diffsynth/models/__pycache__/sd_controlnet.cpython-310.pyc b/diffsynth/models/__pycache__/sd_controlnet.cpython-310.pyc new file mode 100644 index 0000000..629d988 Binary files /dev/null and b/diffsynth/models/__pycache__/sd_controlnet.cpython-310.pyc differ diff --git a/diffsynth/models/__pycache__/sd_ipadapter.cpython-310.pyc b/diffsynth/models/__pycache__/sd_ipadapter.cpython-310.pyc new file mode 100644 index 0000000..550c8a2 Binary files /dev/null and b/diffsynth/models/__pycache__/sd_ipadapter.cpython-310.pyc differ diff --git a/diffsynth/models/__pycache__/sd_motion.cpython-310.pyc b/diffsynth/models/__pycache__/sd_motion.cpython-310.pyc new file mode 100644 index 0000000..31d6980 Binary files /dev/null and b/diffsynth/models/__pycache__/sd_motion.cpython-310.pyc differ diff --git a/diffsynth/models/__pycache__/sd_text_encoder.cpython-310.pyc b/diffsynth/models/__pycache__/sd_text_encoder.cpython-310.pyc new file mode 100644 index 0000000..84cb35c Binary files /dev/null and b/diffsynth/models/__pycache__/sd_text_encoder.cpython-310.pyc differ diff --git a/diffsynth/models/__pycache__/sd_unet.cpython-310.pyc b/diffsynth/models/__pycache__/sd_unet.cpython-310.pyc new file mode 100644 index 0000000..906ddae Binary files /dev/null and b/diffsynth/models/__pycache__/sd_unet.cpython-310.pyc differ diff --git a/diffsynth/models/__pycache__/sd_vae_decoder.cpython-310.pyc b/diffsynth/models/__pycache__/sd_vae_decoder.cpython-310.pyc new file mode 100644 index 0000000..0ef2104 Binary files /dev/null and b/diffsynth/models/__pycache__/sd_vae_decoder.cpython-310.pyc differ diff --git a/diffsynth/models/__pycache__/sd_vae_encoder.cpython-310.pyc b/diffsynth/models/__pycache__/sd_vae_encoder.cpython-310.pyc new file mode 100644 index 0000000..88a665a Binary files /dev/null and b/diffsynth/models/__pycache__/sd_vae_encoder.cpython-310.pyc differ diff --git a/diffsynth/models/__pycache__/sdxl_controlnet.cpython-310.pyc b/diffsynth/models/__pycache__/sdxl_controlnet.cpython-310.pyc new file mode 100644 index 0000000..46b32b7 Binary files /dev/null and b/diffsynth/models/__pycache__/sdxl_controlnet.cpython-310.pyc differ diff --git a/diffsynth/models/__pycache__/sdxl_ipadapter.cpython-310.pyc b/diffsynth/models/__pycache__/sdxl_ipadapter.cpython-310.pyc new file mode 100644 index 0000000..90ae52c Binary files /dev/null and b/diffsynth/models/__pycache__/sdxl_ipadapter.cpython-310.pyc differ diff --git a/diffsynth/models/__pycache__/sdxl_motion.cpython-310.pyc b/diffsynth/models/__pycache__/sdxl_motion.cpython-310.pyc new file mode 100644 index 0000000..2388220 Binary files /dev/null and b/diffsynth/models/__pycache__/sdxl_motion.cpython-310.pyc differ diff --git a/diffsynth/models/__pycache__/sdxl_text_encoder.cpython-310.pyc b/diffsynth/models/__pycache__/sdxl_text_encoder.cpython-310.pyc new file mode 100644 index 0000000..c30896e Binary files /dev/null and b/diffsynth/models/__pycache__/sdxl_text_encoder.cpython-310.pyc differ diff --git a/diffsynth/models/__pycache__/sdxl_unet.cpython-310.pyc b/diffsynth/models/__pycache__/sdxl_unet.cpython-310.pyc new file mode 100644 index 0000000..b507240 Binary files /dev/null and b/diffsynth/models/__pycache__/sdxl_unet.cpython-310.pyc differ diff --git a/diffsynth/models/__pycache__/sdxl_vae_decoder.cpython-310.pyc b/diffsynth/models/__pycache__/sdxl_vae_decoder.cpython-310.pyc new file mode 100644 index 0000000..83e43c1 Binary files /dev/null and b/diffsynth/models/__pycache__/sdxl_vae_decoder.cpython-310.pyc differ diff --git a/diffsynth/models/__pycache__/sdxl_vae_encoder.cpython-310.pyc b/diffsynth/models/__pycache__/sdxl_vae_encoder.cpython-310.pyc new file mode 100644 index 0000000..f31abe6 Binary files /dev/null and b/diffsynth/models/__pycache__/sdxl_vae_encoder.cpython-310.pyc differ diff --git a/diffsynth/models/__pycache__/step1x_connector.cpython-310.pyc b/diffsynth/models/__pycache__/step1x_connector.cpython-310.pyc new file mode 100644 index 0000000..9e00094 Binary files /dev/null and b/diffsynth/models/__pycache__/step1x_connector.cpython-310.pyc differ diff --git a/diffsynth/models/__pycache__/stepvideo_dit.cpython-310.pyc b/diffsynth/models/__pycache__/stepvideo_dit.cpython-310.pyc new file mode 100644 index 0000000..c5fd431 Binary files /dev/null and b/diffsynth/models/__pycache__/stepvideo_dit.cpython-310.pyc differ diff --git a/diffsynth/models/__pycache__/stepvideo_text_encoder.cpython-310.pyc b/diffsynth/models/__pycache__/stepvideo_text_encoder.cpython-310.pyc new file mode 100644 index 0000000..d6d1b88 Binary files /dev/null and b/diffsynth/models/__pycache__/stepvideo_text_encoder.cpython-310.pyc differ diff --git a/diffsynth/models/__pycache__/stepvideo_vae.cpython-310.pyc b/diffsynth/models/__pycache__/stepvideo_vae.cpython-310.pyc new file mode 100644 index 0000000..8b6264e Binary files /dev/null and b/diffsynth/models/__pycache__/stepvideo_vae.cpython-310.pyc differ diff --git a/diffsynth/models/__pycache__/svd_image_encoder.cpython-310.pyc b/diffsynth/models/__pycache__/svd_image_encoder.cpython-310.pyc new file mode 100644 index 0000000..c2a17d6 Binary files /dev/null and b/diffsynth/models/__pycache__/svd_image_encoder.cpython-310.pyc differ diff --git a/diffsynth/models/__pycache__/svd_unet.cpython-310.pyc b/diffsynth/models/__pycache__/svd_unet.cpython-310.pyc new file mode 100644 index 0000000..211e031 Binary files /dev/null and b/diffsynth/models/__pycache__/svd_unet.cpython-310.pyc differ diff --git a/diffsynth/models/__pycache__/svd_vae_decoder.cpython-310.pyc b/diffsynth/models/__pycache__/svd_vae_decoder.cpython-310.pyc new file mode 100644 index 0000000..aff5c83 Binary files /dev/null and b/diffsynth/models/__pycache__/svd_vae_decoder.cpython-310.pyc differ diff --git a/diffsynth/models/__pycache__/svd_vae_encoder.cpython-310.pyc b/diffsynth/models/__pycache__/svd_vae_encoder.cpython-310.pyc new file mode 100644 index 0000000..df39468 Binary files /dev/null and b/diffsynth/models/__pycache__/svd_vae_encoder.cpython-310.pyc differ diff --git a/diffsynth/models/__pycache__/tiler.cpython-310.pyc b/diffsynth/models/__pycache__/tiler.cpython-310.pyc new file mode 100644 index 0000000..2297791 Binary files /dev/null and b/diffsynth/models/__pycache__/tiler.cpython-310.pyc differ diff --git a/diffsynth/models/__pycache__/utils.cpython-310.pyc b/diffsynth/models/__pycache__/utils.cpython-310.pyc new file mode 100644 index 0000000..c136265 Binary files /dev/null and b/diffsynth/models/__pycache__/utils.cpython-310.pyc differ diff --git a/diffsynth/models/__pycache__/wan_video_dit.cpython-310.pyc b/diffsynth/models/__pycache__/wan_video_dit.cpython-310.pyc new file mode 100644 index 0000000..bf3b099 Binary files /dev/null and b/diffsynth/models/__pycache__/wan_video_dit.cpython-310.pyc differ diff --git a/diffsynth/models/__pycache__/wan_video_image_encoder.cpython-310.pyc b/diffsynth/models/__pycache__/wan_video_image_encoder.cpython-310.pyc new file mode 100644 index 0000000..b5af93a Binary files /dev/null and b/diffsynth/models/__pycache__/wan_video_image_encoder.cpython-310.pyc differ diff --git a/diffsynth/models/__pycache__/wan_video_motion_controller.cpython-310.pyc b/diffsynth/models/__pycache__/wan_video_motion_controller.cpython-310.pyc new file mode 100644 index 0000000..65bebb5 Binary files /dev/null and b/diffsynth/models/__pycache__/wan_video_motion_controller.cpython-310.pyc differ diff --git a/diffsynth/models/__pycache__/wan_video_text_encoder.cpython-310.pyc b/diffsynth/models/__pycache__/wan_video_text_encoder.cpython-310.pyc new file mode 100644 index 0000000..4f75543 Binary files /dev/null and b/diffsynth/models/__pycache__/wan_video_text_encoder.cpython-310.pyc differ diff --git a/diffsynth/models/__pycache__/wan_video_vace.cpython-310.pyc b/diffsynth/models/__pycache__/wan_video_vace.cpython-310.pyc new file mode 100644 index 0000000..9a47c1d Binary files /dev/null and b/diffsynth/models/__pycache__/wan_video_vace.cpython-310.pyc differ diff --git a/diffsynth/models/__pycache__/wan_video_vae.cpython-310.pyc b/diffsynth/models/__pycache__/wan_video_vae.cpython-310.pyc new file mode 100644 index 0000000..75921a4 Binary files /dev/null and b/diffsynth/models/__pycache__/wan_video_vae.cpython-310.pyc differ diff --git a/diffsynth/models/wan_video_dit.py b/diffsynth/models/wan_video_dit.py index 1ca3239..ab63bfb 100644 --- a/diffsynth/models/wan_video_dit.py +++ b/diffsynth/models/wan_video_dit.py @@ -5,6 +5,10 @@ import math from typing import Tuple, Optional from einops import rearrange from .utils import hash_state_dict_keys + +from dchen.camera_adapter import SimpleAdapter + + try: import flash_attn_interface FLASH_ATTN_3_AVAILABLE = True @@ -273,6 +277,8 @@ class WanModel(torch.nn.Module): has_image_input: bool, has_image_pos_emb: bool = False, has_ref_conv: bool = False, + add_control_adapter: bool = False, + in_dim_control_adapter: int = 24, ): super().__init__() self.dim = dim @@ -309,8 +315,17 @@ class WanModel(torch.nn.Module): self.has_image_pos_emb = has_image_pos_emb self.has_ref_conv = has_ref_conv - def patchify(self, x: torch.Tensor): + if add_control_adapter: + self.control_adapter = SimpleAdapter(in_dim_control_adapter, dim, kernel_size=patch_size[1:], stride=patch_size[1:]) + else: + self.control_adapter = None + + def patchify(self, x: torch.Tensor, control_camera_latents_input: torch.Tensor = None): x = self.patch_embedding(x) + if self.control_adapter is not None and control_camera_latents_input is not None: + y_camera = self.control_adapter(control_camera_latents_input) + x = [u + v for u, v in zip(x, y_camera)] + x = x[0].unsqueeze(0) grid_size = x.shape[2:] x = rearrange(x, 'b c f h w -> b (f h w) c').contiguous() return x, grid_size # x, grid_size: (f, h, w) @@ -612,6 +627,42 @@ class WanModelStateDictConverter: "eps": 1e-6, "has_ref_conv": True } + elif hash_state_dict_keys(state_dict) == "ac6a5aa74f4a0aab6f64eb9a72f19901": + # 1.3B PAI control-camera v1.1 + config = { + "has_image_input": True, + "patch_size": [1, 2, 2], + "in_dim": 32, + "dim": 1536, + "ffn_dim": 8960, + "freq_dim": 256, + "text_dim": 4096, + "out_dim": 16, + "num_heads": 12, + "num_layers": 30, + "eps": 1e-6, + "has_ref_conv": False, + "add_control_adapter": True, + "in_dim_control_adapter": 24, + } + elif hash_state_dict_keys(state_dict) == "b61c605c2adbd23124d152ed28e049ae": + # 14B PAI control-camera v1.1 + config = { + "has_image_input": True, + "patch_size": [1, 2, 2], + "in_dim": 32, + "dim": 5120, + "ffn_dim": 13824, + "freq_dim": 256, + "text_dim": 4096, + "out_dim": 16, + "num_heads": 40, + "num_layers": 40, + "eps": 1e-6, + "has_ref_conv": False, + "add_control_adapter": True, + "in_dim_control_adapter": 24, + } else: config = {} return state_dict, config diff --git a/diffsynth/pipelines/__pycache__/__init__.cpython-310.pyc b/diffsynth/pipelines/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000..9c917b0 Binary files /dev/null and b/diffsynth/pipelines/__pycache__/__init__.cpython-310.pyc differ diff --git a/diffsynth/pipelines/__pycache__/base.cpython-310.pyc b/diffsynth/pipelines/__pycache__/base.cpython-310.pyc new file mode 100644 index 0000000..f26b6b1 Binary files /dev/null and b/diffsynth/pipelines/__pycache__/base.cpython-310.pyc differ diff --git a/diffsynth/pipelines/__pycache__/cog_video.cpython-310.pyc b/diffsynth/pipelines/__pycache__/cog_video.cpython-310.pyc new file mode 100644 index 0000000..d8b0ac9 Binary files /dev/null and b/diffsynth/pipelines/__pycache__/cog_video.cpython-310.pyc differ diff --git a/diffsynth/pipelines/__pycache__/dancer.cpython-310.pyc b/diffsynth/pipelines/__pycache__/dancer.cpython-310.pyc new file mode 100644 index 0000000..01d2488 Binary files /dev/null and b/diffsynth/pipelines/__pycache__/dancer.cpython-310.pyc differ diff --git a/diffsynth/pipelines/__pycache__/flux_image.cpython-310.pyc b/diffsynth/pipelines/__pycache__/flux_image.cpython-310.pyc new file mode 100644 index 0000000..c3ce9b4 Binary files /dev/null and b/diffsynth/pipelines/__pycache__/flux_image.cpython-310.pyc differ diff --git a/diffsynth/pipelines/__pycache__/hunyuan_image.cpython-310.pyc b/diffsynth/pipelines/__pycache__/hunyuan_image.cpython-310.pyc new file mode 100644 index 0000000..12f9f95 Binary files /dev/null and b/diffsynth/pipelines/__pycache__/hunyuan_image.cpython-310.pyc differ diff --git a/diffsynth/pipelines/__pycache__/hunyuan_video.cpython-310.pyc b/diffsynth/pipelines/__pycache__/hunyuan_video.cpython-310.pyc new file mode 100644 index 0000000..6a8ff5e Binary files /dev/null and b/diffsynth/pipelines/__pycache__/hunyuan_video.cpython-310.pyc differ diff --git a/diffsynth/pipelines/__pycache__/omnigen_image.cpython-310.pyc b/diffsynth/pipelines/__pycache__/omnigen_image.cpython-310.pyc new file mode 100644 index 0000000..95b912f Binary files /dev/null and b/diffsynth/pipelines/__pycache__/omnigen_image.cpython-310.pyc differ diff --git a/diffsynth/pipelines/__pycache__/pipeline_runner.cpython-310.pyc b/diffsynth/pipelines/__pycache__/pipeline_runner.cpython-310.pyc new file mode 100644 index 0000000..cedc1e4 Binary files /dev/null and b/diffsynth/pipelines/__pycache__/pipeline_runner.cpython-310.pyc differ diff --git a/diffsynth/pipelines/__pycache__/sd3_image.cpython-310.pyc b/diffsynth/pipelines/__pycache__/sd3_image.cpython-310.pyc new file mode 100644 index 0000000..3b15ff2 Binary files /dev/null and b/diffsynth/pipelines/__pycache__/sd3_image.cpython-310.pyc differ diff --git a/diffsynth/pipelines/__pycache__/sd_image.cpython-310.pyc b/diffsynth/pipelines/__pycache__/sd_image.cpython-310.pyc new file mode 100644 index 0000000..aa89749 Binary files /dev/null and b/diffsynth/pipelines/__pycache__/sd_image.cpython-310.pyc differ diff --git a/diffsynth/pipelines/__pycache__/sd_video.cpython-310.pyc b/diffsynth/pipelines/__pycache__/sd_video.cpython-310.pyc new file mode 100644 index 0000000..e57d7be Binary files /dev/null and b/diffsynth/pipelines/__pycache__/sd_video.cpython-310.pyc differ diff --git a/diffsynth/pipelines/__pycache__/sdxl_image.cpython-310.pyc b/diffsynth/pipelines/__pycache__/sdxl_image.cpython-310.pyc new file mode 100644 index 0000000..c61c4af Binary files /dev/null and b/diffsynth/pipelines/__pycache__/sdxl_image.cpython-310.pyc differ diff --git a/diffsynth/pipelines/__pycache__/sdxl_video.cpython-310.pyc b/diffsynth/pipelines/__pycache__/sdxl_video.cpython-310.pyc new file mode 100644 index 0000000..9b445fb Binary files /dev/null and b/diffsynth/pipelines/__pycache__/sdxl_video.cpython-310.pyc differ diff --git a/diffsynth/pipelines/__pycache__/step_video.cpython-310.pyc b/diffsynth/pipelines/__pycache__/step_video.cpython-310.pyc new file mode 100644 index 0000000..8c86b2b Binary files /dev/null and b/diffsynth/pipelines/__pycache__/step_video.cpython-310.pyc differ diff --git a/diffsynth/pipelines/__pycache__/svd_video.cpython-310.pyc b/diffsynth/pipelines/__pycache__/svd_video.cpython-310.pyc new file mode 100644 index 0000000..6471e96 Binary files /dev/null and b/diffsynth/pipelines/__pycache__/svd_video.cpython-310.pyc differ diff --git a/diffsynth/pipelines/__pycache__/wan_video.cpython-310.pyc b/diffsynth/pipelines/__pycache__/wan_video.cpython-310.pyc new file mode 100644 index 0000000..aeca1d6 Binary files /dev/null and b/diffsynth/pipelines/__pycache__/wan_video.cpython-310.pyc differ diff --git a/diffsynth/pipelines/__pycache__/wan_video_new.cpython-310.pyc b/diffsynth/pipelines/__pycache__/wan_video_new.cpython-310.pyc new file mode 100644 index 0000000..31d0868 Binary files /dev/null and b/diffsynth/pipelines/__pycache__/wan_video_new.cpython-310.pyc differ diff --git a/diffsynth/pipelines/wan_video_new.py b/diffsynth/pipelines/wan_video_new.py index eb9dfba..8c146b9 100644 --- a/diffsynth/pipelines/wan_video_new.py +++ b/diffsynth/pipelines/wan_video_new.py @@ -24,7 +24,6 @@ from ..vram_management import enable_vram_management, AutoWrappedModule, AutoWra from ..lora import GeneralLoRALoader - class BasePipeline(torch.nn.Module): def __init__( @@ -208,6 +207,7 @@ class WanVideoPipeline(BasePipeline): WanVideoUnit_InputVideoEmbedder(), WanVideoUnit_PromptEmbedder(), WanVideoUnit_ImageEmbedder(), + WanVideoUnit_FunCamera(), WanVideoUnit_FunControl(), WanVideoUnit_FunReference(), WanVideoUnit_SpeedControl(), @@ -473,6 +473,8 @@ class WanVideoPipeline(BasePipeline): tea_cache_model_id: Optional[str] = "", # progress_bar progress_bar_cmd=tqdm, + # Camera control + control_camera_video: Optional[torch.Tensor] = None ): # Scheduler self.scheduler.set_timesteps(num_inference_steps, denoising_strength=denoising_strength, shift=sigma_shift) @@ -491,6 +493,7 @@ class WanVideoPipeline(BasePipeline): "end_image": end_image, "input_video": input_video, "denoising_strength": denoising_strength, "control_video": control_video, "reference_image": reference_image, + "control_camera_video": control_camera_video, "vace_video": vace_video, "vace_video_mask": vace_video_mask, "vace_reference_image": vace_reference_image, "vace_scale": vace_scale, "seed": seed, "rand_device": rand_device, "height": height, "width": width, "num_frames": num_frames, @@ -653,15 +656,17 @@ class WanVideoUnit_PromptEmbedder(PipelineUnit): class WanVideoUnit_ImageEmbedder(PipelineUnit): def __init__(self): super().__init__( - input_params=("input_image", "end_image", "num_frames", "height", "width", "tiled", "tile_size", "tile_stride"), + input_params=("input_image", "end_image", "num_frames", "height", "width", "tiled", "tile_size", "tile_stride", "control_camera_video","latents"), onload_model_names=("image_encoder", "vae") ) - def process(self, pipe: WanVideoPipeline, input_image, end_image, num_frames, height, width, tiled, tile_size, tile_stride): + def process(self, pipe: WanVideoPipeline, input_image, end_image, num_frames, height, width, tiled, tile_size, tile_stride, control_camera_video,latents): if input_image is None: return {} + pipe.load_models_to_device(self.onload_model_names) image = pipe.preprocess_image(input_image.resize((width, height))).to(pipe.device) + clip_context = pipe.image_encoder.encode_image([image]) msk = torch.ones(1, num_frames, height//8, width//8, device=pipe.device) msk[:, 1:] = 0 @@ -673,14 +678,13 @@ class WanVideoUnit_ImageEmbedder(PipelineUnit): msk[:, -1:] = 1 else: vae_input = torch.concat([image.transpose(0, 1), torch.zeros(3, num_frames-1, height, width).to(image.device)], dim=1) - + msk = torch.concat([torch.repeat_interleave(msk[:, 0:1], repeats=4, dim=1), msk[:, 1:]], dim=1) msk = msk.view(1, msk.shape[1] // 4, 4, height//8, width//8) msk = msk.transpose(1, 2)[0] - + y = pipe.vae.encode([vae_input.to(dtype=pipe.torch_dtype, device=pipe.device)], device=pipe.device, tiled=tiled, tile_size=tile_size, tile_stride=tile_stride)[0] y = y.to(dtype=pipe.torch_dtype, device=pipe.device) - y = torch.concat([msk, y]) y = y.unsqueeze(0) clip_context = clip_context.to(dtype=pipe.torch_dtype, device=pipe.device) y = y.to(dtype=pipe.torch_dtype, device=pipe.device) @@ -730,6 +734,37 @@ class WanVideoUnit_FunReference(PipelineUnit): clip_feature = pipe.image_encoder.encode_image([clip_feature]) return {"reference_latents": reference_latents, "clip_feature": clip_feature} +class WanVideoUnit_FunCamera(PipelineUnit): + def __init__(self): + super().__init__( + input_params=("control_camera_video", "cfg_merge", "num_frames", "height", "width", "input_image", "latents"), + onload_model_names=("vae") + ) + + def process(self, pipe: WanVideoPipeline, control_camera_video, cfg_merge, num_frames, height, width, input_image, latents): + if control_camera_video is None: + return {} + control_camera_video = control_camera_video[:num_frames].permute([3, 0, 1, 2]).unsqueeze(0) + control_camera_latents = torch.concat( + [ + torch.repeat_interleave(control_camera_video[:, :, 0:1], repeats=4, dim=2), + control_camera_video[:, :, 1:] + ], dim=2 + ).transpose(1, 2) + b, f, c, h, w = control_camera_latents.shape + control_camera_latents = control_camera_latents.contiguous().view(b, f // 4, 4, c, h, w).transpose(2, 3) + control_camera_latents = control_camera_latents.contiguous().view(b, f // 4, c * 4, h, w).transpose(1, 2) + control_camera_latents_input = control_camera_latents.to(device=pipe.device, dtype=pipe.torch_dtype) + + input_image = input_image.resize((width, height)) + input_latents = pipe.preprocess_video([input_image]) + input_latents = pipe.vae.encode(input_latents, device=pipe.device) + y = torch.zeros_like(latents).to(pipe.device) + if latents.size()[2] != 1: + y[:, :, :1] = input_latents + y = y.to(dtype=pipe.torch_dtype, device=pipe.device) + + return {"control_camera_latents": control_camera_latents, "control_camera_latents_input": control_camera_latents_input, "y":y} class WanVideoUnit_SpeedControl(PipelineUnit): @@ -954,6 +989,8 @@ def model_fn_wan_video( cfg_merge: bool = False, use_gradient_checkpointing: bool = False, use_gradient_checkpointing_offload: bool = False, + control_camera_latents = None, + control_camera_latents_input = None, **kwargs, ): if sliding_window_size is not None and sliding_window_stride is not None: @@ -1000,13 +1037,14 @@ def model_fn_wan_video( x = torch.concat([x] * context.shape[0], dim=0) if timestep.shape[0] != context.shape[0]: timestep = torch.concat([timestep] * context.shape[0], dim=0) - + if dit.has_image_input: x = torch.cat([x, y], dim=1) # (b, c_x + c_y, f, h, w) clip_embdding = dit.img_emb(clip_feature) context = torch.cat([clip_embdding, context], dim=1) - x, (f, h, w) = dit.patchify(x) + # Add camera control + x, (f, h, w) = dit.patchify(x, control_camera_latents_input) # Reference image if reference_latents is not None: diff --git a/diffsynth/processors/__pycache__/__init__.cpython-310.pyc b/diffsynth/processors/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000..cb65091 Binary files /dev/null and b/diffsynth/processors/__pycache__/__init__.cpython-310.pyc differ diff --git a/diffsynth/processors/__pycache__/base.cpython-310.pyc b/diffsynth/processors/__pycache__/base.cpython-310.pyc new file mode 100644 index 0000000..dc65819 Binary files /dev/null and b/diffsynth/processors/__pycache__/base.cpython-310.pyc differ diff --git a/diffsynth/processors/__pycache__/sequencial_processor.cpython-310.pyc b/diffsynth/processors/__pycache__/sequencial_processor.cpython-310.pyc new file mode 100644 index 0000000..321942d Binary files /dev/null and b/diffsynth/processors/__pycache__/sequencial_processor.cpython-310.pyc differ diff --git a/diffsynth/prompters/__pycache__/__init__.cpython-310.pyc b/diffsynth/prompters/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000..4154d84 Binary files /dev/null and b/diffsynth/prompters/__pycache__/__init__.cpython-310.pyc differ diff --git a/diffsynth/prompters/__pycache__/base_prompter.cpython-310.pyc b/diffsynth/prompters/__pycache__/base_prompter.cpython-310.pyc new file mode 100644 index 0000000..902acfc Binary files /dev/null and b/diffsynth/prompters/__pycache__/base_prompter.cpython-310.pyc differ diff --git a/diffsynth/prompters/__pycache__/cog_prompter.cpython-310.pyc b/diffsynth/prompters/__pycache__/cog_prompter.cpython-310.pyc new file mode 100644 index 0000000..d5f7376 Binary files /dev/null and b/diffsynth/prompters/__pycache__/cog_prompter.cpython-310.pyc differ diff --git a/diffsynth/prompters/__pycache__/flux_prompter.cpython-310.pyc b/diffsynth/prompters/__pycache__/flux_prompter.cpython-310.pyc new file mode 100644 index 0000000..68e9b53 Binary files /dev/null and b/diffsynth/prompters/__pycache__/flux_prompter.cpython-310.pyc differ diff --git a/diffsynth/prompters/__pycache__/hunyuan_dit_prompter.cpython-310.pyc b/diffsynth/prompters/__pycache__/hunyuan_dit_prompter.cpython-310.pyc new file mode 100644 index 0000000..2a72d65 Binary files /dev/null and b/diffsynth/prompters/__pycache__/hunyuan_dit_prompter.cpython-310.pyc differ diff --git a/diffsynth/prompters/__pycache__/hunyuan_video_prompter.cpython-310.pyc b/diffsynth/prompters/__pycache__/hunyuan_video_prompter.cpython-310.pyc new file mode 100644 index 0000000..536be36 Binary files /dev/null and b/diffsynth/prompters/__pycache__/hunyuan_video_prompter.cpython-310.pyc differ diff --git a/diffsynth/prompters/__pycache__/kolors_prompter.cpython-310.pyc b/diffsynth/prompters/__pycache__/kolors_prompter.cpython-310.pyc new file mode 100644 index 0000000..c2efea5 Binary files /dev/null and b/diffsynth/prompters/__pycache__/kolors_prompter.cpython-310.pyc differ diff --git a/diffsynth/prompters/__pycache__/omnigen_prompter.cpython-310.pyc b/diffsynth/prompters/__pycache__/omnigen_prompter.cpython-310.pyc new file mode 100644 index 0000000..32e8e19 Binary files /dev/null and b/diffsynth/prompters/__pycache__/omnigen_prompter.cpython-310.pyc differ diff --git a/diffsynth/prompters/__pycache__/omost.cpython-310.pyc b/diffsynth/prompters/__pycache__/omost.cpython-310.pyc new file mode 100644 index 0000000..51e8053 Binary files /dev/null and b/diffsynth/prompters/__pycache__/omost.cpython-310.pyc differ diff --git a/diffsynth/prompters/__pycache__/prompt_refiners.cpython-310.pyc b/diffsynth/prompters/__pycache__/prompt_refiners.cpython-310.pyc new file mode 100644 index 0000000..bec1243 Binary files /dev/null and b/diffsynth/prompters/__pycache__/prompt_refiners.cpython-310.pyc differ diff --git a/diffsynth/prompters/__pycache__/sd3_prompter.cpython-310.pyc b/diffsynth/prompters/__pycache__/sd3_prompter.cpython-310.pyc new file mode 100644 index 0000000..27bf0f1 Binary files /dev/null and b/diffsynth/prompters/__pycache__/sd3_prompter.cpython-310.pyc differ diff --git a/diffsynth/prompters/__pycache__/sd_prompter.cpython-310.pyc b/diffsynth/prompters/__pycache__/sd_prompter.cpython-310.pyc new file mode 100644 index 0000000..6b0f45f Binary files /dev/null and b/diffsynth/prompters/__pycache__/sd_prompter.cpython-310.pyc differ diff --git a/diffsynth/prompters/__pycache__/sdxl_prompter.cpython-310.pyc b/diffsynth/prompters/__pycache__/sdxl_prompter.cpython-310.pyc new file mode 100644 index 0000000..d2f0752 Binary files /dev/null and b/diffsynth/prompters/__pycache__/sdxl_prompter.cpython-310.pyc differ diff --git a/diffsynth/prompters/__pycache__/stepvideo_prompter.cpython-310.pyc b/diffsynth/prompters/__pycache__/stepvideo_prompter.cpython-310.pyc new file mode 100644 index 0000000..035fc47 Binary files /dev/null and b/diffsynth/prompters/__pycache__/stepvideo_prompter.cpython-310.pyc differ diff --git a/diffsynth/prompters/__pycache__/wan_prompter.cpython-310.pyc b/diffsynth/prompters/__pycache__/wan_prompter.cpython-310.pyc new file mode 100644 index 0000000..d10035c Binary files /dev/null and b/diffsynth/prompters/__pycache__/wan_prompter.cpython-310.pyc differ diff --git a/diffsynth/schedulers/__pycache__/__init__.cpython-310.pyc b/diffsynth/schedulers/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000..0c9872a Binary files /dev/null and b/diffsynth/schedulers/__pycache__/__init__.cpython-310.pyc differ diff --git a/diffsynth/schedulers/__pycache__/continuous_ode.cpython-310.pyc b/diffsynth/schedulers/__pycache__/continuous_ode.cpython-310.pyc new file mode 100644 index 0000000..e21f510 Binary files /dev/null and b/diffsynth/schedulers/__pycache__/continuous_ode.cpython-310.pyc differ diff --git a/diffsynth/schedulers/__pycache__/ddim.cpython-310.pyc b/diffsynth/schedulers/__pycache__/ddim.cpython-310.pyc new file mode 100644 index 0000000..ea9a353 Binary files /dev/null and b/diffsynth/schedulers/__pycache__/ddim.cpython-310.pyc differ diff --git a/diffsynth/schedulers/__pycache__/flow_match.cpython-310.pyc b/diffsynth/schedulers/__pycache__/flow_match.cpython-310.pyc new file mode 100644 index 0000000..884c581 Binary files /dev/null and b/diffsynth/schedulers/__pycache__/flow_match.cpython-310.pyc differ diff --git a/diffsynth/vram_management/__pycache__/__init__.cpython-310.pyc b/diffsynth/vram_management/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000..74c52a7 Binary files /dev/null and b/diffsynth/vram_management/__pycache__/__init__.cpython-310.pyc differ diff --git a/diffsynth/vram_management/__pycache__/layers.cpython-310.pyc b/diffsynth/vram_management/__pycache__/layers.cpython-310.pyc new file mode 100644 index 0000000..62ff84a Binary files /dev/null and b/diffsynth/vram_management/__pycache__/layers.cpython-310.pyc differ diff --git a/examples/wanvideo/model_inference/Wan2.1-Fun-V1.1-1.3B-Control-Camera.py b/examples/wanvideo/model_inference/Wan2.1-Fun-V1.1-1.3B-Control-Camera.py new file mode 100644 index 0000000..d0f5fab --- /dev/null +++ b/examples/wanvideo/model_inference/Wan2.1-Fun-V1.1-1.3B-Control-Camera.py @@ -0,0 +1,48 @@ +import torch +from PIL import Image +from diffsynth import save_video, VideoData +from diffsynth.pipelines.wan_video_new import WanVideoPipeline, ModelConfig +from modelscope import dataset_snapshot_download +from dchen.camera_compute import process_pose_file + +pipe = WanVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="PAI/Wan2.1-Fun-V1.1-1.3B-Control-Camera", origin_file_pattern="diffusion_pytorch_model*.safetensors", offload_device="cpu"), + ModelConfig(model_id="PAI/Wan2.1-Fun-V1.1-1.3B-Control-Camera", origin_file_pattern="models_t5_umt5-xxl-enc-bf16.pth", offload_device="cpu"), + ModelConfig(model_id="PAI/Wan2.1-Fun-V1.1-1.3B-Control-Camera", origin_file_pattern="Wan2.1_VAE.pth", offload_device="cpu"), + ModelConfig(model_id="PAI/Wan2.1-Fun-V1.1-1.3B-Control-Camera", origin_file_pattern="models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth", offload_device="cpu"), + ], +) +pipe.enable_vram_management() + +dataset_snapshot_download( + dataset_id="DiffSynth-Studio/examples_in_diffsynth", + local_dir="./", + allow_file_pattern=["data/examples/wan/control_video.mp4", "data/examples/wan/reference_image_girl.png"] +) + +# Control video +control_video = None +reference_image = None +control_camera_text = "/mnt/nas2/dchen/Work/add_0609/DiffSynth-Studio/dchen/camera_information.txt" +input_image = Image.open("/mnt/nas2/dchen/Work/add_0609/DiffSynth-Studio/dchen/7.png") +sigma_shift = 3 +height = 480 +width = 832 + +control_camera_video = process_pose_file(control_camera_text, width, height) + +video = pipe( + prompt="一个小女孩正在户外玩耍。她穿着一件蓝色的短袖上衣和粉色的短裤，头发扎成一个可爱的辫子。她的脚上没有穿鞋，显得非常自然和随意。她正用一把红色的小铲子在泥土里挖土，似乎在进行某种有趣的活动，可能是种花或是挖掘宝藏。地上有一根长长的水管，可能是用来浇水的。背景是一片草地和一些绿色植物，阳光明媚，整个场景充满了童趣和生机。小女孩专注的表情和认真的动作让人感受到她的快乐和好奇心。", + negative_prompt="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走", + control_video=control_video, reference_image=reference_image, + height=height, width=width, num_frames=81, + seed=1, tiled=True, + + control_camera_video = control_camera_video, + input_image = input_image, + sigma_shift = sigma_shift, +) +save_video(video, "video.mp4", fps=15, quality=5) diff --git a/examples/wanvideo/model_inference/Wan2.1-Fun-V1.1-1.3B-InP.py b/examples/wanvideo/model_inference/Wan2.1-Fun-V1.1-1.3B-InP.py new file mode 100644 index 0000000..f2fc560 --- /dev/null +++ b/examples/wanvideo/model_inference/Wan2.1-Fun-V1.1-1.3B-InP.py @@ -0,0 +1,36 @@ +import torch +from PIL import Image +from diffsynth import save_video, VideoData +from diffsynth.pipelines.wan_video_new import WanVideoPipeline, ModelConfig +from modelscope import dataset_snapshot_download + + +pipe = WanVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="PAI/Wan2.1-Fun-V1.1-1.3B-InP", origin_file_pattern="diffusion_pytorch_model*.safetensors", offload_device="cpu"), + ModelConfig(model_id="PAI/Wan2.1-Fun-V1.1-1.3B-InP", origin_file_pattern="models_t5_umt5-xxl-enc-bf16.pth", offload_device="cpu"), + ModelConfig(model_id="PAI/Wan2.1-Fun-V1.1-1.3B-InP", origin_file_pattern="Wan2.1_VAE.pth", offload_device="cpu"), + ModelConfig(model_id="PAI/Wan2.1-Fun-V1.1-1.3B-InP", origin_file_pattern="models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth", offload_device="cpu"), + ], +) +pipe.enable_vram_management() + +dataset_snapshot_download( + dataset_id="DiffSynth-Studio/examples_in_diffsynth", + local_dir="./", + allow_file_pattern=f"data/examples/wan/input_image.jpg" +) +image = Image.open("data/examples/wan/input_image.jpg") + +# First and last frame to video +video = pipe( + prompt="一艘小船正勇敢地乘风破浪前行。蔚蓝的大海波涛汹涌，白色的浪花拍打着船身，但小船毫不畏惧，坚定地驶向远方。阳光洒在水面上，闪烁着金色的光芒，为这壮丽的场景增添了一抹温暖。镜头拉近，可以看到船上的旗帜迎风飘扬，象征着不屈的精神与冒险的勇气。这段画面充满力量，激励人心，展现了面对挑战时的无畏与执着。", + negative_prompt="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走", + input_image=image, + seed=0, tiled=True + # You can input `end_image=xxx` to control the last frame of the video. + # The model will automatically generate the dynamic content between `input_image` and `end_image`. +) +save_video(video, "video.mp4", fps=15, quality=5) diff --git a/examples/wanvideo/model_inference/Wan2.1-Fun-V1.1-14B-Control-Camera.py b/examples/wanvideo/model_inference/Wan2.1-Fun-V1.1-14B-Control-Camera.py new file mode 100644 index 0000000..1e43c59 --- /dev/null +++ b/examples/wanvideo/model_inference/Wan2.1-Fun-V1.1-14B-Control-Camera.py @@ -0,0 +1,50 @@ +import torch +from PIL import Image +from diffsynth import save_video, VideoData +from diffsynth.pipelines.wan_video_new import WanVideoPipeline, ModelConfig +from modelscope import dataset_snapshot_download +from dchen.camera_compute import process_pose_file + + +pipe = WanVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="PAI/Wan2.1-Fun-V1.1-14B-Control-Camera", origin_file_pattern="diffusion_pytorch_model*.safetensors", offload_device="cpu"), + ModelConfig(model_id="PAI/Wan2.1-Fun-V1.1-14B-Control-Camera", origin_file_pattern="models_t5_umt5-xxl-enc-bf16.pth", offload_device="cpu"), + ModelConfig(model_id="PAI/Wan2.1-Fun-V1.1-14B-Control-Camera", origin_file_pattern="Wan2.1_VAE.pth", offload_device="cpu"), + ModelConfig(model_id="PAI/Wan2.1-Fun-V1.1-14B-Control-Camera", origin_file_pattern="models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth", offload_device="cpu"), + ], +) +pipe.enable_vram_management() +print("success!") + +dataset_snapshot_download( + dataset_id="DiffSynth-Studio/examples_in_diffsynth", + local_dir="./", + allow_file_pattern=["data/examples/wan/control_video.mp4", "data/examples/wan/reference_image_girl.png"] +) + +# Control video +control_video = None +reference_image = None +control_camera_text = "/mnt/nas2/dchen/Work/add_0609/DiffSynth-Studio/dchen/camera_information.txt" +input_image = Image.open("/mnt/nas2/dchen/Work/add_0609/DiffSynth-Studio/dchen/7.png") +sigma_shift = 3 +height = 480 +width = 832 + +control_camera_video = process_pose_file(control_camera_text, width, height) + +video = pipe( + prompt="一个小女孩正在户外玩耍。她穿着一件蓝色的短袖上衣和粉色的短裤，头发扎成一个可爱的辫子。她的脚上没有穿鞋，显得非常自然和随意。她正用一把红色的小铲子在泥土里挖土，似乎在进行某种有趣的活动，可能是种花或是挖掘宝藏。地上有一根长长的水管，可能是用来浇水的。背景是一片草地和一些绿色植物，阳光明媚，整个场景充满了童趣和生机。小女孩专注的表情和认真的动作让人感受到她的快乐和好奇心。", + negative_prompt="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走", + control_video=control_video, reference_image=reference_image, + height=height, width=width, num_frames=81, + seed=1, tiled=True, + + control_camera_video = control_camera_video, + input_image = input_image, + sigma_shift = sigma_shift, +) +save_video(video, "video2.mp4", fps=15, quality=5) \ No newline at end of file diff --git a/examples/wanvideo/model_inference/Wan2.1-Fun-V1.1-14B-InP.py b/examples/wanvideo/model_inference/Wan2.1-Fun-V1.1-14B-InP.py new file mode 100644 index 0000000..334e981 --- /dev/null +++ b/examples/wanvideo/model_inference/Wan2.1-Fun-V1.1-14B-InP.py @@ -0,0 +1,36 @@ +import torch +from PIL import Image +from diffsynth import save_video, VideoData +from diffsynth.pipelines.wan_video_new import WanVideoPipeline, ModelConfig +from modelscope import dataset_snapshot_download + + +pipe = WanVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="PAI/Wan2.1-Fun-V1.1-14B-InP", origin_file_pattern="diffusion_pytorch_model*.safetensors", offload_device="cpu"), + ModelConfig(model_id="PAI/Wan2.1-Fun-V1.1-14B-InP", origin_file_pattern="models_t5_umt5-xxl-enc-bf16.pth", offload_device="cpu"), + ModelConfig(model_id="PAI/Wan2.1-Fun-V1.1-14B-InP", origin_file_pattern="Wan2.1_VAE.pth", offload_device="cpu"), + ModelConfig(model_id="PAI/Wan2.1-Fun-V1.1-14B-InP", origin_file_pattern="models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth", offload_device="cpu"), + ], +) +pipe.enable_vram_management() + +dataset_snapshot_download( + dataset_id="DiffSynth-Studio/examples_in_diffsynth", + local_dir="./", + allow_file_pattern=f"data/examples/wan/input_image.jpg" +) +image = Image.open("data/examples/wan/input_image.jpg") + +# First and last frame to video +video = pipe( + prompt="一艘小船正勇敢地乘风破浪前行。蔚蓝的大海波涛汹涌，白色的浪花拍打着船身，但小船毫不畏惧，坚定地驶向远方。阳光洒在水面上，闪烁着金色的光芒，为这壮丽的场景增添了一抹温暖。镜头拉近，可以看到船上的旗帜迎风飘扬，象征着不屈的精神与冒险的勇气。这段画面充满力量，激励人心，展现了面对挑战时的无畏与执着。", + negative_prompt="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走", + input_image=image, + seed=0, tiled=True + # You can input `end_image=xxx` to control the last frame of the video. + # The model will automatically generate the dynamic content between `input_image` and `end_image`. +) +save_video(video, "video.mp4", fps=15, quality=5) diff --git a/examples/wanvideo/model_training/full/Wan2.1-Fun-V1.1-1.3B-InP.sh b/examples/wanvideo/model_training/full/Wan2.1-Fun-V1.1-1.3B-InP.sh new file mode 100644 index 0000000..d3b280f --- /dev/null +++ b/examples/wanvideo/model_training/full/Wan2.1-Fun-V1.1-1.3B-InP.sh @@ -0,0 +1,14 @@ +accelerate launch examples/wanvideo/model_training/train.py \ + --dataset_base_path data/example_video_dataset \ + --dataset_metadata_path data/example_video_dataset/metadata.csv \ + --height 480 \ + --width 832 \ + --dataset_repeat 100 \ + --model_id_with_origin_paths "PAI/Wan2.1-Fun-V1.1-1.3B-InP:diffusion_pytorch_model*.safetensors,PAI/Wan2.1-Fun-V1.1-1.3B-InP:models_t5_umt5-xxl-enc-bf16.pth,PAI/Wan2.1-Fun-V1.1-1.3B-InP:Wan2.1_VAE.pth,PAI/Wan2.1-Fun-V1.1-1.3B-InP:models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth" \ + --learning_rate 1e-5 \ + --num_epochs 2 \ + --remove_prefix_in_ckpt "pipe.dit." \ + --output_path "./models/train/Wan2.1-Fun-V1.1-1.3B-InP_full" \ + --trainable_models "dit" \ + --input_contains_input_image \ + --input_contains_end_image \ No newline at end of file diff --git a/examples/wanvideo/model_training/full/Wan2.1-Fun-V1.1-14B-InP.sh b/examples/wanvideo/model_training/full/Wan2.1-Fun-V1.1-14B-InP.sh new file mode 100644 index 0000000..11e7cc3 --- /dev/null +++ b/examples/wanvideo/model_training/full/Wan2.1-Fun-V1.1-14B-InP.sh @@ -0,0 +1,14 @@ +accelerate launch --config_file examples/wanvideo/model_training/full/accelerate_config_14B.yaml examples/wanvideo/model_training/train.py \ + --dataset_base_path data/example_video_dataset \ + --dataset_metadata_path data/example_video_dataset/metadata.csv \ + --height 480 \ + --width 832 \ + --dataset_repeat 100 \ + --model_id_with_origin_paths "PAI/Wan2.1-Fun-V1.1-14B-InP:diffusion_pytorch_model*.safetensors,PAI/Wan2.1-Fun-V1.1-14B-InP:models_t5_umt5-xxl-enc-bf16.pth,PAI/Wan2.1-Fun-V1.1-14B-InP:Wan2.1_VAE.pth,PAI/Wan2.1-Fun-V1.1-14B-InP:models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth" \ + --learning_rate 1e-5 \ + --num_epochs 2 \ + --remove_prefix_in_ckpt "pipe.dit." \ + --output_path "./models/train/Wan2.1-Fun-V1.1-14B-InP_full" \ + --trainable_models "dit" \ + --input_contains_input_image \ + --input_contains_end_image \ No newline at end of file diff --git a/examples/wanvideo/model_training/validate_full/Wan2.1-Fun-V1.1-1.3B-InP.py b/examples/wanvideo/model_training/validate_full/Wan2.1-Fun-V1.1-1.3B-InP.py new file mode 100644 index 0000000..cd8ee20 --- /dev/null +++ b/examples/wanvideo/model_training/validate_full/Wan2.1-Fun-V1.1-1.3B-InP.py @@ -0,0 +1,31 @@ +import torch +from PIL import Image +from diffsynth import save_video, VideoData, load_state_dict +from diffsynth.pipelines.wan_video_new import WanVideoPipeline, ModelConfig +from modelscope import dataset_snapshot_download + + +pipe = WanVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="PAI/Wan2.1-Fun-V1.1-1.3B-InP", origin_file_pattern="diffusion_pytorch_model*.safetensors", offload_device="cpu"), + ModelConfig(model_id="PAI/Wan2.1-Fun-V1.1-1.3B-InP", origin_file_pattern="models_t5_umt5-xxl-enc-bf16.pth", offload_device="cpu"), + ModelConfig(model_id="PAI/Wan2.1-Fun-V1.1-1.3B-InP", origin_file_pattern="Wan2.1_VAE.pth", offload_device="cpu"), + ModelConfig(model_id="PAI/Wan2.1-Fun-V1.1-1.3B-InP", origin_file_pattern="models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth", offload_device="cpu"), + ], +) +state_dict = load_state_dict("models/train/Wan2.1-Fun-V1.1-1.3B-InP_full/epoch-1.safetensors") +pipe.dit.load_state_dict(state_dict) +pipe.enable_vram_management() + +video = VideoData("data/example_video_dataset/video1.mp4", height=480, width=832) + +# First and last frame to video +video = pipe( + prompt="from sunset to night, a small town, light, house, river", + negative_prompt="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走", + input_image=video[0], end_image=video[80], + seed=0, tiled=True +) +save_video(video, "video_Wan2.1-Fun-V1.1-1.3B-InP.mp4", fps=15, quality=5) diff --git a/examples/wanvideo/model_training/validate_full/Wan2.1-Fun-V1.1-14B-InP.py b/examples/wanvideo/model_training/validate_full/Wan2.1-Fun-V1.1-14B-InP.py new file mode 100644 index 0000000..7e944b0 --- /dev/null +++ b/examples/wanvideo/model_training/validate_full/Wan2.1-Fun-V1.1-14B-InP.py @@ -0,0 +1,31 @@ +import torch +from PIL import Image +from diffsynth import save_video, VideoData, load_state_dict +from diffsynth.pipelines.wan_video_new import WanVideoPipeline, ModelConfig +from modelscope import dataset_snapshot_download + + +pipe = WanVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="PAI/Wan2.1-Fun-V1.1-14B-InP", origin_file_pattern="diffusion_pytorch_model*.safetensors", offload_device="cpu"), + ModelConfig(model_id="PAI/Wan2.1-Fun-V1.1-14B-InP", origin_file_pattern="models_t5_umt5-xxl-enc-bf16.pth", offload_device="cpu"), + ModelConfig(model_id="PAI/Wan2.1-Fun-V1.1-14B-InP", origin_file_pattern="Wan2.1_VAE.pth", offload_device="cpu"), + ModelConfig(model_id="PAI/Wan2.1-Fun-V1.1-14B-InP", origin_file_pattern="models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth", offload_device="cpu"), + ], +) +state_dict = load_state_dict("models/train/Wan2.1-Fun-V1.1-14B-InP_full/epoch-1.safetensors") +pipe.dit.load_state_dict(state_dict) +pipe.enable_vram_management() + +video = VideoData("data/example_video_dataset/video1.mp4", height=480, width=832) + +# First and last frame to video +video = pipe( + prompt="from sunset to night, a small town, light, house, river", + negative_prompt="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走", + input_image=video[0], end_image=video[80], + seed=0, tiled=True +) +save_video(video, "video_Wan2.1-Fun-V1.1-14B-InP.mp4", fps=15, quality=5)