From 0479618f5ef019cf75b193aae97ef4a8b63c3689 Mon Sep 17 00:00:00 2001 From: Artiprocher Date: Sat, 3 Feb 2024 17:40:10 +0800 Subject: [PATCH] add a jupyter example --- examples/Diffutoon_toon_shading.ipynb | 282 ++++++++++++++++++++++++++ 1 file changed, 282 insertions(+) create mode 100644 examples/Diffutoon_toon_shading.ipynb diff --git a/examples/Diffutoon_toon_shading.ipynb b/examples/Diffutoon_toon_shading.ipynb new file mode 100644 index 0000000..bfcedbd --- /dev/null +++ b/examples/Diffutoon_toon_shading.ipynb @@ -0,0 +1,282 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "provenance": [], + "gpuType": "T4", + "collapsed_sections": [ + "tII_XRY-PJeo" + ] + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + }, + "language_info": { + "name": "python" + }, + "accelerator": "GPU" + }, + "cells": [ + { + "cell_type": "markdown", + "source": [ + "# DiffSynth Studio\n", + "\n", + "Welcome to DiffSynth Studio! This is an example of Diffutoon." + ], + "metadata": { + "id": "8ObdI5jCB8xy" + } + }, + { + "cell_type": "markdown", + "source": [ + "## Install" + ], + "metadata": { + "id": "XSkKX7O2BwuM" + } + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "msCpt0pLnT8W", + "outputId": "48e084bc-c5ad-4d99-e5d9-8be686a57675" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Cloning into 'DiffSynth-Studio'...\n", + "remote: Enumerating objects: 259, done.\u001b[K\n", + "remote: Counting objects: 100% (259/259), done.\u001b[K\n", + "remote: Compressing objects: 100% (168/168), done.\u001b[K\n", + "remote: Total 259 (delta 128), reused 203 (delta 81), pack-reused 0\u001b[K\n", + "Receiving objects: 100% (259/259), 967.07 KiB | 3.58 MiB/s, done.\n", + "Resolving deltas: 100% (128/128), done.\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m202.4/202.4 kB\u001b[0m \u001b[31m6.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m8.4/8.4 MB\u001b[0m \u001b[31m96.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m59.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m44.6/44.6 kB\u001b[0m \u001b[31m6.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.2/2.2 MB\u001b[0m \u001b[31m15.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m196.4/196.4 kB\u001b[0m \u001b[31m28.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4.8/4.8 MB\u001b[0m \u001b[31m89.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m82.1/82.1 kB\u001b[0m \u001b[31m12.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.7/62.7 kB\u001b[0m \u001b[31m9.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25h Building wheel for controlnet-aux (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + "/content/DiffSynth-Studio\n" + ] + } + ], + "source": [ + "!git clone https://github.com/Artiprocher/DiffSynth-Studio.git\n", + "!pip install -q transformers controlnet-aux==0.0.7 streamlit streamlit-drawable-canvas imageio imageio[ffmpeg] safetensors einops\n", + "%cd /content/DiffSynth-Studio" + ] + }, + { + "cell_type": "markdown", + "source": [ + "## Download Models" + ], + "metadata": { + "id": "5eCu_rlKB3kK" + } + }, + { + "cell_type": "code", + "source": [ + "import requests\n", + "\n", + "\n", + "def download_model(url, file_path):\n", + " model_file = requests.get(url, allow_redirects=True)\n", + " with open(file_path, \"wb\") as f:\n", + " f.write(model_file.content)\n", + "\n", + "download_model(\"https://civitai.com/api/download/models/229575\", \"models/stable_diffusion/aingdiffusion_v12.safetensors\")\n", + "download_model(\"https://huggingface.co/guoyww/animatediff/resolve/main/mm_sd_v15_v2.ckpt\", \"models/AnimateDiff/mm_sd_v15_v2.ckpt\")\n", + "download_model(\"https://huggingface.co/lllyasviel/ControlNet-v1-1/resolve/main/control_v11p_sd15_lineart.pth\", \"models/ControlNet/control_v11p_sd15_lineart.pth\")\n", + "download_model(\"https://huggingface.co/lllyasviel/ControlNet-v1-1/resolve/main/control_v11f1e_sd15_tile.pth\", \"models/ControlNet/control_v11f1e_sd15_tile.pth\")\n", + "download_model(\"https://huggingface.co/lllyasviel/Annotators/resolve/main/sk_model.pth\", \"models/Annotators/sk_model.pth\")\n", + "download_model(\"https://huggingface.co/lllyasviel/Annotators/resolve/main/sk_model2.pth\", \"models/Annotators/sk_model2.pth\")\n", + "download_model(\"https://civitai.com/api/download/models/25820?type=Model&format=PickleTensor&size=full&fp=fp16\", \"models/textual_inversion/verybadimagenegative_v1.3.pt\")" + ], + "metadata": { + "id": "9znMkpVj3qZ1" + }, + "execution_count": 2, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "## Run Diffutoon" + ], + "metadata": { + "id": "iwOq2lWtKVYS" + } + }, + { + "cell_type": "markdown", + "source": [ + "### Config Template" + ], + "metadata": { + "id": "tII_XRY-PJeo" + } + }, + { + "cell_type": "code", + "source": [ + "config_template = {\n", + " \"models\": {\n", + " \"model_list\": [\n", + " \"models/stable_diffusion/aingdiffusion_v12.safetensors\",\n", + " \"models/AnimateDiff/mm_sd_v15_v2.ckpt\",\n", + " \"models/ControlNet/control_v11f1e_sd15_tile.pth\",\n", + " \"models/ControlNet/control_v11p_sd15_lineart.pth\"\n", + " ],\n", + " \"textual_inversion_folder\": \"models/textual_inversion\",\n", + " \"device\": \"cuda\",\n", + " \"lora_alphas\": [],\n", + " \"controlnet_units\": [\n", + " {\n", + " \"processor_id\": \"tile\",\n", + " \"model_path\": \"models/ControlNet/control_v11f1e_sd15_tile.pth\",\n", + " \"scale\": 0.5\n", + " },\n", + " {\n", + " \"processor_id\": \"lineart\",\n", + " \"model_path\": \"models/ControlNet/control_v11p_sd15_lineart.pth\",\n", + " \"scale\": 0.5\n", + " }\n", + " ]\n", + " },\n", + " \"data\": {\n", + " \"input_frames\": {\n", + " \"video_file\": \"/content/video_guide.mp4\",\n", + " \"image_folder\": None,\n", + " \"height\": 1024,\n", + " \"width\": 1024,\n", + " \"start_frame_id\": 0,\n", + " \"end_frame_id\": 30\n", + " },\n", + " \"controlnet_frames\": [\n", + " {\n", + " \"video_file\": \"/content/video_guide.mp4\",\n", + " \"image_folder\": None,\n", + " \"height\": 1024,\n", + " \"width\": 1024,\n", + " \"start_frame_id\": 0,\n", + " \"end_frame_id\": 30\n", + " },\n", + " {\n", + " \"video_file\": \"/content/video_guide.mp4\",\n", + " \"image_folder\": None,\n", + " \"height\": 1024,\n", + " \"width\": 1024,\n", + " \"start_frame_id\": 0,\n", + " \"end_frame_id\": 30\n", + " }\n", + " ],\n", + " \"output_folder\": \"/content/output\",\n", + " \"fps\": 30\n", + " },\n", + " \"pipeline\": {\n", + " \"seed\": 0,\n", + " \"pipeline_inputs\": {\n", + " \"prompt\": \"best quality, perfect anime illustration, light, a girl is dancing, smile, solo\",\n", + " \"negative_prompt\": \"verybadimagenegative_v1.3\",\n", + " \"cfg_scale\": 7.0,\n", + " \"clip_skip\": 2,\n", + " \"denoising_strength\": 1.0,\n", + " \"num_inference_steps\": 10,\n", + " \"animatediff_batch_size\": 16,\n", + " \"animatediff_stride\": 8,\n", + " \"unet_batch_size\": 1,\n", + " \"controlnet_batch_size\": 1,\n", + " \"cross_frame_attention\": False,\n", + " # The following parameters will be overwritten. You don't need to modify them.\n", + " \"input_frames\": [],\n", + " \"num_frames\": 30,\n", + " \"width\": 1536,\n", + " \"height\": 1536,\n", + " \"controlnet_frames\": []\n", + " }\n", + " }\n", + "}" + ], + "metadata": { + "id": "vsd2alA3PrGe" + }, + "execution_count": 3, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "### Run\n", + "\n", + "Before you run the following code, please upload your input video.\n", + "\n", + "We highly recommend you to use a higher resolution for better visual quality. The default resolution of Diffutoon is 1536x1536, which requires 22GB VRAM. If you don't have enough VRAM, 1024x1024 is also acceptable." + ], + "metadata": { + "id": "113QAmNHP6T_" + } + }, + { + "cell_type": "code", + "source": [ + "from diffsynth import SDVideoPipelineRunner\n", + "\n", + "\n", + "config = config_template.copy()\n", + "config[\"data\"][\"input_frames\"] = {\n", + " \"video_file\": \"/content/input_video.mp4\",\n", + " \"image_folder\": None,\n", + " \"height\": 1024,\n", + " \"width\": 1024,\n", + " \"start_frame_id\": 0,\n", + " \"end_frame_id\": 16\n", + "}\n", + "config[\"data\"][\"controlnet_frames\"] = [config[\"data\"][\"input_frames\"], config[\"data\"][\"input_frames\"]]\n", + "config[\"data\"][\"output_folder\"] = \"/content/output\"\n", + "config[\"data\"][\"fps\"] = 30\n", + "\n", + "runner = SDVideoPipelineRunner()\n", + "runner.run(config)" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "761nbrgeKMvj", + "outputId": "aea6f1fe-8485-4eb1-ac23-9c1023b3b9cd" + }, + "execution_count": 6, + "outputs": [ + { + "output_type": "stream", + "name": "stderr", + "text": [ + "100%|██████████| 16/16 [00:00<00:00, 82.74it/s]\n", + "100%|██████████| 16/16 [00:04<00:00, 3.71it/s]\n", + "100%|██████████| 10/10 [05:17<00:00, 31.78s/it]\n", + "Saving images: 100%|██████████| 16/16 [00:06<00:00, 2.38it/s]\n", + "Saving video: 100%|██████████| 16/16 [00:00<00:00, 31.93it/s]\n" + ] + } + ] + } + ] +} \ No newline at end of file