Upload folder using huggingface_hub

AnimatedDiff_Colab.ipynb

@@ -646,7 +646,41 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 12,
+      "execution_count": 15,
+      "metadata": {},
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "Requirement already satisfied: scipy in c:\\users\\texas\\anaconda3\\envs\\animatediff\\lib\\site-packages (1.11.4)\n",
+            "Collecting ffmpeg\n",
+            "  Downloading ffmpeg-1.4.tar.gz (5.1 kB)\n",
+            "  Preparing metadata (setup.py): started\n",
+            "  Preparing metadata (setup.py): finished with status 'done'\n",
+            "Collecting ffmpeg-python\n",
+            "  Downloading ffmpeg_python-0.2.0-py3-none-any.whl (25 kB)\n",
+            "Requirement already satisfied: numpy<1.28.0,>=1.21.6 in c:\\users\\texas\\anaconda3\\envs\\animatediff\\lib\\site-packages (from scipy) (1.26.0)\n",
+            "Collecting future (from ffmpeg-python)\n",
+            "  Using cached future-0.18.3-py3-none-any.whl\n",
+            "Building wheels for collected packages: ffmpeg\n",
+            "  Building wheel for ffmpeg (setup.py): started\n",
+            "  Building wheel for ffmpeg (setup.py): finished with status 'done'\n",
+            "  Created wheel for ffmpeg: filename=ffmpeg-1.4-py3-none-any.whl size=6089 sha256=8fb011d5946d5a48b6a2391b3b8c21b5476fb1298d9d99378a174c78764604be\n",
+            "  Stored in directory: c:\\users\\texas\\appdata\\local\\pip\\cache\\wheels\\8e\\7a\\69\\cd6aeb83b126a7f04cbe7c9d929028dc52a6e7d525ff56003a\n",
+            "Successfully built ffmpeg\n",
+            "Installing collected packages: ffmpeg, future, ffmpeg-python\n",
+            "Successfully installed ffmpeg-1.4 ffmpeg-python-0.2.0 future-0.18.3\n"
+          ]
+        }
+      ],
+      "source": [
+        "!pip install scipy ffmpeg ffmpeg-python"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 17,
       "metadata": {
         "colab": {
           "base_uri": "https://localhost:8080/"
@@ -654,13 +688,88 @@
         "id": "kpUetphRI9D3",
         "outputId": "d0bba9b2-b93c-476a-9909-94e9f7d5239b"
       },
-      "outputs": [],
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "loaded temporal unet's pretrained weights from models/StableDiffusion/unet ...\n",
+            "### missing keys: 560; \n",
+            "### unexpected keys: 0;\n",
+            "### Temporal Module Parameters: 417.1376 M\n",
+            "optimizer values 0.0003 0.9 0.999 0.01 1e-08\n"
+          ]
+        },
+        {
+          "name": "stderr",
+          "output_type": "stream",
+          "text": [
+            "The following values were not passed to `accelerate launch` and had defaults used instead:\n",
+            "\t`--num_processes` was set to a value of `1`\n",
+            "\t`--num_machines` was set to a value of `1`\n",
+            "\t`--mixed_precision` was set to a value of `'no'`\n",
+            "\t`--dynamo_backend` was set to a value of `'no'`\n",
+            "To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`.\n",
+            "11/24/2023 22:12:30 - INFO - __main__ - Distributed environment: NO\n",
+            "Num processes: 1\n",
+            "Process index: 0\n",
+            "Local process index: 0\n",
+            "Device: cuda\n",
+            "\n",
+            "Mixed precision type: fp16\n",
+            "\n",
+            "{'variance_type', 'prediction_type'} was not found in config. Values will be initialized to default values.\n",
+            "{'use_linear_projection', 'use_inflated_groupnorm', 'num_class_embeds', 'upcast_attention', 'resnet_time_scale_shift', 'mid_block_type', 'class_embed_type', 'dual_cross_attention', 'only_cross_attention'} was not found in config. Values will be initialized to default values.\n",
+            "{'prediction_type'} was not found in config. Values will be initialized to default values.\n",
+            "Traceback (most recent call last):\n",
+            "  File \"c:\\Users\\texas\\Desktop\\School\\SceneFusion\\AnimateDiff-SceneFusion\\train.py\", line 457, in <module>\n",
+            "    main(**OmegaConf.load(args.config))\n",
+            "  File \"c:\\Users\\texas\\Desktop\\School\\SceneFusion\\AnimateDiff-SceneFusion\\train.py\", line 257, in main\n",
+            "    unet, optimizer, train_dataloader, lr_scheduler = accelerator.prepare(\n",
+            "  File \"C:\\Users\\texas\\anaconda3\\envs\\animatediff\\lib\\site-packages\\accelerate\\accelerator.py\", line 1288, in prepare\n",
+            "    result = tuple(\n",
+            "  File \"C:\\Users\\texas\\anaconda3\\envs\\animatediff\\lib\\site-packages\\accelerate\\accelerator.py\", line 1289, in <genexpr>\n",
+            "    self._prepare_one(obj, first_pass=True, device_placement=d) for obj, d in zip(args, device_placement)\n",
+            "  File \"C:\\Users\\texas\\anaconda3\\envs\\animatediff\\lib\\site-packages\\accelerate\\accelerator.py\", line 1094, in _prepare_one\n",
+            "    return self.prepare_model(obj, device_placement=device_placement)\n",
+            "  File \"C:\\Users\\texas\\anaconda3\\envs\\animatediff\\lib\\site-packages\\accelerate\\accelerator.py\", line 1387, in prepare_model\n",
+            "    model = model.to(self.device)\n",
+            "  File \"C:\\Users\\texas\\anaconda3\\envs\\animatediff\\lib\\site-packages\\torch\\nn\\modules\\module.py\", line 989, in to\n",
+            "    return self._apply(convert)\n",
+            "  File \"C:\\Users\\texas\\anaconda3\\envs\\animatediff\\lib\\site-packages\\torch\\nn\\modules\\module.py\", line 641, in _apply\n",
+            "    module._apply(fn)\n",
+            "  File \"C:\\Users\\texas\\anaconda3\\envs\\animatediff\\lib\\site-packages\\torch\\nn\\modules\\module.py\", line 641, in _apply\n",
+            "    module._apply(fn)\n",
+            "  File \"C:\\Users\\texas\\anaconda3\\envs\\animatediff\\lib\\site-packages\\torch\\nn\\modules\\module.py\", line 641, in _apply\n",
+            "    module._apply(fn)\n",
+            "  [Previous line repeated 2 more times]\n",
+            "  File \"C:\\Users\\texas\\anaconda3\\envs\\animatediff\\lib\\site-packages\\torch\\nn\\modules\\module.py\", line 664, in _apply\n",
+            "    param_applied = fn(param)\n",
+            "  File \"C:\\Users\\texas\\anaconda3\\envs\\animatediff\\lib\\site-packages\\torch\\nn\\modules\\module.py\", line 987, in convert\n",
+            "    return t.to(device, dtype if t.is_floating_point() or t.is_complex() else None, non_blocking)\n",
+            "torch.cuda.OutOfMemoryError: CUDA out of memory. Tried to allocate 86.00 MiB (GPU 0; 4.00 GiB total capacity; 3.34 GiB already allocated; 0 bytes free; 3.44 GiB reserved in total by PyTorch) If reserved memory is >> allocated memory try setting max_split_size_mb to avoid fragmentation.  See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF\n",
+            "Traceback (most recent call last):\n",
+            "  File \"C:\\Users\\texas\\anaconda3\\envs\\animatediff\\lib\\runpy.py\", line 196, in _run_module_as_main\n",
+            "    return _run_code(code, main_globals, None,\n",
+            "  File \"C:\\Users\\texas\\anaconda3\\envs\\animatediff\\lib\\runpy.py\", line 86, in _run_code\n",
+            "    exec(code, run_globals)\n",
+            "  File \"C:\\Users\\texas\\anaconda3\\envs\\animatediff\\Scripts\\accelerate.exe\\__main__.py\", line 7, in <module>\n",
+            "  File \"C:\\Users\\texas\\anaconda3\\envs\\animatediff\\lib\\site-packages\\accelerate\\commands\\accelerate_cli.py\", line 47, in main\n",
+            "    args.func(args)\n",
+            "  File \"C:\\Users\\texas\\anaconda3\\envs\\animatediff\\lib\\site-packages\\accelerate\\commands\\launch.py\", line 994, in launch_command\n",
+            "    simple_launcher(args)\n",
+            "  File \"C:\\Users\\texas\\anaconda3\\envs\\animatediff\\lib\\site-packages\\accelerate\\commands\\launch.py\", line 636, in simple_launcher\n",
+            "    raise subprocess.CalledProcessError(returncode=process.returncode, cmd=cmd)\n",
+            "subprocess.CalledProcessError: Command '['C:\\\\Users\\\\texas\\\\anaconda3\\\\envs\\\\animatediff\\\\python.exe', 'train.py', '--config=configs/test.yaml']' returned non-zero exit status 1.\n"
+          ]
+        }
+      ],
       "source": [
         "# %cd /content/AnimateDiff-SceneFusion\n",
         "# !python -m torch.distributed.run --nnodes=1 --nproc_per_node=1 train.py --config configs/training/training.yaml\n",
         "\n",
         "# !accelerate launch train.py --config=configs/training/training.yaml\n",
-        "!accelerate launch  train.py --config=configs/{PROJECT}.yaml --python={3.10}"
+        "!accelerate launch  train.py --config=configs/{PROJECT}.yaml"
       ]
     },
     {

handler.py

@@ -10,6 +10,8 @@ import os
 from diffusers.utils.import_utils import is_xformers_available
 from typing import Any
 import torch
+import torchvision
+import numpy as np
 from einops import rearrange
 
 from animatediff.models.unet import UNet3DConditionModel
@@ -52,7 +54,6 @@ class EndpointHandler():
         # huggingface download motion module from bluestarburst/AnimateDiff-SceneFusion/models/Motion_Module/mm_sd_v15.ckpt
 
         motion_module = hf_hub_download(repo_id="bluestarburst/AnimateDiff-SceneFusion", filename="models/Motion_Module/mm_sd_v15.ckpt")
-        
 
         self.pipeline = load_weights(
             self.pipeline,
@@ -65,11 +66,17 @@ class EndpointHandler():
             lora_alpha                 = 0.8,
         ).to("cuda")
     
-    def __call__(self, prompt, negative_prompt, steps, guidance_scale):
+    def __call__(self, data : Any):
         """
         __call__ method will be called once per request. This can be used to
         run inference.
         """
+        
+        prompt = data.pop("prompt", "")
+        negative_prompt = data.pop("negative_prompt", "easynegative,bad_construction,bad_structure,bad_wail,bad_windows,blurry,cloned_window,cropped,deformed,disfigured,error,extra_windows,extra_chimney,extra_door,extra_structure,extra_frame,fewer_digits,fused_structure,gross_proportions,jpeg_artifacts,long_roof,low_quality,structure_limbs,missing_windows,missing_doors,missing_roofs,mutated_structure,mutation,normal_quality,out_of_frame,owres,poorly_drawn_structure,poorly_drawn_house,signature,text,too_many_windows,ugly,username,uta,watermark,worst_quality")
+        steps = data.pop("steps", 25)
+        guidance_scale = data.pop("guidance_scale", 12.5)
+        
         vids = self.pipeline(
             prompt=prompt, 
             negative_prompt=negative_prompt,