Upload 29 files

.gitignore

@@ -0,0 +1,192 @@
+# Images
+*jpg
+*png
+*mp4
+.idea/
+# Src
+env_cfg/src
+results/
+checkpoints/*
+temp/*
+
+
+
+# Logging
+*.out
+*.bak
+
+# OS generated files #
+######################
+.DS_Store
+.DS_Store?
+._*
+.Spotlight-V100
+.Trashes
+ehthumbs.db
+Thumbs.db
+
+# Ignore ckpt, pt
+*.pt
+*.ckpt
+*.pth
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 TIGER Lab
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

build_pypi.sh

@@ -0,0 +1,6 @@
+rm -rf dist
+rm -rf build
+python setup.py sdist
+python setup.py bdist_wheel
+twine check dist/*
+twine upload dist/*

conftest.py

@@ -0,0 +1,10 @@
+import os
+import sys
+
+# Calculate the path to the root of the project
+root_path = os.path.dirname(os.path.abspath(__file__))
+
+# Add the project root to the sys.path
+sys.path.append(root_path)
+
+sys.path.append(os.path.join(root_path, "src"))

correct_mp4.py

@@ -0,0 +1,32 @@
+import argparse
+import os
+from moviepy.editor import VideoFileClip
+
+def reprocess_video(input_path, output_path):
+    # Load the video file
+    clip = VideoFileClip(input_path)
+    
+    # Write the clip to a new file with the desired encoding.
+    clip.write_videofile(output_path, codec="libx264", audio_codec="aac")
+
+def find_and_replace_videos(directory):
+    for root, dirs, files in os.walk(directory):
+        for file in files:
+            if file.endswith(".mp4"):
+                full_path = os.path.join(root, file)
+                print(f"Processing {full_path}...")
+                
+                # Define the output path, could overwrite or create a new file
+                output_path = full_path  # This will overwrite the original file
+                
+                # To prevent overwriting, uncomment the following line and comment out the above line
+                # output_path = os.path.splitext(full_path)[0] + "_corrected.mp4"
+                reprocess_video(full_path, output_path)
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Reprocess MP4 files in a folder with correct encoding.")
+    parser.add_argument("directory", help="The directory to search for MP4 files")
+
+    args = parser.parse_args()
+
+    find_and_replace_videos(args.directory)

docs/Contributing/basics.md

@@ -0,0 +1,29 @@
+# The Basics
+We believe that everyone can contribute and make a difference. Whether it's writing code 💻, fixing bugs 🐛, or simply sharing feedback 💬, your contributions are definitely welcome and appreciated 🙌
+
+And if you like the project, but just don't have time to contribute, that's fine. There are other easy ways to support the project and show your appreciation, which we would also be very happy about:
+- Star the project
+- Tweet about it
+- Refer this project in your project's readme
+- Mention the project at local meetups and tell your friends/colleagues
+
+## Fork and Pull Request
+
+When contributing to the codebase of VideoGenHub, the first step is to create a fork of the repository. Then, it’s best practice to create a separate branch for each new pull request (PR) you create. This can be done using:
+
+```shell
+git checkout -b name_of_your_branch
+```
+
+The main branch then simply has the role of being kept up to date with upstream. You can create PRs based on the main branch of your fork, but this will make things more complicated if you would then like to create additional PRs in the future.
+
+
+## Each Pull Request point to an issue
+
+We suggest using one pull-request for each issue. This makes changes clear and simple.
+
+* Link each Pull Request to an issue: `- [ ] #issue_number`
+
+### Commits across minimum files on each Pull Request
+
+Since each Pull Request links to one issue, don’t change too many files at once. Keeping changes small makes it easier for others to check and understand. It also reduces mistakes and mix-ups. So, always keep it simple and focused.

docs/Contributing/coding.md

@@ -0,0 +1,112 @@
+# Code Contribution
+
+* We expect Type Hints in all methods.
+
+You don't need to type hint every variable or function. Start with critical components, or where types might be confusing.
+Type hints are not enforced at runtime. They're purely for the developer and tooling.
+For large projects, consider gradual typing. You can add hints incrementally.
+Type hints, when used judiciously, can make your Python code more readable and maintainable, and can catch potential bugs before runtime.
+
+* We expect Docstrings in all methods and classes.
+
+For modules, the docstring should list the classes, exceptions, and functions (and any other objects) that are exported by the module. For classes, document its methods and instance variables.
+
+## Type Hints
+
+Type hints, introduced in Python 3.5 via PEP 484, provide a way to specify the expected data types for variables, function arguments, and return values. This can greatly improve the readability of your code, facilitate debugging, and enable better tooling (like type checkers and IDE assistance). Here's a guide to get you started with type hints.
+
+### Function and Method Signatures
+
+Type hints can be added to function arguments and return values:
+```python
+def greet(name: str) -> None:
+    print(f"Hello, {name}!")
+
+def add(a: int, b: int) -> int:
+    return a + b
+```
+
+### Type Hinting Lists, Dictionaries, and Other Collections
+
+For collections, you can use the `typing` module:
+
+```python
+from typing import List, Dict
+
+def get_names() -> List[str]:
+    return ["Alice", "Bob", "Charlie"]
+
+def get_age_mapping() -> Dict[str, int]:
+    return {"Alice": 30, "Bob": 25, "Charlie": 28}
+```
+
+### Optional Types
+
+If a variable might be of a certain type or None, use `Optional`:
+
+```python
+from typing import Optional
+
+def find_user(username: str) -> Optional[Dict[str, str]]:
+    # Return user dict if found, else None
+    pass
+```
+
+### Union Types
+
+If a variable can be one of several types, use `Union`:
+
+```python
+from typing import Union
+
+def process_data(data: Union[str, bytes]) -> None:
+    pass
+```
+
+### Classes and Type Hints
+
+You can also use type hints in class definitions:
+
+```python
+class Person:
+    def __init__(self, name: str, age: int) -> None:
+        self.name: str = name
+        self.age: int = age
+```
+
+## Docstrings
+
+Docstrings, or documentation strings, are an essential aspect of Python programming. They provide concise descriptions of how a module, class, method, or function works. The PEP-8 style guide suggests a specific format for these descriptions. We'll walk through the process of adding PEP-8 style docstrings to your Python code. This style is commonly associated with Google's Python style guide.
+
+### What is PEP-8
+
+PEP-8 is the Python Enhancement Proposal that provides coding conventions for the Python code comprising the standard library in the main Python distribution. These conventions help in making the code readable and maintainable.
+
+### Why Docstrings
+
+Docstrings provide a built-in system for associating blocks of documentation with modules, classes, methods, and functions. This documentation can be accessed at runtime using the `help()` function or outside the runtime using tools like Sphinx.
+
+### PEP-8 Docstring Format
+
+Here's the basic structure:
+
+```python
+def function_name(arg1, arg2):
+    """Brief description.
+
+    More detailed description.
+
+    Args:
+        arg1 (type): Description of arg1.
+        arg2 (type): Description of arg2.
+
+    Returns:
+        type: Description of the return value.
+
+    Raises:
+        ExceptionType: Description of the circumstances under which the exception is raised.
+    """
+    pass
+
+```
+

docs/Contributing/docs.md

@@ -0,0 +1,15 @@
+# Improving The Documentation
+
+We are looking for help in improving the documentation.
+
+## Building the Docs locally
+
+The easiest way to build the docs is to use the `docs/make_docs.sh` script.
+
+Make sure that you are in the correct conda environment:
+```shell
+cd docs
+./make_docs.sh
+```
+
+This script will build the docs and store it in `docs/_build`.

docs/Guidelines/custommodel.md

@@ -0,0 +1,75 @@
+# Adding new models
+
+* You developed a new model / framework that perform very good results. Now you want to benchmark it with other models. How you can do it?
+
+In this guide we will be adding new model to the codebase and extend the code.
+
+## Integrating your model into VideoGenHub
+
+
+To add your model codebase into VideoGenHub codebase, you must modify the following folders:
+
+* `src/videogen_hub/infermodels` : where you create a class interface for the model inference.
+* `src/videogen_hub/pipelines` : where you move your codebase into it without much tidy up work.
+
+### How to write the infermodel class
+The infermodel class is designed to have minimal methods. However, it must contain the following methods:
+
+* `__init__(args)` for class initialization.
+* `infer_one_video(args)` to produce 1 video output. Please try to set the seed as 42.
+
+In that case, you will add a new file in `infermodels` folder.
+`infermodels/awesome_model.py`
+```python
+import torch
+from videogen_hub.pipelines.awesome_model import AwesomeModelPipeline
+class AwesomeModelClass():
+    """
+    A wrapper ...
+    """
+    def __init__(self, device="cuda"):
+        """
+        docstring
+        """
+        self.pipe = AwesomeModelPipeline(device=device)
+
+    def infer_one_video(self, prompt, seed=42):
+        """
+        docstring
+        """
+        self.pipe.set_seed(seed)
+        video = self.pipe(prompt=prompt)
+        return video
+```
+Then you can add a line in `infermodels/__init__.py`:
+```shell
+from .awesome_model import AwesomeModelClass
+```
+
+### Writing your pipeline
+About `AwesomeModelPipeline`, it means you need to write a Pipeline file that wraps the function of your codebase, such that the infermodel class can call it with ease.
+
+We recommend structuring code in the `pipelines` folder in this way:
+
+```shell
+└── awesome_model
+    ├── pipeline_awesome_model.py
+    ├── awesome_model_src
+    │   └── ...
+    └── __init__.py
+```
+
+## Running experiment with new model
+After finishing and reinstalling the package through 
+```shell
+pip install -e .
+```
+You should be able to use the new model.
+
+
+### Matching environment
+Make sure the code can be run with the VideoGenHub environment. If new dependency is added, please add them to the env_cfg file.
+
+## Submitting your model as through a PR
+
+Finally, you can submit this new model through submiting a Pull Request! Make sure it match the code style in our contribution guide.

docs/Guidelines/install.md

@@ -0,0 +1,33 @@
+# Installation
+
+## Installing the project
+
+To install from pypi:
+```shall
+pip install videogen-hub
+```
+
+To install from github:
+```shall
+git clone https://github.com/TIGER-AI-Lab/VideoGenHub.git
+cd VideoGenHub
+cd env_cfg
+pip install -r requirements.txt
+cd ..
+pip install -e .
+```
+The requirement of opensora is in `env_cfg/opensora.txt`
+
+For some models like show one, you need to login through `huggingface-cli`.
+
+## Verify the installation
+```python
+import videogen_hub
+
+print(videogen_hub.__version__) # should print a string
+```
+
+## **Downloading weights** into `checkpoints` folder
+```shell
+./download_models.sh
+```

docs/Guidelines/quickstart.md

@@ -0,0 +1,11 @@
+# Quickstart
+
+## Running single model
+```python
+import videogen_hub
+
+model = videogen_hub.load('VideoCrafter2')
+video = model.infer_one_video(prompt="A child excitedly swings on a rusty swing set, laughter filling the air.")
+
+# Here video is a torch tensor of shape torch.Size([16, 3, 320, 512])
+```

docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/Overview/intro.md

@@ -0,0 +1,20 @@
+# Introduction
+
+As a video generation correspondence of ImagenHub, VideoGenHub is a centralized framework to standardize the evaluation of conditional video generation models by curating unified datasets, building an inference library and a benchmark that align with real life applications. This is continuous effort to publish leaderboard to help everyone track the progress in the field.
+
+## Why VideoGenHub?
+
+
+### What sets #VideoGenHub apart?
+1) Unified Datasets: We’ve meticulously curated evaluation datasets for 2 video generation tasks. This ensures comprehensive testing of models across diverse scenarios.
+2) Inference Library: Say goodbye to inconsistent comparisons. Our unified inference pipeline ensures that every model is evaluated on a level playing field with full transparency.
+3) Human-centric Evaluation: Beyond traditional metrics, we’ve innovated with human evaluation scores that measure Semantic Consistency & Perceptual Quality. This aligns evaluations closer to human perceptions, while better than existing Human-preference evaluation methods.
+
+### Why should you use #VideoGenHub?
+1) Streamlined Research: We’ve taken the guesswork out of research by defining clear tasks and providing curated datasets.
+Objective Evaluation: Our framework ensures a bias-free, standardized evaluation, giving a true measure of a model’s capabilities.
+2) Experiment Transparency: By standardizing the human-evaluation dataset, human evaluation results would be come far more convincing with the experiment transparency.
+3) Collaborative Spirit: We believe in the power of community. Our platform is designed to foster collaboration, idea exchange, and innovation in the realm of image generation.
+4) Comprehensive Functionality: From common GenAI metrics to visualization tools, we’ve got you covered. Also stay tuned for our upcoming Amazon Mechanical Turk Templates!
+5) Engineering Excellence: We emphasize good engineering practice. Documentations, type hints, and (coming soon!) extensive code coverage.
+

docs/Overview/models.md

@@ -0,0 +1,20 @@
+# Model Zoo
+
+We included about 10 models in Video Generation.
+Please see https://github.com/TIGER-AI-Lab/VideoGenHub for more information.
+
+##  Text to Video Generation Model
+
+* [LaVie](https://vchitect.github.io/LaVie-project/)
+* [VideoCrafter2](https://ailab-cvc.github.io/videocrafter2/)
+* [ModelScope](https://modelscope.cn/models/iic/text-to-video-synthesis/summary)
+* [StreamingT2V](https://streamingt2v.github.io/)
+* [Show 1](https://showlab.github.io/Show-1/)
+* [OpenSora](https://hpcaitech.github.io/Open-Sora/)
+
+##  Image to Video Generation Model
+
+* [DynamiCrafter2](https://doubiiu.github.io/projects/DynamiCrafter/)
+* [SEINE](https://vchitect.github.io/SEINE-project/)
+* [Consisti2v](https://tiger-ai-lab.github.io/ConsistI2V/)
+* [I2VGenXL](https://i2vgen-xl.github.io/)

docs/Overview/philosophy.rst

@@ -0,0 +1,17 @@
+Philosophy
++++++++++++++++++++++
+
+As the video correspondence of Imagenhub, VideoGenHub plays a pivotal role in propelling the field of Video Generation
+by streamlining research and collaboration.
+
+Purity of Evaluation
+--------------------
+We ensure a fair and consistent evaluation for all models, eliminating biases.
+
+Research Roadmap
+----------------
+By defining tasks and curating datasets, we provide clear direction for researchers.
+
+Open Collaboration
+------------------
+Our platform fosters the exchange and cooperation of related technologies, bringing together minds and innovations.

docs/conf.py

@@ -0,0 +1,57 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+
+# -- Project information -----------------------------------------------------
+
+project = 'VideoGenHub'
+copyright = '2024, TIGER Lab'
+author = 'TIGER Lab'
+
+# -- General configuration ---------------------------------------------------
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    'sphinx.ext.napoleon',
+    'sphinx.ext.duration',
+    'sphinx.ext.doctest',
+    'sphinx.ext.autodoc',
+    'myst_parser'
+]
+
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'sphinx_rtd_theme'
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']

docs/index.rst

@@ -0,0 +1,66 @@
+.. VideoGenHub documentation master file, created by
+   sphinx-quickstart on Wed May 22 16:32:32 2024.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Welcome to VideoGenHub's documentation!
+=======================================
+
+VideoGenHub is a centralized framework to standardize the evaluation of conditional video generation models by curating unified datasets, building an inference library and a benchmark that align with real life applications.
+
+.. _GitHub Code: https://github.com/TIGER-AI-Lab/VideoGenHub
+.. _Pypi Page: https://pypi.org/project/videogen-hub/
+
+* `GitHub Code`_
+* `Pypi Page`_
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Overview
+
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Overview
+
+   Overview/intro
+   Overview/philosophy
+   Overview/models
+   Overview/datasets
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Guidelines
+
+   Guidelines/install
+   Guidelines/quickstart
+   Guidelines/custombenchmark
+   Guidelines/deepdive
+
+.. toctree::
+   :maxdepth: 2
+   :hidden:
+   :caption: Researcher Guidelines
+
+   Guidelines/custommodel
+   Guidelines/humaneval
+   Guidelines/imagenmuseum
+
+.. toctree::
+   :maxdepth: 2
+   :hidden:
+   :caption: Contributing
+
+   Contributing/basics
+   Contributing/coding
+   Contributing/docs
+
+.. toctree::
+   :maxdepth: 2
+   :hidden:
+   :caption: API Reference
+
+   source/modules
+   source/videogen_hub.infermodels
+   source/videogen_hub.benchmark
+   source/videogen_hub.pipelines

docs/make.bat

@@ -0,0 +1,35 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=.
+set BUILDDIR=_build
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.https://www.sphinx-doc.org/
+	exit /b 1
+)
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+
+:end
+popd

docs/make_docs.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+# Get the directory of the current script.
+SCRIPT_DIR=$(dirname "$0")
+
+# Change to the directory where the script is located.
+cd "$SCRIPT_DIR"
+
+cd ..
+## rm -rf docs/source/
+pip install -e .
+## sphinx-apidoc -o docs/source/ src/videogen_hub
+cd docs
+
+make clean
+make html

download_models.sh

@@ -0,0 +1 @@
+python load_models.py

env_cfg/opensora.txt

@@ -0,0 +1,17 @@
+colossalai; platform_system != "Windows"
+accelerate
+diffusers
+ftfy
+gdown
+mmengine
+pandas
+pre-commit
+pyarrow
+pyav
+tensorboard
+timm
+tqdm
+transformers
+wandb
+rotary_embedding_torch
+pandarallel

env_cfg/requirements.txt

@@ -0,0 +1,64 @@
+deepspeed; platform_system != "Windows"
+bitsandbytes
+flask
+flask_restful
+flask_cors
+faiss-cpu
+fire
+h5py
+numpy>=1.23.5
+pandas<2.0.0
+peft
+torch
+torchvision 
+torchaudio
+xformers~=0.0.20
+jupyterlab>=4.0.2
+notebook>=6.5.4
+albumentations>=1.1.0
+opencv-python>=4.2.0
+pudb~=2019.2
+imageio>=2.14.1
+imageio-ffmpeg>=0.4.7
+pytorch-lightning>=1.5.9
+omegaconf~=2.1.1
+gradio==5.0.0
+pillow~=9.5.0
+einops>=0.4.1
+torch-fidelity>=0.3.0
+setuptools>=59.5.0
+transformers==4.37.2
+torchmetrics>=0.6.0
+lpips
+dreamsim
+image-reward
+kornia>=0.6
+diffusers>=0.18.0
+accelerate>=0.20.3
+safetensors
+datasets
+tqdm>=4.64.1
+matplotlib>=3.7.1
+taming-transformers-rom1504~=0.0.6
+madgrad>=1.1
+-e git+https://github.com/openai/CLIP.git@main#egg=clip
+dominate>=2.8.0
+-e git+https://github.com/CompVis/latent-diffusion.git#egg=latent-diffusion #ldm
+jsonargparse
+openai
+nltk~=3.8.1
+krippendorff
+statsmodels
+plotly
+fal_client
+open_clip_torch
+decord
+huggingface_hub
+open-clip-torch-any-py3
+modelscope
+protobuf==3.20.*
+rotary_embedding_torch
+av
+natsort
+tensorboard
+wandb

load_models.py

@@ -0,0 +1,132 @@
+import os
+import sys
+import traceback
+
+import torch
+
+
+def load_all_models():
+    """
+    Download models of Lavie, VideoCrafter2, SEINE, ModelScope, and DynamiCrafter,
+    into the directory defined by MODEL_PATH,
+    with cuda cache emptied.
+    Returns: None
+    """
+    sys.path.insert(0, './src/')
+    #from src.videogen_hub.infermodels import CogVideo
+    from src.videogen_hub.infermodels import ConsistI2V
+    from src.videogen_hub.infermodels import DynamiCrafter
+    from src.videogen_hub.infermodels import I2VGenXL
+    from src.videogen_hub.infermodels import LaVie
+    from src.videogen_hub.infermodels import ModelScope
+    from src.videogen_hub.infermodels import OpenSora
+    from src.videogen_hub.infermodels import OpenSoraPlan
+    from src.videogen_hub.infermodels import SEINE
+    from src.videogen_hub.infermodels import ShowOne
+    from src.videogen_hub.infermodels import StreamingT2V
+    from src.videogen_hub.infermodels import T2VTurbo
+    from src.videogen_hub.infermodels import VideoCrafter2
+
+    from src.videogen_hub import MODEL_PATH
+
+    try:
+        ConsistI2V()
+    except:
+        pass
+    torch.cuda.empty_cache()
+    assert os.path.exists(os.path.join(MODEL_PATH, 'ConsistI2V'))
+    print("ConsistI2V has already been downloaded!")
+
+    try:
+        DynamiCrafter()
+    except:
+        pass
+    torch.cuda.empty_cache()
+    assert os.path.exists(os.path.join(MODEL_PATH, 'dynamicrafter_256_v1'))
+    print("DynamiCrafter has already been downloaded!")
+
+    try:
+        I2VGenXL()
+    except:
+        pass
+    torch.cuda.empty_cache()
+    assert os.path.exists(os.path.join(MODEL_PATH, 'i2vgen-xl'))
+    print("I2VGenXL has already been downloaded!")
+
+    try:
+        LaVie()
+    except:
+        pass
+    torch.cuda.empty_cache()
+    assert os.path.exists(os.path.join(MODEL_PATH, 'lavie'))
+    print("Lavie Model has already been downloaded!")
+
+    try:
+        ModelScope()
+    except:
+        pass
+    torch.cuda.empty_cache()
+    assert os.path.exists(os.path.join(MODEL_PATH, 'modelscope'))
+    print("ModelScope has already been downloaded!")
+
+    try:
+        SEINE()
+    except:
+        pass
+    torch.cuda.empty_cache()
+    assert os.path.exists(os.path.join(MODEL_PATH, 'SEINE'))
+    print("SEINE has already been downloaded!")
+
+    try:
+        ShowOne()
+    except:
+        pass
+    torch.cuda.empty_cache()
+    assert os.path.exists(os.path.join(MODEL_PATH, 'showlab'))
+    print("ShowOne has already been downloaded!")
+
+    try:
+        StreamingT2V()
+    except:
+        pass
+    torch.cuda.empty_cache()
+    assert os.path.exists(os.path.join(MODEL_PATH, 'streamingtv2'))
+    print("StreamingTV has already been downloaded!")
+
+    try:
+        T2VTurbo()
+    except:
+        pass
+    torch.cuda.empty_cache()
+    assert os.path.exists(os.path.join(MODEL_PATH, 'T2V-Turbo-VC2'))
+    print("T2VTurbo has already been downloaded!")
+
+    try:
+        VideoCrafter2()
+    except:
+        pass
+    torch.cuda.empty_cache()
+    assert os.path.exists(os.path.join(MODEL_PATH, 'videocrafter2'))
+    print("VideoCrafter has already been downloaded!")
+
+    # Do these last, as they're linux-only...
+    try:
+        OpenSora()
+    except:
+        pass
+    torch.cuda.empty_cache()
+    assert os.path.exists(os.path.join(MODEL_PATH, 'STDiT2-XL_2'))
+    print("OpenSora has already been downloaded!")
+
+    try:
+        OpenSoraPlan()
+    except:
+        pass
+    torch.cuda.empty_cache()
+    assert os.path.exists(os.path.join(MODEL_PATH, 'Open-Sora-Plan-v1.1.0'))
+    print("OpenSoraPlan has already been downloaded!")
+
+
+
+if __name__ == '__main__':
+    load_all_models()

pyproject.toml

@@ -0,0 +1,2 @@
+[build-system]
+requires = ["setuptools", "wheel"]

setup.py

@@ -0,0 +1,38 @@
+from setuptools import setup, find_packages
+import os
+
+__version__ = None
+
+with open('README.md') as f:
+    readme = f.read()
+
+with open('LICENSE') as f:
+    license = f.read()
+
+with open(os.path.join("src", "videogen_hub", "_version.py")) as f:
+    exec(f.read(), __version__)
+
+setup(
+    name='videogen_hub',
+    version=__version__,
+    packages=find_packages(where='src'),
+    package_dir={'': 'src'},
+    description='VideoGenHub is a one-stop library to standardize the inference and evaluation of all the conditional '
+                'video generation models.',
+    long_description=readme,
+    long_description_content_type='text/markdown',
+    author='Max Ku',
+    author_email='m3ku@uwaterloo.ca',
+    url='https://github.com/TIGER-AI-Lab/VideoGenHub',
+    license=license,
+    classifiers=[
+            "Development Status :: 4 - Beta",
+            "Environment :: GPU :: NVIDIA CUDA",
+            "Intended Audience :: Developers",
+            "Intended Audience :: Education",
+            "Intended Audience :: Science/Research",
+            "Operating System :: OS Independent",
+            "Topic :: Scientific/Engineering :: Artificial Intelligence",
+            "Programming Language :: Python :: 3",
+        ]
+)

t2v_inference.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# Check if model and device parameters are provided
+if [ "$#" -lt 2 ]; then
+    echo "Usage: $0 <model> <device>"
+    exit 1
+fi
+
+# The first command line argument is the model, the second is the device
+model="$1"
+device="$2"
+
+CUDA_VISIBLE_DEVICES="$device" python3 ./src/videogen_hub/benchmark/text_guided_t2v.py --model_name "$model"

tests/cluster_label2.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b87880fdbe89670f12844377b9cf97a9733b1f54e3a9b73cbb9835084c4e02ec
+size 160128

tests/test_i2v.py

@@ -0,0 +1,47 @@
+import os, sys
+import torch
+from diffusers.utils import load_image
+# Directly run `python -m pytest` or
+# Directly run `python -m pytest -v -s --disable-warnings` for Debugging
+
+# To test single function:
+# pytest tests/test_i2v.py::test_function_name
+
+dummy_prompt = "A tiger in a lab coat with a 1980s Miami vibe, turning a well oiled science content machine."
+dummy_image = load_image("https://chromaica.github.io/Museum/ImagenHub_Text-Guided_IG/DALLE3/sample_69.jpg")
+
+import sys
+sys.path.append("src")
+
+def test_SEINE():
+    from videogen_hub.infermodels import SEINE
+
+    model = SEINE()
+    assert model is not None
+    out_video = model.infer_one_video(dummy_image, dummy_prompt)
+    assert out_video is not None
+    # check if out_video is a tensor or not
+    assert isinstance(out_video, torch.Tensor)
+    print(out_video.shape)
+
+def test_ConsistI2V():
+    from videogen_hub.infermodels import ConsistI2V
+
+    model = ConsistI2V()
+    assert model is not None
+    out_video = model.infer_one_video(dummy_image, dummy_prompt)
+    assert out_video is not None
+    # check if out_video is a tensor or not
+    assert isinstance(out_video, torch.Tensor)
+    print(out_video.shape)
+
+def test_DynamiCrafter():
+    from videogen_hub.infermodels import DynamiCrafter
+
+    model = DynamiCrafter()
+    assert model is not None
+    out_video = model.infer_one_video(dummy_image, dummy_prompt)
+    assert out_video is not None
+    # check if out_video is a tensor or not
+    assert isinstance(out_video, torch.Tensor)
+    print(out_video.shape)

tests/test_t2v.py

@@ -0,0 +1,95 @@
+import os, sys
+import torch
+
+# Directly run `python -m pytest` or
+# Directly run `python -m pytest -v -s --disable-warnings` for Debugging
+
+# To test single function:
+# pytest tests/test_t2v.py::test_function_name
+
+dummy_prompts = [
+    "a teddy bear walking on the street, 2k, high quality",
+    "a panda taking a selfie, 2k, high quality",
+    "a polar bear playing drum kit in NYC Times Square, 4k, high resolution",
+    "jungle river at sunset, ultra quality",
+    "a shark swimming in clear Carribean ocean, 2k, high quality",
+    "a Corgi walking in the park at sunrise, oil painting style",
+]
+
+import sys
+sys.path.append("src")
+
+def test_LaVie():
+    from videogen_hub.infermodels import LaVie
+
+    model = LaVie()
+    assert model is not None
+    out_video = model.infer_one_video(dummy_prompts[0])
+    assert out_video is not None
+    # check if out_video is a tensor or not
+    assert isinstance(out_video, torch.Tensor)
+    print(out_video.shape)
+
+
+def test_VideoCrafter2():
+    from videogen_hub.infermodels import VideoCrafter2
+
+    model = VideoCrafter2()
+    assert model is not None
+    out_video = model.infer_one_video(dummy_prompts[0])
+    assert out_video is not None
+    # check if out_video is a tensor or not
+    assert isinstance(out_video, torch.Tensor)
+    print(out_video.shape)
+
+def test_ModelScope():
+    from videogen_hub.infermodels import ModelScope
+    model = ModelScope()
+    assert model is not None
+    out_video = model.infer_one_video(dummy_prompts[0])
+    print("video ouputted")
+    assert out_video is not None
+    # check if out_video is a tensor or not
+    assert isinstance(out_video, torch.Tensor)
+    print(out_video.shape)
+
+def test_StreamingT2V():
+    from videogen_hub.infermodels import StreamingT2V
+
+    model = StreamingT2V()
+    assert model is not None
+    out_video = model.infer_one_video(dummy_prompts[0])
+    print("video ouputted")
+    assert out_video is not None
+    # check if out_video is a tensor or not
+    assert isinstance(out_video, torch.Tensor)
+    print(out_video.shape)
+
+def test_OpenSora():
+    from videogen_hub.infermodels import OpenSora
+
+    model = OpenSora()
+    assert model is not None
+    out_video = model.infer_one_video(dummy_prompts[0])
+    assert out_video is not None
+    # check if out_video is a tensor or not
+    assert isinstance(out_video, torch.Tensor)
+    print(out_video.shape)
+
+
+def test_ShowOne():
+    from videogen_hub.infermodels import ShowOne
+
+    model = ShowOne()
+    assert model is not None
+    out_video = model.infer_one_video(dummy_prompts[0])
+    assert out_video is not None
+    # check if out_video is a tensor or not
+    assert isinstance(out_video, torch.Tensor)
+    print(out_video.shape)
+
+
+if __name__ == "__main__":
+    test_ShowOne()
+    print("Everything passed")
+    pass