change to InstantMesh

README.md

@@ -1,10 +1,10 @@
 ---
-title: ShoeGen + V3D
+title: InstantMesh
 emoji: 🏆
 colorFrom: red
 colorTo: gray
 sdk: gradio
-sdk_version: 4.21.0
+sdk_version: 4.26.0
 app_file: app.py
 pinned: false
 ---

app.py

@@ -1,271 +1,367 @@
-# TODO
+import spaces
+
+import os
+import imageio
 import numpy as np
-import argparse
 import torch
-from torchvision.utils import make_grid
-import tempfile
-import gradio as gr
-from omegaconf import OmegaConf
-from einops import rearrange
-from scripts.pub.V3D_512 import (
-    sample_one,
-    get_batch,
-    get_unique_embedder_keys_from_conditioner,
-    load_model,
-)
-from sgm.util import default, instantiate_from_config
-from safetensors.torch import load_file as load_safetensors
+import rembg
 from PIL import Image
-from kiui.op import recenter
-from torchvision.transforms import ToTensor
+from torchvision.transforms import v2
+from pytorch_lightning import seed_everything
+from omegaconf import OmegaConf
 from einops import rearrange, repeat
-import rembg
-import os
-from glob import glob
-from mediapy import write_video
-from pathlib import Path
-import spaces
+from tqdm import tqdm
+from diffusers import DiffusionPipeline, EulerAncestralDiscreteScheduler
+
+from src.utils.train_util import instantiate_from_config
+from src.utils.camera_util import (
+    FOV_to_intrinsics, 
+    get_zero123plus_input_cameras,
+    get_circular_camera_poses,
+)
+from src.utils.mesh_util import save_obj
+from src.utils.infer_util import remove_background, resize_foreground, images_to_video
+
+import tempfile
+from functools import partial
+
 from huggingface_hub import hf_hub_download
-import imageio
 
-import cv2
+import gradio as gr
 
 
-@spaces.GPU
-def do_sample(
-    image,
-    num_frames,
-    num_steps,
-    decoding_t,
-    border_ratio,
-    ignore_alpha,
-    output_folder,
-    seed,
-):
-    # if image.mode == "RGBA":
-    #     image = image.convert("RGB")
-    torch.manual_seed(seed)
-    image = Image.fromarray(image)
-    w, h = image.size
-
-    if border_ratio > 0:
-        if image.mode != "RGBA" or ignore_alpha:
-            image = image.convert("RGB")
-            image = np.asarray(image)
-            carved_image = rembg.remove(image, session=rembg_session)  # [H, W, 4]
-        else:
-            image = np.asarray(image)
-            carved_image = image
-        mask = carved_image[..., -1] > 0
-        image = recenter(carved_image, mask, border_ratio=border_ratio)
-        image = image.astype(np.float32) / 255.0
-        if image.shape[-1] == 4:
-            image = image[..., :3] * image[..., 3:4] + (1 - image[..., 3:4])
-        image = Image.fromarray((image * 255).astype(np.uint8))
+def get_render_cameras(batch_size=1, M=120, radius=2.5, elevation=10.0, is_flexicubes=False):
+    """
+    Get the rendering camera parameters.
+    """
+    c2ws = get_circular_camera_poses(M=M, radius=radius, elevation=elevation)
+    if is_flexicubes:
+        cameras = torch.linalg.inv(c2ws)
+        cameras = cameras.unsqueeze(0).repeat(batch_size, 1, 1, 1)
     else:
-        print("Ignore border ratio")
-    image = image.resize((512, 512))
-
-    image = ToTensor()(image)
-    image = image * 2.0 - 1.0
-
-    image = image.unsqueeze(0).to(device)
-    H, W = image.shape[2:]
-    assert image.shape[1] == 3
-    F = 8
-    C = 4
-    shape = (num_frames, C, H // F, W // F)
-
-    value_dict = {}
-    value_dict["motion_bucket_id"] = 0
-    value_dict["fps_id"] = 0
-    value_dict["cond_aug"] = 0.05
-    value_dict["cond_frames_without_noise"] = clip_model(image)
-    value_dict["cond_frames"] = ae_model.encode(image)
-    value_dict["cond_frames"] += 0.05 * torch.randn_like(value_dict["cond_frames"])
-    value_dict["cond_aug"] = 0.05
-
-    print(device)
-    with torch.no_grad():
-        with torch.autocast(device_type="cuda"):
-            batch, batch_uc = get_batch(
-                get_unique_embedder_keys_from_conditioner(model.conditioner),
-                value_dict,
-                [1, num_frames],
-                T=num_frames,
-                device=device,
-            )
-            c, uc = model.conditioner.get_unconditional_conditioning(
-                batch,
-                batch_uc=batch_uc,
-                force_uc_zero_embeddings=[
-                    "cond_frames",
-                    "cond_frames_without_noise",
-                ],
-            )
-
-            for k in ["crossattn", "concat"]:
-                uc[k] = repeat(uc[k], "b ... -> b t ...", t=num_frames)
-                uc[k] = rearrange(uc[k], "b t ... -> (b t) ...", t=num_frames)
-                c[k] = repeat(c[k], "b ... -> b t ...", t=num_frames)
-                c[k] = rearrange(c[k], "b t ... -> (b t) ...", t=num_frames)
-
-            randn = torch.randn(shape, device=device)
-            randn = randn.to(device)
-
-            additional_model_inputs = {}
-            additional_model_inputs["image_only_indicator"] = torch.zeros(
-                2, num_frames
-            ).to(device)
-            additional_model_inputs["num_video_frames"] = batch["num_video_frames"]
-
-            def denoiser(input, sigma, c):
-                return model.denoiser(
-                    model.model, input, sigma, c, **additional_model_inputs
-                )
+        extrinsics = c2ws.flatten(-2)
+        intrinsics = FOV_to_intrinsics(50.0).unsqueeze(0).repeat(M, 1, 1).float().flatten(-2)
+        cameras = torch.cat([extrinsics, intrinsics], dim=-1)
+        cameras = cameras.unsqueeze(0).repeat(batch_size, 1, 1)
+    return cameras
 
-            samples_z = model.sampler(denoiser, randn, cond=c, uc=uc)
-            model.en_and_decode_n_samples_a_time = decoding_t
-            samples_x = model.decode_first_stage(samples_z)
-            samples = torch.clamp((samples_x + 1.0) / 2.0, min=0.0, max=1.0)
-
-            os.makedirs(output_folder, exist_ok=True)
-            base_count = len(glob(os.path.join(output_folder, "*.mp4")))
-            video_path = os.path.join(output_folder, f"{base_count:06d}.mp4")
-
-            frames = (
-                (rearrange(samples, "t c h w -> t h w c") * 255)
-                .cpu()
-                .numpy()
-                .astype(np.uint8)
-            )
-            # write_video(video_path, frames, fps=6)
-            # writer = cv2.VideoWriter(
-            #     video_path,
-            #     cv2.VideoWriter_fourcc("m", "p", "4", "v"),
-            #     6,
-            #     (frames.shape[-1], frames.shape[-2]),
-            # )
-            # for fr in frames:
-            #     writer.write(cv2.cvtColor(fr, cv2.COLOR_RGB2BGR))
-            # writer.release()
-            imageio.mimwrite(video_path, frames, fps=6)
-
-    return video_path
-
-
-device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-
-# download
-V3D_ckpt_path = hf_hub_download(repo_id="heheyas/V3D", filename="V3D.ckpt")
-svd_xt_ckpt_path = hf_hub_download(
-    repo_id="stabilityai/stable-video-diffusion-img2vid-xt",
-    filename="svd_xt.safetensors",
-)
 
-model_config = "./scripts/pub/configs/V3D_512.yaml"
-num_frames = OmegaConf.load(
-    model_config
-).model.params.sampler_config.params.guider_config.params.num_frames
-print("Detected num_frames:", num_frames)
-# num_steps = default(num_steps, 25)
-num_steps = 25
-output_folder = "outputs/V3D_512"
-
-sd = load_safetensors(svd_xt_ckpt_path)
-clip_model_config = OmegaConf.load("./configs/embedder/clip_image.yaml")
-clip_model = instantiate_from_config(clip_model_config).eval()
-clip_sd = dict()
-for k, v in sd.items():
-    if "conditioner.embedders.0" in k:
-        clip_sd[k.replace("conditioner.embedders.0.", "")] = v
-clip_model.load_state_dict(clip_sd)
-clip_model = clip_model.to(device)
-
-ae_model_config = OmegaConf.load("./configs/ae/video.yaml")
-ae_model = instantiate_from_config(ae_model_config).eval()
-encoder_sd = dict()
-for k, v in sd.items():
-    if "first_stage_model" in k:
-        encoder_sd[k.replace("first_stage_model.", "")] = v
-ae_model.load_state_dict(encoder_sd)
-ae_model = ae_model.to(device)
-rembg_session = rembg.new_session()
-
-model, _ = load_model(
-    model_config,
-    device,
-    num_frames,
-    num_steps,
-    min_cfg=3.5,
-    max_cfg=3.5,
-    ckpt_path=V3D_ckpt_path,
+def images_to_video(images, output_path, fps=30):
+    # images: (N, C, H, W)
+    os.makedirs(os.path.dirname(output_path), exist_ok=True)
+    frames = []
+    for i in range(images.shape[0]):
+        frame = (images[i].permute(1, 2, 0).cpu().numpy() * 255).astype(np.uint8).clip(0, 255)
+        assert frame.shape[0] == images.shape[2] and frame.shape[1] == images.shape[3], \
+            f"Frame shape mismatch: {frame.shape} vs {images.shape}"
+        assert frame.min() >= 0 and frame.max() <= 255, \
+            f"Frame value out of range: {frame.min()} ~ {frame.max()}"
+        frames.append(frame)
+    imageio.mimwrite(output_path, np.stack(frames), fps=fps, codec='h264')
+
+
+###############################################################################
+# Configuration.
+###############################################################################
+
+import shutil
+
+def find_cuda():
+    # Check if CUDA_HOME or CUDA_PATH environment variables are set
+    cuda_home = os.environ.get('CUDA_HOME') or os.environ.get('CUDA_PATH')
+
+    if cuda_home and os.path.exists(cuda_home):
+        return cuda_home
+
+    # Search for the nvcc executable in the system's PATH
+    nvcc_path = shutil.which('nvcc')
+
+    if nvcc_path:
+        # Remove the 'bin/nvcc' part to get the CUDA installation path
+        cuda_path = os.path.dirname(os.path.dirname(nvcc_path))
+        return cuda_path
+
+    return None
+
+cuda_path = find_cuda()
+
+if cuda_path:
+    print(f"CUDA installation found at: {cuda_path}")
+else:
+    print("CUDA installation not found")
+
+config_path = 'configs/instant-mesh-large.yaml'
+config = OmegaConf.load(config_path)
+config_name = os.path.basename(config_path).replace('.yaml', '')
+model_config = config.model_config
+infer_config = config.infer_config
+
+IS_FLEXICUBES = True if config_name.startswith('instant-mesh') else False
+
+device = torch.device('cuda')
+
+# load diffusion model
+print('Loading diffusion model ...')
+pipeline = DiffusionPipeline.from_pretrained(
+    "sudo-ai/zero123plus-v1.2", 
+    custom_pipeline="zero123plus",
+    torch_dtype=torch.float16,
+)
+pipeline.scheduler = EulerAncestralDiscreteScheduler.from_config(
+    pipeline.scheduler.config, timestep_spacing='trailing'
 )
+
+# load custom white-background UNet
+unet_ckpt_path = hf_hub_download(repo_id="TencentARC/InstantMesh", filename="diffusion_pytorch_model.bin", repo_type="model")
+state_dict = torch.load(unet_ckpt_path, map_location='cpu')
+pipeline.unet.load_state_dict(state_dict, strict=True)
+
+pipeline = pipeline.to(device)
+
+# load reconstruction model
+print('Loading reconstruction model ...')
+model_ckpt_path = hf_hub_download(repo_id="TencentARC/InstantMesh", filename="instant_mesh_large.ckpt", repo_type="model")
+model = instantiate_from_config(model_config)
+state_dict = torch.load(model_ckpt_path, map_location='cpu')['state_dict']
+state_dict = {k[14:]: v for k, v in state_dict.items() if k.startswith('lrm_generator.') and 'source_camera' not in k}
+model.load_state_dict(state_dict, strict=True)
+
 model = model.to(device)
 
-with gr.Blocks(title="V3D", theme=gr.themes.Monochrome()) as demo:
-    with gr.Row(equal_height=True):
+print('Loading Finished!')
+
+
+def check_input_image(input_image):
+    if input_image is None:
+        raise gr.Error("No image uploaded!")
+
+
+def preprocess(input_image, do_remove_background):
+
+    rembg_session = rembg.new_session() if do_remove_background else None
+
+    if do_remove_background:
+        input_image = remove_background(input_image, rembg_session)
+        input_image = resize_foreground(input_image, 0.85)
+
+    return input_image
+
+
+@spaces.GPU
+def generate_mvs(input_image, sample_steps, sample_seed):
+
+    seed_everything(sample_seed)
+    
+    # sampling
+    z123_image = pipeline(
+        input_image, 
+        num_inference_steps=sample_steps
+    ).images[0]
+
+    show_image = np.asarray(z123_image, dtype=np.uint8)
+    show_image = torch.from_numpy(show_image)     # (960, 640, 3)
+    show_image = rearrange(show_image, '(n h) (m w) c -> (n m) h w c', n=3, m=2)
+    show_image = rearrange(show_image, '(n m) h w c -> (n h) (m w) c', n=2, m=3)
+    show_image = Image.fromarray(show_image.numpy())
+
+    return z123_image, show_image
+
+
+@spaces.GPU
+def make3d(images):
+
+    global model
+    if IS_FLEXICUBES:
+        model.init_flexicubes_geometry(device, use_renderer=False)
+    model = model.eval()
+
+    images = np.asarray(images, dtype=np.float32) / 255.0
+    images = torch.from_numpy(images).permute(2, 0, 1).contiguous().float()     # (3, 960, 640)
+    images = rearrange(images, 'c (n h) (m w) -> (n m) c h w', n=3, m=2)        # (6, 3, 320, 320)
+
+    input_cameras = get_zero123plus_input_cameras(batch_size=1, radius=4.0).to(device)
+    render_cameras = get_render_cameras(batch_size=1, radius=2.5, is_flexicubes=IS_FLEXICUBES).to(device)
+
+    images = images.unsqueeze(0).to(device)
+    images = v2.functional.resize(images, (320, 320), interpolation=3, antialias=True).clamp(0, 1)
+
+    mesh_fpath = tempfile.NamedTemporaryFile(suffix=f".obj", delete=False).name
+    print(mesh_fpath)
+    mesh_basename = os.path.basename(mesh_fpath).split('.')[0]
+    mesh_dirname = os.path.dirname(mesh_fpath)
+    video_fpath = os.path.join(mesh_dirname, f"{mesh_basename}.mp4")
+
+    with torch.no_grad():
+        # get triplane
+        planes = model.forward_planes(images, input_cameras)
+
+        # # get video
+        # chunk_size = 20 if IS_FLEXICUBES else 1
+        # render_size = 384
+        
+        # frames = []
+        # for i in tqdm(range(0, render_cameras.shape[1], chunk_size)):
+        #     if IS_FLEXICUBES:
+        #         frame = model.forward_geometry(
+        #             planes,
+        #             render_cameras[:, i:i+chunk_size],
+        #             render_size=render_size,
+        #         )['img']
+        #     else:
+        #         frame = model.synthesizer(
+        #             planes,
+        #             cameras=render_cameras[:, i:i+chunk_size],
+        #             render_size=render_size,
+        #         )['images_rgb']
+        #     frames.append(frame)
+        # frames = torch.cat(frames, dim=1)
+
+        # images_to_video(
+        #     frames[0],
+        #     video_fpath,
+        #     fps=30,
+        # )
+
+        # print(f"Video saved to {video_fpath}")
+
+        # get mesh
+        mesh_out = model.extract_mesh(
+            planes,
+            use_texture_map=False,
+            **infer_config,
+        )
+
+        vertices, faces, vertex_colors = mesh_out
+        vertices = vertices[:, [1, 2, 0]]
+        vertices[:, -1] *= -1
+        faces = faces[:, [2, 1, 0]]
+
+        save_obj(vertices, faces, vertex_colors, mesh_fpath)
+        
+        print(f"Mesh saved to {mesh_fpath}")
+
+    return mesh_fpath
+
+
+_HEADER_ = '''
+<h2><b>Official 🤗 Gradio Demo</b></h2><h2><a href='https://github.com/TencentARC/InstantMesh' target='_blank'><b>InstantMesh: Efficient 3D Mesh Generation from a Single Image with Sparse-view Large Reconstruction Models</b></a></h2>
+'''
+
+_LINKS_ = '''
+<h3>Code is available at <a href='https://github.com/TencentARC/InstantMesh' target='_blank'>GitHub</a></h3>
+<h3>Report is available at <a href='https://arxiv.org/abs/2404.07191' target='_blank'>ArXiv</a></h3>
+'''
+
+_CITE_ = r"""
+```bibtex
+@article{xu2024instantmesh,
+  title={InstantMesh: Efficient 3D Mesh Generation from a Single Image with Sparse-view Large Reconstruction Models},
+  author={Xu, Jiale and Cheng, Weihao and Gao, Yiming and Wang, Xintao and Gao, Shenghua and Shan, Ying},
+  journal={arXiv preprint arXiv:2404.07191},
+  year={2024}
+}
+```
+"""
+
+
+with gr.Blocks() as demo:
+    gr.Markdown(_HEADER_)
+    with gr.Row(variant="panel"):
         with gr.Column():
-            input_image = gr.Image(value=None, label="Input Image")
-
-            border_ratio_slider = gr.Slider(
-                value=0.3,
-                label="Border Ratio",
-                minimum=0.05,
-                maximum=0.5,
-                step=0.05,
-            )
-            seed_input = gr.Number(value=42)
-            decoding_t_slider = gr.Slider(
-                value=1,
-                label="Number of Decoding frames",
-                minimum=1,
-                maximum=num_frames,
-                step=1,
-            )
-            min_guidance_slider = gr.Slider(
-                value=3.5,
-                label="Min CFG Value",
-                minimum=0.05,
-                maximum=5,
-                step=0.05,
-            )
-            max_guidance_slider = gr.Slider(
-                value=3.5,
-                label="Max CFG Value",
-                minimum=0.05,
-                maximum=5,
-                step=0.05,
-            )
-            run_button = gr.Button(value="Run V3D")
+            with gr.Row():
+                input_image = gr.Image(
+                    label="Input Image",
+                    image_mode="RGBA",
+                    sources="upload",
+                    #width=256,
+                    #height=256,
+                    type="pil",
+                    elem_id="content_image",
+                )
+                processed_image = gr.Image(
+                    label="Processed Image", 
+                    image_mode="RGBA", 
+                    #width=256,
+                    #height=256,
+                    type="pil", 
+                    interactive=False
+                )
+            with gr.Row():
+                with gr.Group():
+                    do_remove_background = gr.Checkbox(
+                        label="Remove Background", value=True
+                    )
+                    sample_seed = gr.Number(value=42, label="Seed Value", precision=0)
+
+                    sample_steps = gr.Slider(
+                        label="Sample Steps",
+                        minimum=30,
+                        maximum=75,
+                        value=75,
+                        step=5
+                    )
+
+            with gr.Row():
+                submit = gr.Button("Generate", elem_id="generate", variant="primary")
+
+            with gr.Row(variant="panel"):
+                gr.Examples(
+                    examples=[
+                        os.path.join("examples", img_name) for img_name in sorted(os.listdir("examples"))
+                    ],
+                    inputs=[input_image],
+                    label="Examples",
+                    cache_examples=False,
+                    examples_per_page=12
+                )
 
         with gr.Column():
-            output_video = gr.Video(value=None, label="Output Orbit Video")
-
-    @run_button.click(
-        inputs=[
-            input_image,
-            border_ratio_slider,
-            min_guidance_slider,
-            max_guidance_slider,
-            decoding_t_slider,
-            seed_input,
-        ],
-        outputs=[output_video],
-    )
-    def _(image, border_ratio, min_guidance, max_guidance, decoding_t, seed):
-        model.sampler.guider.max_scale = max_guidance
-        model.sampler.guider.min_scale = min_guidance
-        return do_sample(
-            image,
-            num_frames,
-            num_steps,
-            int(decoding_t),
-            border_ratio,
-            False,
-            output_folder,
-            seed,
-        )
 
+            with gr.Row():
+
+                with gr.Column():
+                    mv_show_images = gr.Image(
+                        label="Generated Multi-views",
+                        type="pil",
+                        width=379,
+                        interactive=False
+                    )
+
+                # with gr.Column():
+                #     output_video = gr.Video(
+                #         label="video", format="mp4",
+                #         width=379,
+                #         autoplay=True,
+                #         interactive=False
+                #     )
+
+            with gr.Row():
+                output_model_obj = gr.Model3D(
+                    label="Output Model (OBJ Format)",
+                    interactive=False,
+                )
+
+            with gr.Row():
+                gr.Markdown('''Try a different <b>seed value</b> if the result is unsatisfying (Default: 42).''')
+
+    gr.Markdown(_LINKS_)
+    gr.Markdown(_CITE_)
+
+    mv_images = gr.State()
+
+    submit.click(fn=check_input_image, inputs=[input_image]).success(
+        fn=preprocess,
+        inputs=[input_image, do_remove_background],
+        outputs=[processed_image],
+    ).success(
+        fn=generate_mvs,
+        inputs=[processed_image, sample_steps, sample_seed],
+        outputs=[mv_images, mv_show_images]
+        
+    ).success(
+        fn=make3d,
+        inputs=[mv_images],
+        outputs=[output_model_obj]
+    )
 
-demo.launch()
+demo.launch()

ckpts/shoes.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:e66a57b2174aff462c3bc0c9f9e3b1142617d856a1f5ddbada3b696dcc057b73
-size 170543188

ckpts/snckrsgen.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:e80bf5f4ded84793d74c9939b0fc1a09b76af31bafe2ac3190c21c9be5eb6965
-size 151112168

configs/ae/video.yaml

@@ -1,35 +0,0 @@
-target: sgm.models.autoencoder.AutoencodingEngine
-params:
-  loss_config:
-    target: torch.nn.Identity
-  regularizer_config:
-    target: sgm.modules.autoencoding.regularizers.DiagonalGaussianRegularizer
-  encoder_config: 
-    target: sgm.modules.diffusionmodules.model.Encoder
-    params:
-      attn_type: vanilla
-      double_z: True
-      z_channels: 4
-      resolution: 256
-      in_channels: 3
-      out_ch: 3
-      ch: 128
-      ch_mult: [1, 2, 4, 4]
-      num_res_blocks: 2
-      attn_resolutions: []
-      dropout: 0.0
-  decoder_config:
-    target: sgm.modules.autoencoding.temporal_ae.VideoDecoder
-    params:
-      attn_type: vanilla
-      double_z: True
-      z_channels: 4
-      resolution: 256
-      in_channels: 3
-      out_ch: 3
-      ch: 128
-      ch_mult: [1, 2, 4, 4]
-      num_res_blocks: 2
-      attn_resolutions: []
-      dropout: 0.0
-      video_kernel_size: [3, 1, 1]

configs/embedder/clip_image.yaml

@@ -1,8 +0,0 @@
-target: sgm.modules.encoders.modules.FrozenOpenCLIPImagePredictionEmbedder
-params:
-  n_cond_frames: 1
-  n_copies: 1
-  open_clip_embedding_config:
-    target: sgm.modules.encoders.modules.FrozenOpenCLIPImageEmbedder
-    params:
-      freeze: True

configs/example_training/autoencoder/kl-f4/imagenet-attnfree-logvar.yaml

@@ -1,104 +0,0 @@
-model:
-  base_learning_rate: 4.5e-6
-  target: sgm.models.autoencoder.AutoencodingEngine
-  params:
-    input_key: jpg
-    monitor: val/rec_loss
-
-    loss_config:
-      target: sgm.modules.autoencoding.losses.GeneralLPIPSWithDiscriminator
-      params:
-        perceptual_weight: 0.25
-        disc_start: 20001
-        disc_weight: 0.5
-        learn_logvar: True
-
-        regularization_weights:
-          kl_loss: 1.0
-
-    regularizer_config:
-      target: sgm.modules.autoencoding.regularizers.DiagonalGaussianRegularizer
-
-    encoder_config:
-      target: sgm.modules.diffusionmodules.model.Encoder
-      params:
-        attn_type: none
-        double_z: True
-        z_channels: 4
-        resolution: 256
-        in_channels: 3
-        out_ch: 3
-        ch: 128
-        ch_mult: [1, 2, 4]
-        num_res_blocks: 4
-        attn_resolutions: []
-        dropout: 0.0
-
-    decoder_config:
-      target: sgm.modules.diffusionmodules.model.Decoder
-      params: ${model.params.encoder_config.params}
-
-data:
-  target: sgm.data.dataset.StableDataModuleFromConfig
-  params:
-    train:
-      datapipeline:
-        urls:
-          - DATA-PATH
-        pipeline_config:
-          shardshuffle: 10000
-          sample_shuffle: 10000
-
-        decoders:
-          - pil
-
-        postprocessors:
-          - target: sdata.mappers.TorchVisionImageTransforms
-            params:
-              key: jpg
-              transforms:
-                - target: torchvision.transforms.Resize
-                  params:
-                    size: 256
-                    interpolation: 3
-                - target: torchvision.transforms.ToTensor
-          - target: sdata.mappers.Rescaler
-          - target: sdata.mappers.AddOriginalImageSizeAsTupleAndCropToSquare
-            params:
-              h_key: height
-              w_key: width
-
-      loader:
-        batch_size: 8
-        num_workers: 4
-
-
-lightning:
-  strategy:
-    target: pytorch_lightning.strategies.DDPStrategy
-    params:
-      find_unused_parameters: True
-
-  modelcheckpoint:
-    params:
-      every_n_train_steps: 5000
-
-  callbacks:
-    metrics_over_trainsteps_checkpoint:
-      params:
-        every_n_train_steps: 50000
-
-    image_logger:
-      target: main.ImageLogger
-      params:
-        enable_autocast: False
-        batch_frequency: 1000
-        max_images: 8
-        increase_log_steps: True
-
-  trainer:
-    devices: 0,
-    limit_val_batches: 50
-    benchmark: True
-    accumulate_grad_batches: 1
-    val_check_interval: 10000

configs/example_training/autoencoder/kl-f4/imagenet-kl_f8_8chn.yaml

@@ -1,105 +0,0 @@
-model:
-  base_learning_rate: 4.5e-6
-  target: sgm.models.autoencoder.AutoencodingEngine
-  params:
-    input_key: jpg
-    monitor: val/loss/rec
-    disc_start_iter: 0
-
-    encoder_config:
-      target: sgm.modules.diffusionmodules.model.Encoder
-      params:
-        attn_type: vanilla-xformers
-        double_z: true
-        z_channels: 8
-        resolution: 256
-        in_channels: 3
-        out_ch: 3
-        ch: 128
-        ch_mult: [1, 2, 4, 4]
-        num_res_blocks: 2
-        attn_resolutions: []
-        dropout: 0.0
-
-    decoder_config:
-      target: sgm.modules.diffusionmodules.model.Decoder
-      params: ${model.params.encoder_config.params}
-
-    regularizer_config:
-      target: sgm.modules.autoencoding.regularizers.DiagonalGaussianRegularizer
-
-    loss_config:
-      target: sgm.modules.autoencoding.losses.GeneralLPIPSWithDiscriminator
-      params:
-        perceptual_weight: 0.25
-        disc_start: 20001
-        disc_weight: 0.5
-        learn_logvar: True
-
-        regularization_weights:
-          kl_loss: 1.0
-
-data:
-  target: sgm.data.dataset.StableDataModuleFromConfig
-  params:
-    train:
-      datapipeline:
-        urls:
-          - DATA-PATH
-        pipeline_config:
-          shardshuffle: 10000
-          sample_shuffle: 10000
-
-        decoders:
-          - pil
-
-        postprocessors:
-          - target: sdata.mappers.TorchVisionImageTransforms
-            params:
-              key: jpg
-              transforms:
-                - target: torchvision.transforms.Resize
-                  params:
-                    size: 256
-                    interpolation: 3
-                - target: torchvision.transforms.ToTensor
-          - target: sdata.mappers.Rescaler
-          - target: sdata.mappers.AddOriginalImageSizeAsTupleAndCropToSquare
-            params:
-              h_key: height
-              w_key: width
-
-      loader:
-        batch_size: 8
-        num_workers: 4
-
-
-lightning:
-  strategy:
-    target: pytorch_lightning.strategies.DDPStrategy
-    params:
-      find_unused_parameters: True
-
-  modelcheckpoint:
-    params:
-      every_n_train_steps: 5000
-
-  callbacks:
-    metrics_over_trainsteps_checkpoint:
-      params:
-        every_n_train_steps: 50000
-
-    image_logger:
-      target: main.ImageLogger
-      params:
-        enable_autocast: False
-        batch_frequency: 1000
-        max_images: 8
-        increase_log_steps: True
-
-  trainer:
-    devices: 0,
-    limit_val_batches: 50
-    benchmark: True
-    accumulate_grad_batches: 1
-    val_check_interval: 10000

configs/example_training/imagenet-f8_cond.yaml

@@ -1,185 +0,0 @@
-model:
-  base_learning_rate: 1.0e-4
-  target: sgm.models.diffusion.DiffusionEngine
-  params:
-    scale_factor: 0.13025
-    disable_first_stage_autocast: True
-    log_keys:
-      - cls
-
-    scheduler_config:
-      target: sgm.lr_scheduler.LambdaLinearScheduler
-      params:
-        warm_up_steps: [10000]
-        cycle_lengths: [10000000000000]
-        f_start: [1.e-6]
-        f_max: [1.]
-        f_min: [1.]
-
-    denoiser_config:
-      target: sgm.modules.diffusionmodules.denoiser.DiscreteDenoiser
-      params:
-        num_idx: 1000
-
-        scaling_config:
-          target: sgm.modules.diffusionmodules.denoiser_scaling.EpsScaling
-        discretization_config:
-          target: sgm.modules.diffusionmodules.discretizer.LegacyDDPMDiscretization
-
-    network_config:
-      target: sgm.modules.diffusionmodules.openaimodel.UNetModel
-      params:
-        use_checkpoint: True
-        in_channels: 4
-        out_channels: 4
-        model_channels: 256
-        attention_resolutions: [1, 2, 4]
-        num_res_blocks: 2
-        channel_mult: [1, 2, 4]
-        num_head_channels: 64
-        num_classes: sequential
-        adm_in_channels: 1024
-        transformer_depth: 1
-        context_dim: 1024
-        spatial_transformer_attn_type: softmax-xformers
-
-    conditioner_config:
-      target: sgm.modules.GeneralConditioner
-      params:
-        emb_models:
-          - is_trainable: True
-            input_key: cls
-            ucg_rate: 0.2
-            target: sgm.modules.encoders.modules.ClassEmbedder
-            params:
-              add_sequence_dim: True
-              embed_dim: 1024
-              n_classes: 1000
-
-          - is_trainable: False
-            ucg_rate: 0.2
-            input_key: original_size_as_tuple
-            target: sgm.modules.encoders.modules.ConcatTimestepEmbedderND
-            params:
-              outdim: 256
-
-          - is_trainable: False
-            input_key: crop_coords_top_left
-            ucg_rate: 0.2
-            target: sgm.modules.encoders.modules.ConcatTimestepEmbedderND
-            params:
-              outdim: 256
-
-    first_stage_config:
-      target: sgm.models.autoencoder.AutoencoderKL
-      params:
-        ckpt_path: CKPT_PATH
-        embed_dim: 4
-        monitor: val/rec_loss
-        ddconfig:
-          attn_type: vanilla-xformers
-          double_z: true
-          z_channels: 4
-          resolution: 256
-          in_channels: 3
-          out_ch: 3
-          ch: 128
-          ch_mult: [1, 2, 4, 4]
-          num_res_blocks: 2
-          attn_resolutions: []
-          dropout: 0.0
-        lossconfig:
-          target: torch.nn.Identity
-
-    loss_fn_config:
-      target: sgm.modules.diffusionmodules.loss.StandardDiffusionLoss
-      params:        
-        loss_weighting_config:
-          target: sgm.modules.diffusionmodules.loss_weighting.EpsWeighting
-        sigma_sampler_config:
-          target: sgm.modules.diffusionmodules.sigma_sampling.DiscreteSampling
-          params:
-            num_idx: 1000
-
-            discretization_config:
-              target: sgm.modules.diffusionmodules.discretizer.LegacyDDPMDiscretization
-
-    sampler_config:
-      target: sgm.modules.diffusionmodules.sampling.EulerEDMSampler
-      params:
-        num_steps: 50
-
-        discretization_config:
-          target: sgm.modules.diffusionmodules.discretizer.LegacyDDPMDiscretization
-
-        guider_config:
-          target: sgm.modules.diffusionmodules.guiders.VanillaCFG
-          params:
-            scale: 5.0
-
-data:
-  target: sgm.data.dataset.StableDataModuleFromConfig
-  params:
-    train:
-      datapipeline:
-        urls:
-          # USER: adapt this path the root of your custom dataset
-          - DATA_PATH
-        pipeline_config:
-          shardshuffle: 10000
-          sample_shuffle: 10000 # USER: you might wanna adapt depending on your available RAM
-
-        decoders:
-          - pil
-
-        postprocessors:
-          - target: sdata.mappers.TorchVisionImageTransforms
-            params:
-              key: jpg # USER: you might wanna adapt this for your custom dataset
-              transforms:
-                - target: torchvision.transforms.Resize
-                  params:
-                    size: 256
-                    interpolation: 3
-                - target: torchvision.transforms.ToTensor
-          - target: sdata.mappers.Rescaler
-
-          - target: sdata.mappers.AddOriginalImageSizeAsTupleAndCropToSquare
-            params:
-              h_key: height # USER: you might wanna adapt this for your custom dataset
-              w_key: width # USER: you might wanna adapt this for your custom dataset
-
-      loader:
-        batch_size: 64
-        num_workers: 6
-
-lightning:
-  modelcheckpoint:
-    params:
-      every_n_train_steps: 5000
-
-  callbacks:
-    metrics_over_trainsteps_checkpoint:
-      params:
-        every_n_train_steps: 25000
-
-    image_logger:
-      target: main.ImageLogger
-      params:
-        disabled: False
-        enable_autocast: False
-        batch_frequency: 1000
-        max_images: 8
-        increase_log_steps: True
-        log_first_step: False
-        log_images_kwargs:
-          use_ema_scope: False
-          N: 8
-          n_rows: 2
-
-  trainer:
-    devices: 0,
-    benchmark: True
-    num_sanity_val_steps: 0
-    accumulate_grad_batches: 1
-    max_epochs: 1000

configs/example_training/toy/cifar10_cond.yaml

@@ -1,98 +0,0 @@
-model:
-  base_learning_rate: 1.0e-4
-  target: sgm.models.diffusion.DiffusionEngine
-  params:
-    denoiser_config:
-      target: sgm.modules.diffusionmodules.denoiser.Denoiser
-      params:
-        scaling_config:
-          target: sgm.modules.diffusionmodules.denoiser_scaling.EDMScaling
-          params:
-            sigma_data: 1.0
-
-    network_config:
-      target: sgm.modules.diffusionmodules.openaimodel.UNetModel
-      params:
-        in_channels: 3
-        out_channels: 3
-        model_channels: 32
-        attention_resolutions: []
-        num_res_blocks: 4
-        channel_mult: [1, 2, 2]
-        num_head_channels: 32
-        num_classes: sequential
-        adm_in_channels: 128
-
-    conditioner_config:
-      target: sgm.modules.GeneralConditioner
-      params:
-        emb_models:
-          - is_trainable: True
-            input_key: cls
-            ucg_rate: 0.2
-            target: sgm.modules.encoders.modules.ClassEmbedder
-            params:
-              embed_dim: 128
-              n_classes: 10
-
-    first_stage_config:
-      target: sgm.models.autoencoder.IdentityFirstStage
-
-    loss_fn_config:
-      target: sgm.modules.diffusionmodules.loss.StandardDiffusionLoss
-      params:
-        loss_weighting_config:
-          target: sgm.modules.diffusionmodules.loss_weighting.EDMWeighting
-          params:
-            sigma_data: 1.0
-        sigma_sampler_config:
-          target: sgm.modules.diffusionmodules.sigma_sampling.EDMSampling
-
-    sampler_config:
-      target: sgm.modules.diffusionmodules.sampling.EulerEDMSampler
-      params:
-        num_steps: 50
-
-        discretization_config:
-          target: sgm.modules.diffusionmodules.discretizer.EDMDiscretization
-
-        guider_config:
-          target: sgm.modules.diffusionmodules.guiders.VanillaCFG
-          params:
-            scale: 3.0
-
-data:
-  target: sgm.data.cifar10.CIFAR10Loader
-  params:
-    batch_size: 512
-    num_workers: 1
-
-lightning:
-  modelcheckpoint:
-    params:
-      every_n_train_steps: 5000
-
-  callbacks:
-    metrics_over_trainsteps_checkpoint:
-      params:
-        every_n_train_steps: 25000
-
-    image_logger:
-      target: main.ImageLogger
-      params:
-        disabled: False
-        batch_frequency: 1000
-        max_images: 64
-        increase_log_steps: True
-        log_first_step: False
-        log_images_kwargs:
-          use_ema_scope: False
-          N: 64
-          n_rows: 8
-
-  trainer:
-    devices: 0,
-    benchmark: True
-    num_sanity_val_steps: 0
-    accumulate_grad_batches: 1
-    max_epochs: 20

configs/example_training/toy/mnist.yaml

@@ -1,79 +0,0 @@
-model:
-  base_learning_rate: 1.0e-4
-  target: sgm.models.diffusion.DiffusionEngine
-  params:
-    denoiser_config:
-      target: sgm.modules.diffusionmodules.denoiser.Denoiser
-      params:
-        scaling_config:
-          target: sgm.modules.diffusionmodules.denoiser_scaling.EDMScaling
-          params:
-            sigma_data: 1.0
-
-    network_config:
-      target: sgm.modules.diffusionmodules.openaimodel.UNetModel
-      params:
-        in_channels: 1
-        out_channels: 1
-        model_channels: 32
-        attention_resolutions: []
-        num_res_blocks: 4
-        channel_mult: [1, 2, 2]
-        num_head_channels: 32
-
-    first_stage_config:
-      target: sgm.models.autoencoder.IdentityFirstStage
-
-    loss_fn_config:
-      target: sgm.modules.diffusionmodules.loss.StandardDiffusionLoss
-      params:
-        loss_weighting_config:
-          target: sgm.modules.diffusionmodules.loss_weighting.EDMWeighting
-          params:
-            sigma_data: 1.0
-        sigma_sampler_config:
-          target: sgm.modules.diffusionmodules.sigma_sampling.EDMSampling
-
-    sampler_config:
-      target: sgm.modules.diffusionmodules.sampling.EulerEDMSampler
-      params:
-        num_steps: 50
-
-        discretization_config:
-          target: sgm.modules.diffusionmodules.discretizer.EDMDiscretization
-
-data:
-  target: sgm.data.mnist.MNISTLoader
-  params:
-    batch_size: 512
-    num_workers: 1
-
-lightning:
-  modelcheckpoint:
-    params:
-      every_n_train_steps: 5000
-
-  callbacks:
-    metrics_over_trainsteps_checkpoint:
-      params:
-        every_n_train_steps: 25000
-
-    image_logger:
-      target: main.ImageLogger
-      params:
-        disabled: False
-        batch_frequency: 1000
-        max_images: 64
-        increase_log_steps: False
-        log_first_step: False
-        log_images_kwargs:
-          use_ema_scope: False
-          N: 64
-          n_rows: 8
-
-  trainer:
-    devices: 0,
-    benchmark: True
-    num_sanity_val_steps: 0
-    accumulate_grad_batches: 1
-    max_epochs: 10

configs/example_training/toy/mnist_cond.yaml

@@ -1,98 +0,0 @@
-model:
-  base_learning_rate: 1.0e-4
-  target: sgm.models.diffusion.DiffusionEngine
-  params:
-    denoiser_config:
-      target: sgm.modules.diffusionmodules.denoiser.Denoiser
-      params:
-        scaling_config:
-          target: sgm.modules.diffusionmodules.denoiser_scaling.EDMScaling
-          params:
-            sigma_data: 1.0
-
-    network_config:
-      target: sgm.modules.diffusionmodules.openaimodel.UNetModel
-      params:
-        in_channels: 1
-        out_channels: 1
-        model_channels: 32
-        attention_resolutions: []
-        num_res_blocks: 4
-        channel_mult: [1, 2, 2]
-        num_head_channels: 32
-        num_classes: sequential
-        adm_in_channels: 128
-
-    conditioner_config:
-      target: sgm.modules.GeneralConditioner
-      params:
-        emb_models:
-          - is_trainable: True
-            input_key: cls
-            ucg_rate: 0.2
-            target: sgm.modules.encoders.modules.ClassEmbedder
-            params:
-              embed_dim: 128
-              n_classes: 10
-
-    first_stage_config:
-      target: sgm.models.autoencoder.IdentityFirstStage
-
-    loss_fn_config:
-      target: sgm.modules.diffusionmodules.loss.StandardDiffusionLoss
-      params:
-        loss_weighting_config:
-          target: sgm.modules.diffusionmodules.loss_weighting.EDMWeighting
-          params:
-            sigma_data: 1.0
-        sigma_sampler_config:
-          target: sgm.modules.diffusionmodules.sigma_sampling.EDMSampling
-
-    sampler_config:
-      target: sgm.modules.diffusionmodules.sampling.EulerEDMSampler
-      params:
-        num_steps: 50
-
-        discretization_config:
-          target: sgm.modules.diffusionmodules.discretizer.EDMDiscretization
-
-        guider_config:
-          target: sgm.modules.diffusionmodules.guiders.VanillaCFG
-          params:
-            scale: 3.0
-
-data:
-  target: sgm.data.mnist.MNISTLoader
-  params:
-    batch_size: 512
-    num_workers: 1
-
-lightning:
-  modelcheckpoint:
-    params:
-      every_n_train_steps: 5000
-
-  callbacks:
-    metrics_over_trainsteps_checkpoint:
-      params:
-        every_n_train_steps: 25000
-
-    image_logger:
-      target: main.ImageLogger
-      params:
-        disabled: False
-        batch_frequency: 1000
-        max_images: 16
-        increase_log_steps: True
-        log_first_step: False
-        log_images_kwargs:
-          use_ema_scope: False
-          N: 16
-          n_rows: 4
-
-  trainer:
-    devices: 0,
-    benchmark: True
-    num_sanity_val_steps: 0
-    accumulate_grad_batches: 1
-    max_epochs: 20

configs/example_training/toy/mnist_cond_discrete_eps.yaml

@@ -1,103 +0,0 @@
-model:
-  base_learning_rate: 1.0e-4
-  target: sgm.models.diffusion.DiffusionEngine
-  params:
-    denoiser_config:
-      target: sgm.modules.diffusionmodules.denoiser.DiscreteDenoiser
-      params:
-        num_idx: 1000
-
-        scaling_config:
-          target: sgm.modules.diffusionmodules.denoiser_scaling.EDMScaling
-        discretization_config:
-          target: sgm.modules.diffusionmodules.discretizer.LegacyDDPMDiscretization
-
-    network_config:
-      target: sgm.modules.diffusionmodules.openaimodel.UNetModel
-      params:
-        in_channels: 1
-        out_channels: 1
-        model_channels: 32
-        attention_resolutions: []
-        num_res_blocks: 4
-        channel_mult: [1, 2, 2]
-        num_head_channels: 32
-        num_classes: sequential
-        adm_in_channels: 128
-
-    conditioner_config:
-      target: sgm.modules.GeneralConditioner
-      params:
-        emb_models:
-          - is_trainable: True
-            input_key: cls
-            ucg_rate: 0.2
-            target: sgm.modules.encoders.modules.ClassEmbedder
-            params:
-              embed_dim: 128
-              n_classes: 10
-
-    first_stage_config:
-      target: sgm.models.autoencoder.IdentityFirstStage
-
-    loss_fn_config:
-      target: sgm.modules.diffusionmodules.loss.StandardDiffusionLoss
-      params:
-        loss_weighting_config:
-          target: sgm.modules.diffusionmodules.loss_weighting.EDMWeighting
-        sigma_sampler_config:
-          target: sgm.modules.diffusionmodules.sigma_sampling.DiscreteSampling
-          params:
-            num_idx: 1000
-
-            discretization_config:
-              target: sgm.modules.diffusionmodules.discretizer.LegacyDDPMDiscretization
-
-    sampler_config:
-      target: sgm.modules.diffusionmodules.sampling.EulerEDMSampler
-      params:
-        num_steps: 50
-
-        discretization_config:
-          target: sgm.modules.diffusionmodules.discretizer.LegacyDDPMDiscretization
-
-        guider_config:
-          target: sgm.modules.diffusionmodules.guiders.VanillaCFG
-          params:
-            scale: 5.0
-
-data:
-  target: sgm.data.mnist.MNISTLoader
-  params:
-    batch_size: 512
-    num_workers: 1
-
-lightning:
-  modelcheckpoint:
-    params:
-      every_n_train_steps: 5000
-
-  callbacks:
-    metrics_over_trainsteps_checkpoint:
-      params:
-        every_n_train_steps: 25000
-
-    image_logger:
-      target: main.ImageLogger
-      params:
-        disabled: False
-        batch_frequency: 1000
-        max_images: 16
-        increase_log_steps: True
-        log_first_step: False
-        log_images_kwargs:
-          use_ema_scope: False
-          N: 16
-          n_rows: 4
-
-  trainer:
-    devices: 0,
-    benchmark: True
-    num_sanity_val_steps: 0
-    accumulate_grad_batches: 1
-    max_epochs: 20

configs/example_training/toy/mnist_cond_l1_loss.yaml

@@ -1,99 +0,0 @@
-model:
-  base_learning_rate: 1.0e-4
-  target: sgm.models.diffusion.DiffusionEngine
-  params:
-    denoiser_config:
-      target: sgm.modules.diffusionmodules.denoiser.Denoiser
-      params:
-        scaling_config:
-          target: sgm.modules.diffusionmodules.denoiser_scaling.EDMScaling
-          params:
-            sigma_data: 1.0
-
-    network_config:
-      target: sgm.modules.diffusionmodules.openaimodel.UNetModel
-      params:
-        in_channels: 1
-        out_channels: 1
-        model_channels: 32
-        attention_resolutions: []
-        num_res_blocks: 4
-        channel_mult: [1, 2, 2]
-        num_head_channels: 32
-        num_classes: sequential
-        adm_in_channels: 128
-
-    conditioner_config:
-      target: sgm.modules.GeneralConditioner
-      params:
-        emb_models:
-          - is_trainable: True
-            input_key: cls
-            ucg_rate: 0.2
-            target: sgm.modules.encoders.modules.ClassEmbedder
-            params:
-              embed_dim: 128
-              n_classes: 10
-
-    first_stage_config:
-      target: sgm.models.autoencoder.IdentityFirstStage
-
-    loss_fn_config:
-      target: sgm.modules.diffusionmodules.loss.StandardDiffusionLoss
-      params:
-        loss_type: l1
-        loss_weighting_config:
-          target: sgm.modules.diffusionmodules.loss_weighting.EDMWeighting
-          params:
-            sigma_data: 1.0
-        sigma_sampler_config:
-          target: sgm.modules.diffusionmodules.sigma_sampling.EDMSampling
-
-    sampler_config:
-      target: sgm.modules.diffusionmodules.sampling.EulerEDMSampler
-      params:
-        num_steps: 50
-
-        discretization_config:
-          target: sgm.modules.diffusionmodules.discretizer.EDMDiscretization
-
-        guider_config:
-          target: sgm.modules.diffusionmodules.guiders.VanillaCFG
-          params:
-            scale: 3.0
-
-data:
-  target: sgm.data.mnist.MNISTLoader
-  params:
-    batch_size: 512
-    num_workers: 1
-
-lightning:
-  modelcheckpoint:
-    params:
-      every_n_train_steps: 5000
-
-  callbacks:
-    metrics_over_trainsteps_checkpoint:
-      params:
-        every_n_train_steps: 25000
-
-    image_logger:
-      target: main.ImageLogger
-      params:
-        disabled: False
-        batch_frequency: 1000
-        max_images: 64
-        increase_log_steps: True
-        log_first_step: False
-        log_images_kwargs:
-          use_ema_scope: False
-          N: 64
-          n_rows: 8
-
-  trainer:
-    devices: 0,
-    benchmark: True
-    num_sanity_val_steps: 0
-    accumulate_grad_batches: 1
-    max_epochs: 20

configs/example_training/toy/mnist_cond_with_ema.yaml

@@ -1,100 +0,0 @@
-model:
-  base_learning_rate: 1.0e-4
-  target: sgm.models.diffusion.DiffusionEngine
-  params:
-    use_ema: True
-
-    denoiser_config:
-      target: sgm.modules.diffusionmodules.denoiser.Denoiser
-      params:
-        scaling_config:
-          target: sgm.modules.diffusionmodules.denoiser_scaling.EDMScaling
-          params:
-            sigma_data: 1.0
-
-    network_config:
-      target: sgm.modules.diffusionmodules.openaimodel.UNetModel
-      params:
-        in_channels: 1
-        out_channels: 1
-        model_channels: 32
-        attention_resolutions: []
-        num_res_blocks: 4
-        channel_mult: [1, 2, 2]
-        num_head_channels: 32
-        num_classes: sequential
-        adm_in_channels: 128
-
-    conditioner_config:
-      target: sgm.modules.GeneralConditioner
-      params:
-        emb_models:
-          - is_trainable: True
-            input_key: cls
-            ucg_rate: 0.2
-            target: sgm.modules.encoders.modules.ClassEmbedder
-            params:
-              embed_dim: 128
-              n_classes: 10
-
-    first_stage_config:
-      target: sgm.models.autoencoder.IdentityFirstStage
-
-    loss_fn_config:
-      target: sgm.modules.diffusionmodules.loss.StandardDiffusionLoss
-      params:
-        loss_weighting_config:
-          target: sgm.modules.diffusionmodules.loss_weighting.EDMWeighting
-          params:
-            sigma_data: 1.0
-        sigma_sampler_config:
-          target: sgm.modules.diffusionmodules.sigma_sampling.EDMSampling
-
-    sampler_config:
-      target: sgm.modules.diffusionmodules.sampling.EulerEDMSampler
-      params:
-        num_steps: 50
-
-        discretization_config:
-          target: sgm.modules.diffusionmodules.discretizer.EDMDiscretization
-
-        guider_config:
-          target: sgm.modules.diffusionmodules.guiders.VanillaCFG
-          params:
-            scale: 3.0
-
-data:
-  target: sgm.data.mnist.MNISTLoader
-  params:
-    batch_size: 512
-    num_workers: 1
-
-lightning:
-  modelcheckpoint:
-    params:
-      every_n_train_steps: 5000
-
-  callbacks:
-    metrics_over_trainsteps_checkpoint:
-      params:
-        every_n_train_steps: 25000
-
-    image_logger:
-      target: main.ImageLogger
-      params:
-        disabled: False
-        batch_frequency: 1000
-        max_images: 64
-        increase_log_steps: True
-        log_first_step: False
-        log_images_kwargs:
-          use_ema_scope: False
-          N: 64
-          n_rows: 8
-
-  trainer:
-    devices: 0,
-    benchmark: True
-    num_sanity_val_steps: 0
-    accumulate_grad_batches: 1
-    max_epochs: 20

configs/example_training/txt2img-clipl-legacy-ucg-training.yaml

@@ -1,182 +0,0 @@
-model:
-  base_learning_rate: 1.0e-4
-  target: sgm.models.diffusion.DiffusionEngine
-  params:
-    scale_factor: 0.13025
-    disable_first_stage_autocast: True
-    log_keys:
-      - txt
-
-    scheduler_config:
-      target: sgm.lr_scheduler.LambdaLinearScheduler
-      params:
-        warm_up_steps: [10000]
-        cycle_lengths: [10000000000000]
-        f_start: [1.e-6]
-        f_max: [1.]
-        f_min: [1.]
-
-    denoiser_config:
-      target: sgm.modules.diffusionmodules.denoiser.DiscreteDenoiser
-      params:
-        num_idx: 1000
-
-        scaling_config:
-          target: sgm.modules.diffusionmodules.denoiser_scaling.EpsScaling
-        discretization_config:
-          target: sgm.modules.diffusionmodules.discretizer.LegacyDDPMDiscretization
-
-    network_config:
-      target: sgm.modules.diffusionmodules.openaimodel.UNetModel
-      params:
-        use_checkpoint: True
-        in_channels: 4
-        out_channels: 4
-        model_channels: 320
-        attention_resolutions: [1, 2, 4]
-        num_res_blocks: 2
-        channel_mult: [1, 2, 4, 4]
-        num_head_channels: 64
-        num_classes: sequential
-        adm_in_channels: 1792
-        num_heads: 1
-        transformer_depth: 1
-        context_dim: 768
-        spatial_transformer_attn_type: softmax-xformers
-
-    conditioner_config:
-      target: sgm.modules.GeneralConditioner
-      params:
-        emb_models:
-          - is_trainable: True
-            input_key: txt
-            ucg_rate: 0.1
-            legacy_ucg_value: ""
-            target: sgm.modules.encoders.modules.FrozenCLIPEmbedder
-            params:
-              always_return_pooled: True
-
-          - is_trainable: False
-            ucg_rate: 0.1
-            input_key: original_size_as_tuple
-            target: sgm.modules.encoders.modules.ConcatTimestepEmbedderND
-            params:
-              outdim: 256
-
-          - is_trainable: False
-            input_key: crop_coords_top_left
-            ucg_rate: 0.1
-            target: sgm.modules.encoders.modules.ConcatTimestepEmbedderND
-            params:
-              outdim: 256
-
-    first_stage_config:
-      target: sgm.models.autoencoder.AutoencoderKL
-      params:
-        ckpt_path: CKPT_PATH
-        embed_dim: 4
-        monitor: val/rec_loss
-        ddconfig:
-          attn_type: vanilla-xformers
-          double_z: true
-          z_channels: 4
-          resolution: 256
-          in_channels: 3
-          out_ch: 3
-          ch: 128
-          ch_mult: [ 1, 2, 4, 4 ]
-          num_res_blocks: 2
-          attn_resolutions: [ ]
-          dropout: 0.0
-        lossconfig:
-          target: torch.nn.Identity
-
-    loss_fn_config:
-      target: sgm.modules.diffusionmodules.loss.StandardDiffusionLoss
-      params:
-        loss_weighting_config:
-          target: sgm.modules.diffusionmodules.loss_weighting.EpsWeighting
-        sigma_sampler_config:
-          target: sgm.modules.diffusionmodules.sigma_sampling.DiscreteSampling
-          params:
-            num_idx: 1000
-
-            discretization_config:
-              target: sgm.modules.diffusionmodules.discretizer.LegacyDDPMDiscretization
-
-    sampler_config:
-      target: sgm.modules.diffusionmodules.sampling.EulerEDMSampler
-      params:
-        num_steps: 50
-
-        discretization_config:
-          target: sgm.modules.diffusionmodules.discretizer.LegacyDDPMDiscretization
-
-        guider_config:
-          target: sgm.modules.diffusionmodules.guiders.VanillaCFG
-          params:
-            scale: 7.5
-
-data:
-  target: sgm.data.dataset.StableDataModuleFromConfig
-  params:
-    train:
-      datapipeline:
-        urls:
-          # USER: adapt this path the root of your custom dataset
-          - DATA_PATH
-        pipeline_config:
-          shardshuffle: 10000
-          sample_shuffle: 10000 # USER: you might wanna adapt depending on your available RAM
-
-        decoders:
-          - pil
-
-        postprocessors:
-          - target: sdata.mappers.TorchVisionImageTransforms
-            params:
-              key: jpg # USER: you might wanna adapt this for your custom dataset
-              transforms:
-                - target: torchvision.transforms.Resize
-                  params:
-                    size: 256
-                    interpolation: 3
-                - target: torchvision.transforms.ToTensor
-          - target: sdata.mappers.Rescaler
-          - target: sdata.mappers.AddOriginalImageSizeAsTupleAndCropToSquare
-            # USER: you might wanna use non-default parameters due to your custom dataset
-
-      loader:
-        batch_size: 64
-        num_workers: 6
-
-lightning:
-  modelcheckpoint:
-    params:
-      every_n_train_steps: 5000
-
-  callbacks:
-    metrics_over_trainsteps_checkpoint:
-      params:
-        every_n_train_steps: 25000
-
-    image_logger:
-      target: main.ImageLogger
-      params:
-        disabled: False
-        enable_autocast: False
-        batch_frequency: 1000
-        max_images: 8
-        increase_log_steps: True
-        log_first_step: False
-        log_images_kwargs:
-          use_ema_scope: False
-          N: 8
-          n_rows: 2
-
-  trainer:
-    devices: 0,
-    benchmark: True
-    num_sanity_val_steps: 0
-    accumulate_grad_batches: 1
-    max_epochs: 1000

configs/example_training/txt2img-clipl.yaml

@@ -1,184 +0,0 @@
-model:
-  base_learning_rate: 1.0e-4
-  target: sgm.models.diffusion.DiffusionEngine
-  params:
-    scale_factor: 0.13025
-    disable_first_stage_autocast: True
-    log_keys:
-      - txt
-
-    scheduler_config:
-      target: sgm.lr_scheduler.LambdaLinearScheduler
-      params:
-        warm_up_steps: [10000]
-        cycle_lengths: [10000000000000]
-        f_start: [1.e-6]
-        f_max: [1.]
-        f_min: [1.]
-
-    denoiser_config:
-      target: sgm.modules.diffusionmodules.denoiser.DiscreteDenoiser
-      params:
-        num_idx: 1000
-
-        scaling_config:
-          target: sgm.modules.diffusionmodules.denoiser_scaling.EpsScaling
-        discretization_config:
-          target: sgm.modules.diffusionmodules.discretizer.LegacyDDPMDiscretization
-
-    network_config:
-      target: sgm.modules.diffusionmodules.openaimodel.UNetModel
-      params:
-        use_checkpoint: True
-        in_channels: 4
-        out_channels: 4
-        model_channels: 320
-        attention_resolutions: [1, 2, 4]
-        num_res_blocks: 2
-        channel_mult: [1, 2, 4, 4]
-        num_head_channels: 64
-        num_classes: sequential
-        adm_in_channels: 1792
-        num_heads: 1
-        transformer_depth: 1
-        context_dim: 768
-        spatial_transformer_attn_type: softmax-xformers
-
-    conditioner_config:
-      target: sgm.modules.GeneralConditioner
-      params:
-        emb_models:
-          - is_trainable: True
-            input_key: txt
-            ucg_rate: 0.1
-            legacy_ucg_value: ""
-            target: sgm.modules.encoders.modules.FrozenCLIPEmbedder
-            params:
-              always_return_pooled: True
-
-          - is_trainable: False
-            ucg_rate: 0.1
-            input_key: original_size_as_tuple
-            target: sgm.modules.encoders.modules.ConcatTimestepEmbedderND
-            params:
-              outdim: 256
-
-          - is_trainable: False
-            input_key: crop_coords_top_left
-            ucg_rate: 0.1
-            target: sgm.modules.encoders.modules.ConcatTimestepEmbedderND
-            params:
-              outdim: 256
-
-    first_stage_config:
-      target: sgm.models.autoencoder.AutoencoderKL
-      params:
-        ckpt_path: CKPT_PATH
-        embed_dim: 4
-        monitor: val/rec_loss
-        ddconfig:
-          attn_type: vanilla-xformers
-          double_z: true
-          z_channels: 4
-          resolution: 256
-          in_channels: 3
-          out_ch: 3
-          ch: 128
-          ch_mult: [1, 2, 4, 4]
-          num_res_blocks: 2
-          attn_resolutions: []
-          dropout: 0.0
-        lossconfig:
-          target: torch.nn.Identity
-
-    loss_fn_config:
-      target: sgm.modules.diffusionmodules.loss.StandardDiffusionLoss
-      params:
-        loss_weighting_config:
-          target: sgm.modules.diffusionmodules.loss_weighting.EpsWeighting
-        sigma_sampler_config:
-          target: sgm.modules.diffusionmodules.sigma_sampling.DiscreteSampling
-          params:
-            num_idx: 1000
-
-            discretization_config:
-              target: sgm.modules.diffusionmodules.discretizer.LegacyDDPMDiscretization
-
-    sampler_config:
-      target: sgm.modules.diffusionmodules.sampling.EulerEDMSampler
-      params:
-        num_steps: 50
-
-        discretization_config:
-          target: sgm.modules.diffusionmodules.discretizer.LegacyDDPMDiscretization
-
-        guider_config:
-          target: sgm.modules.diffusionmodules.guiders.VanillaCFG
-          params:
-            scale: 7.5
-
-data:
-  target: sgm.data.dataset.StableDataModuleFromConfig
-  params:
-    train:
-      datapipeline:
-        urls:
-          # USER: adapt this path the root of your custom dataset
-          - DATA_PATH
-        pipeline_config:
-          shardshuffle: 10000
-          sample_shuffle: 10000
-
-
-        decoders:
-          - pil
-
-        postprocessors:
-          - target: sdata.mappers.TorchVisionImageTransforms
-            params:
-              key: jpg # USER: you might wanna adapt this for your custom dataset
-              transforms:
-                - target: torchvision.transforms.Resize
-                  params:
-                    size: 256
-                    interpolation: 3
-                - target: torchvision.transforms.ToTensor
-          - target: sdata.mappers.Rescaler
-            # USER: you might wanna use non-default parameters due to your custom dataset
-          - target: sdata.mappers.AddOriginalImageSizeAsTupleAndCropToSquare
-            # USER: you might wanna use non-default parameters due to your custom dataset
-
-      loader:
-        batch_size: 64
-        num_workers: 6
-
-lightning:
-  modelcheckpoint:
-    params:
-      every_n_train_steps: 5000
-
-  callbacks:
-    metrics_over_trainsteps_checkpoint:
-      params:
-        every_n_train_steps: 25000
-
-    image_logger:
-      target: main.ImageLogger
-      params:
-        disabled: False
-        enable_autocast: False
-        batch_frequency: 1000
-        max_images: 8
-        increase_log_steps: True
-        log_first_step: False
-        log_images_kwargs:
-          use_ema_scope: False
-          N: 8
-          n_rows: 2
-
-  trainer:
-    devices: 0,
-    benchmark: True
-    num_sanity_val_steps: 0
-    accumulate_grad_batches: 1
-    max_epochs: 1000

configs/inference/sd_2_1.yaml

@@ -1,60 +0,0 @@
-model:
-  target: sgm.models.diffusion.DiffusionEngine
-  params:
-    scale_factor: 0.18215
-    disable_first_stage_autocast: True
-
-    denoiser_config:
-      target: sgm.modules.diffusionmodules.denoiser.DiscreteDenoiser
-      params:
-        num_idx: 1000
-
-        scaling_config:
-          target: sgm.modules.diffusionmodules.denoiser_scaling.EpsScaling
-        discretization_config:
-          target: sgm.modules.diffusionmodules.discretizer.LegacyDDPMDiscretization
-
-    network_config:
-      target: sgm.modules.diffusionmodules.openaimodel.UNetModel
-      params:
-        use_checkpoint: True
-        in_channels: 4
-        out_channels: 4
-        model_channels: 320
-        attention_resolutions: [4, 2, 1]
-        num_res_blocks: 2
-        channel_mult: [1, 2, 4, 4]
-        num_head_channels: 64
-        use_linear_in_transformer: True
-        transformer_depth: 1
-        context_dim: 1024
-
-    conditioner_config:
-      target: sgm.modules.GeneralConditioner
-      params:
-        emb_models:
-          - is_trainable: False
-            input_key: txt
-            target: sgm.modules.encoders.modules.FrozenOpenCLIPEmbedder
-            params:
-              freeze: true
-              layer: penultimate
-
-    first_stage_config:
-      target: sgm.models.autoencoder.AutoencoderKL
-      params:
-        embed_dim: 4
-        monitor: val/rec_loss
-        ddconfig:
-          double_z: true
-          z_channels: 4
-          resolution: 256
-          in_channels: 3
-          out_ch: 3
-          ch: 128
-          ch_mult: [1, 2, 4, 4]
-          num_res_blocks: 2
-          attn_resolutions: []
-          dropout: 0.0
-        lossconfig:
-          target: torch.nn.Identity

configs/inference/sd_2_1_768.yaml

@@ -1,60 +0,0 @@
-model:
-  target: sgm.models.diffusion.DiffusionEngine
-  params:
-    scale_factor: 0.18215
-    disable_first_stage_autocast: True
-
-    denoiser_config:
-      target: sgm.modules.diffusionmodules.denoiser.DiscreteDenoiser
-      params:
-        num_idx: 1000
-
-        scaling_config:
-          target: sgm.modules.diffusionmodules.denoiser_scaling.VScaling
-        discretization_config:
-          target: sgm.modules.diffusionmodules.discretizer.LegacyDDPMDiscretization
-
-    network_config:
-      target: sgm.modules.diffusionmodules.openaimodel.UNetModel
-      params:
-        use_checkpoint: True
-        in_channels: 4
-        out_channels: 4
-        model_channels: 320
-        attention_resolutions: [4, 2, 1]
-        num_res_blocks: 2
-        channel_mult: [1, 2, 4, 4]
-        num_head_channels: 64
-        use_linear_in_transformer: True
-        transformer_depth: 1
-        context_dim: 1024
-
-    conditioner_config:
-      target: sgm.modules.GeneralConditioner
-      params:
-        emb_models:
-          - is_trainable: False
-            input_key: txt
-            target: sgm.modules.encoders.modules.FrozenOpenCLIPEmbedder
-            params:
-              freeze: true
-              layer: penultimate
-
-    first_stage_config:
-      target: sgm.models.autoencoder.AutoencoderKL
-      params:
-        embed_dim: 4
-        monitor: val/rec_loss
-        ddconfig:
-          double_z: true
-          z_channels: 4
-          resolution: 256
-          in_channels: 3
-          out_ch: 3
-          ch: 128
-          ch_mult: [1, 2, 4, 4]
-          num_res_blocks: 2
-          attn_resolutions: []
-          dropout: 0.0
-        lossconfig:
-          target: torch.nn.Identity

configs/inference/sd_xl_base.yaml

@@ -1,93 +0,0 @@
-model:
-  target: sgm.models.diffusion.DiffusionEngine
-  params:
-    scale_factor: 0.13025
-    disable_first_stage_autocast: True
-
-    denoiser_config:
-      target: sgm.modules.diffusionmodules.denoiser.DiscreteDenoiser
-      params:
-        num_idx: 1000
-
-        scaling_config:
-          target: sgm.modules.diffusionmodules.denoiser_scaling.EpsScaling
-        discretization_config:
-          target: sgm.modules.diffusionmodules.discretizer.LegacyDDPMDiscretization
-
-    network_config:
-      target: sgm.modules.diffusionmodules.openaimodel.UNetModel
-      params:
-        adm_in_channels: 2816
-        num_classes: sequential
-        use_checkpoint: True
-        in_channels: 4
-        out_channels: 4
-        model_channels: 320
-        attention_resolutions: [4, 2]
-        num_res_blocks: 2
-        channel_mult: [1, 2, 4]
-        num_head_channels: 64
-        use_linear_in_transformer: True
-        transformer_depth: [1, 2, 10]
-        context_dim: 2048
-        spatial_transformer_attn_type: softmax-xformers
-
-    conditioner_config:
-      target: sgm.modules.GeneralConditioner
-      params:
-        emb_models:
-          - is_trainable: False
-            input_key: txt
-            target: sgm.modules.encoders.modules.FrozenCLIPEmbedder
-            params:
-              layer: hidden
-              layer_idx: 11
-
-          - is_trainable: False
-            input_key: txt
-            target: sgm.modules.encoders.modules.FrozenOpenCLIPEmbedder2
-            params:
-              arch: ViT-bigG-14
-              version: laion2b_s39b_b160k
-              freeze: True
-              layer: penultimate
-              always_return_pooled: True
-              legacy: False
-
-          - is_trainable: False
-            input_key: original_size_as_tuple
-            target: sgm.modules.encoders.modules.ConcatTimestepEmbedderND
-            params:
-              outdim: 256
-
-          - is_trainable: False
-            input_key: crop_coords_top_left
-            target: sgm.modules.encoders.modules.ConcatTimestepEmbedderND
-            params:
-              outdim: 256
-
-          - is_trainable: False
-            input_key: target_size_as_tuple
-            target: sgm.modules.encoders.modules.ConcatTimestepEmbedderND
-            params:
-              outdim: 256
-
-    first_stage_config:
-      target: sgm.models.autoencoder.AutoencoderKL
-      params:
-        embed_dim: 4
-        monitor: val/rec_loss
-        ddconfig:
-          attn_type: vanilla-xformers
-          double_z: true
-          z_channels: 4
-          resolution: 256
-          in_channels: 3
-          out_ch: 3
-          ch: 128
-          ch_mult: [1, 2, 4, 4]
-          num_res_blocks: 2
-          attn_resolutions: []
-          dropout: 0.0
-        lossconfig:
-          target: torch.nn.Identity

configs/inference/sd_xl_refiner.yaml

@@ -1,86 +0,0 @@
-model:
-  target: sgm.models.diffusion.DiffusionEngine
-  params:
-    scale_factor: 0.13025
-    disable_first_stage_autocast: True
-
-    denoiser_config:
-      target: sgm.modules.diffusionmodules.denoiser.DiscreteDenoiser
-      params:
-        num_idx: 1000
-
-        scaling_config:
-          target: sgm.modules.diffusionmodules.denoiser_scaling.EpsScaling
-        discretization_config:
-          target: sgm.modules.diffusionmodules.discretizer.LegacyDDPMDiscretization
-
-    network_config:
-      target: sgm.modules.diffusionmodules.openaimodel.UNetModel
-      params:
-        adm_in_channels: 2560
-        num_classes: sequential
-        use_checkpoint: True
-        in_channels: 4
-        out_channels: 4
-        model_channels: 384
-        attention_resolutions: [4, 2]
-        num_res_blocks: 2
-        channel_mult: [1, 2, 4, 4]
-        num_head_channels: 64
-        use_linear_in_transformer: True
-        transformer_depth: 4
-        context_dim: [1280, 1280, 1280, 1280]
-        spatial_transformer_attn_type: softmax-xformers
-
-    conditioner_config:
-      target: sgm.modules.GeneralConditioner
-      params:
-        emb_models:
-          - is_trainable: False
-            input_key: txt
-            target: sgm.modules.encoders.modules.FrozenOpenCLIPEmbedder2
-            params:
-              arch: ViT-bigG-14
-              version: laion2b_s39b_b160k
-              legacy: False
-              freeze: True
-              layer: penultimate
-              always_return_pooled: True
-
-          - is_trainable: False
-            input_key: original_size_as_tuple
-            target: sgm.modules.encoders.modules.ConcatTimestepEmbedderND
-            params:
-              outdim: 256
-
-          - is_trainable: False
-            input_key: crop_coords_top_left
-            target: sgm.modules.encoders.modules.ConcatTimestepEmbedderND
-            params:
-              outdim: 256
-
-          - is_trainable: False
-            input_key: aesthetic_score
-            target: sgm.modules.encoders.modules.ConcatTimestepEmbedderND
-            params:
-              outdim: 256
-
-    first_stage_config:
-      target: sgm.models.autoencoder.AutoencoderKL
-      params:
-        embed_dim: 4
-        monitor: val/rec_loss
-        ddconfig:
-          attn_type: vanilla-xformers
-          double_z: true
-          z_channels: 4
-          resolution: 256
-          in_channels: 3
-          out_ch: 3
-          ch: 128
-          ch_mult: [1, 2, 4, 4]
-          num_res_blocks: 2
-          attn_resolutions: []
-          dropout: 0.0
-        lossconfig:
-          target: torch.nn.Identity

configs/inference/svd.yaml

@@ -1,131 +0,0 @@
-model:
-  target: sgm.models.diffusion.DiffusionEngine
-  params:
-    scale_factor: 0.18215
-    disable_first_stage_autocast: True
-
-    denoiser_config:
-      target: sgm.modules.diffusionmodules.denoiser.Denoiser
-      params:
-        scaling_config:
-          target: sgm.modules.diffusionmodules.denoiser_scaling.VScalingWithEDMcNoise
-
-    network_config:
-      target: sgm.modules.diffusionmodules.video_model.VideoUNet
-      params:
-        adm_in_channels: 768
-        num_classes: sequential
-        use_checkpoint: True
-        in_channels: 8
-        out_channels: 4
-        model_channels: 320
-        attention_resolutions: [4, 2, 1]
-        num_res_blocks: 2
-        channel_mult: [1, 2, 4, 4]
-        num_head_channels: 64
-        use_linear_in_transformer: True
-        transformer_depth: 1
-        context_dim: 1024
-        spatial_transformer_attn_type: softmax-xformers
-        extra_ff_mix_layer: True
-        use_spatial_context: True
-        merge_strategy: learned_with_images
-        video_kernel_size: [3, 1, 1]
-
-    conditioner_config:
-      target: sgm.modules.GeneralConditioner
-      params:
-        emb_models:
-        - is_trainable: False
-          input_key: cond_frames_without_noise
-          target: sgm.modules.encoders.modules.FrozenOpenCLIPImagePredictionEmbedder
-          params:
-            n_cond_frames: 1
-            n_copies: 1
-            open_clip_embedding_config:
-              target: sgm.modules.encoders.modules.FrozenOpenCLIPImageEmbedder
-              params:
-                freeze: True
-
-        - input_key: fps_id
-          is_trainable: False
-          target: sgm.modules.encoders.modules.ConcatTimestepEmbedderND
-          params:
-            outdim: 256
-
-        - input_key: motion_bucket_id
-          is_trainable: False
-          target: sgm.modules.encoders.modules.ConcatTimestepEmbedderND
-          params:
-            outdim: 256
-
-        - input_key: cond_frames
-          is_trainable: False
-          target: sgm.modules.encoders.modules.VideoPredictionEmbedderWithEncoder
-          params:
-            disable_encoder_autocast: True
-            n_cond_frames: 1
-            n_copies: 1
-            is_ae: True
-            encoder_config:
-              target: sgm.models.autoencoder.AutoencoderKLModeOnly
-              params:
-                embed_dim: 4
-                monitor: val/rec_loss
-                ddconfig:
-                  attn_type: vanilla-xformers
-                  double_z: True
-                  z_channels: 4
-                  resolution: 256
-                  in_channels: 3
-                  out_ch: 3
-                  ch: 128
-                  ch_mult: [1, 2, 4, 4]
-                  num_res_blocks: 2
-                  attn_resolutions: []
-                  dropout: 0.0
-                lossconfig:
-                  target: torch.nn.Identity
-
-        - input_key: cond_aug
-          is_trainable: False
-          target: sgm.modules.encoders.modules.ConcatTimestepEmbedderND
-          params:
-            outdim: 256
-
-    first_stage_config:
-      target: sgm.models.autoencoder.AutoencodingEngine
-      params:
-        loss_config:
-          target: torch.nn.Identity
-        regularizer_config:
-          target: sgm.modules.autoencoding.regularizers.DiagonalGaussianRegularizer
-        encoder_config: 
-          target: sgm.modules.diffusionmodules.model.Encoder
-          params:
-            attn_type: vanilla
-            double_z: True
-            z_channels: 4
-            resolution: 256
-            in_channels: 3
-            out_ch: 3
-            ch: 128
-            ch_mult: [1, 2, 4, 4]
-            num_res_blocks: 2
-            attn_resolutions: []
-            dropout: 0.0
-        decoder_config:
-          target: sgm.modules.autoencoding.temporal_ae.VideoDecoder
-          params:
-            attn_type: vanilla
-            double_z: True
-            z_channels: 4
-            resolution: 256
-            in_channels: 3
-            out_ch: 3
-            ch: 128
-            ch_mult: [1, 2, 4, 4]
-            num_res_blocks: 2
-            attn_resolutions: []
-            dropout: 0.0
-            video_kernel_size: [3, 1, 1]

configs/inference/svd_image_decoder.yaml

@@ -1,114 +0,0 @@
-model:
-  target: sgm.models.diffusion.DiffusionEngine
-  params:
-    scale_factor: 0.18215
-    disable_first_stage_autocast: True
-
-    denoiser_config:
-      target: sgm.modules.diffusionmodules.denoiser.Denoiser
-      params:
-        scaling_config:
-          target: sgm.modules.diffusionmodules.denoiser_scaling.VScalingWithEDMcNoise
-
-    network_config:
-      target: sgm.modules.diffusionmodules.video_model.VideoUNet
-      params:
-        adm_in_channels: 768
-        num_classes: sequential
-        use_checkpoint: True
-        in_channels: 8
-        out_channels: 4
-        model_channels: 320
-        attention_resolutions: [4, 2, 1]
-        num_res_blocks: 2
-        channel_mult: [1, 2, 4, 4]
-        num_head_channels: 64
-        use_linear_in_transformer: True
-        transformer_depth: 1
-        context_dim: 1024
-        spatial_transformer_attn_type: softmax-xformers
-        extra_ff_mix_layer: True
-        use_spatial_context: True
-        merge_strategy: learned_with_images
-        video_kernel_size: [3, 1, 1]
-
-    conditioner_config:
-      target: sgm.modules.GeneralConditioner
-      params:
-        emb_models:
-        - is_trainable: False
-          input_key: cond_frames_without_noise
-          target: sgm.modules.encoders.modules.FrozenOpenCLIPImagePredictionEmbedder
-          params:
-            n_cond_frames: 1
-            n_copies: 1
-            open_clip_embedding_config:
-              target: sgm.modules.encoders.modules.FrozenOpenCLIPImageEmbedder
-              params:
-                freeze: True
-
-        - input_key: fps_id
-          is_trainable: False
-          target: sgm.modules.encoders.modules.ConcatTimestepEmbedderND
-          params:
-            outdim: 256
-
-        - input_key: motion_bucket_id
-          is_trainable: False
-          target: sgm.modules.encoders.modules.ConcatTimestepEmbedderND
-          params:
-            outdim: 256
-
-        - input_key: cond_frames
-          is_trainable: False
-          target: sgm.modules.encoders.modules.VideoPredictionEmbedderWithEncoder
-          params:
-            disable_encoder_autocast: True
-            n_cond_frames: 1
-            n_copies: 1
-            is_ae: True
-            encoder_config:
-              target: sgm.models.autoencoder.AutoencoderKLModeOnly
-              params:
-                embed_dim: 4
-                monitor: val/rec_loss
-                ddconfig:
-                  attn_type: vanilla-xformers
-                  double_z: True
-                  z_channels: 4
-                  resolution: 256
-                  in_channels: 3
-                  out_ch: 3
-                  ch: 128
-                  ch_mult: [1, 2, 4, 4]
-                  num_res_blocks: 2
-                  attn_resolutions: []
-                  dropout: 0.0
-                lossconfig:
-                  target: torch.nn.Identity
-
-        - input_key: cond_aug
-          is_trainable: False
-          target: sgm.modules.encoders.modules.ConcatTimestepEmbedderND
-          params:
-            outdim: 256
-
-    first_stage_config:
-      target: sgm.models.autoencoder.AutoencoderKL
-      params:
-        embed_dim: 4
-        monitor: val/rec_loss
-        ddconfig:
-          attn_type: vanilla-xformers
-          double_z: True
-          z_channels: 4
-          resolution: 256
-          in_channels: 3
-          out_ch: 3
-          ch: 128
-          ch_mult: [1, 2, 4, 4]
-          num_res_blocks: 2
-          attn_resolutions: []
-          dropout: 0.0
-        lossconfig:
-          target: torch.nn.Identity

configs/inference/svd_mv.yaml

@@ -1,202 +0,0 @@
-model:
-  base_learning_rate: 1.0e-05
-  target: sgm.models.video_diffusion.DiffusionEngine
-  params:
-    ckpt_path: ckpts/svd_xt.safetensors
-    scale_factor: 0.18215
-    disable_first_stage_autocast: true
-    scheduler_config:
-      target: sgm.lr_scheduler.LambdaLinearScheduler
-      params:
-        warm_up_steps:
-        - 1
-        cycle_lengths:
-        - 10000000000000
-        f_start:
-        - 1.0e-06
-        f_max:
-        - 1.0
-        f_min:
-        - 1.0
-    denoiser_config:
-      target: sgm.modules.diffusionmodules.denoiser.Denoiser
-      params:
-        scaling_config:
-          target: sgm.modules.diffusionmodules.denoiser_scaling.VScalingWithEDMcNoise
-    network_config:
-      target: sgm.modules.diffusionmodules.video_model.VideoUNet
-      params:
-        adm_in_channels: 768
-        num_classes: sequential
-        use_checkpoint: true
-        in_channels: 8
-        out_channels: 4
-        model_channels: 320
-        attention_resolutions:
-        - 4
-        - 2
-        - 1
-        num_res_blocks: 2
-        channel_mult:
-        - 1
-        - 2
-        - 4
-        - 4
-        num_head_channels: 64
-        use_linear_in_transformer: true
-        transformer_depth: 1
-        context_dim: 1024
-        spatial_transformer_attn_type: softmax-xformers
-        extra_ff_mix_layer: true
-        use_spatial_context: true
-        merge_strategy: learned_with_images
-        video_kernel_size:
-        - 3
-        - 1
-        - 1
-    conditioner_config:
-      target: sgm.modules.GeneralConditioner
-      params:
-        emb_models:
-        - is_trainable: false
-          ucg_rate: 0.2
-          input_key: cond_frames_without_noise
-          target: sgm.modules.encoders.modules.FrozenOpenCLIPImagePredictionEmbedder
-          params:
-            n_cond_frames: 1
-            n_copies: 1
-            open_clip_embedding_config:
-              target: sgm.modules.encoders.modules.FrozenOpenCLIPImageEmbedder
-              params:
-                freeze: true
-        - input_key: fps_id
-          is_trainable: true
-          target: sgm.modules.encoders.modules.ConcatTimestepEmbedderND
-          params:
-            outdim: 256
-        - input_key: motion_bucket_id
-          is_trainable: true
-          target: sgm.modules.encoders.modules.ConcatTimestepEmbedderND
-          params:
-            outdim: 256
-        - input_key: cond_frames
-          is_trainable: false
-          ucg_rate: 0.2
-          target: sgm.modules.encoders.modules.VideoPredictionEmbedderWithEncoder
-          params:
-            disable_encoder_autocast: true
-            n_cond_frames: 1
-            n_copies: 1
-            is_ae: true
-            encoder_config:
-              target: sgm.models.autoencoder.AutoencoderKLModeOnly
-              params:
-                embed_dim: 4
-                monitor: val/rec_loss
-                ddconfig:
-                  attn_type: vanilla-xformers
-                  double_z: true
-                  z_channels: 4
-                  resolution: 256
-                  in_channels: 3
-                  out_ch: 3
-                  ch: 128
-                  ch_mult:
-                  - 1
-                  - 2
-                  - 4
-                  - 4
-                  num_res_blocks: 2
-                  attn_resolutions: []
-                  dropout: 0.0
-                lossconfig:
-                  target: torch.nn.Identity
-        - input_key: cond_aug
-          is_trainable: true
-          target: sgm.modules.encoders.modules.ConcatTimestepEmbedderND
-          params:
-            outdim: 256
-    first_stage_config:
-      target: sgm.models.autoencoder.AutoencodingEngine
-      params:
-        loss_config:
-          target: torch.nn.Identity
-        regularizer_config:
-          target: sgm.modules.autoencoding.regularizers.DiagonalGaussianRegularizer
-        encoder_config:
-          target: sgm.modules.diffusionmodules.model.Encoder
-          params:
-            attn_type: vanilla
-            double_z: true
-            z_channels: 4
-            resolution: 256
-            in_channels: 3
-            out_ch: 3
-            ch: 128
-            ch_mult:
-            - 1
-            - 2
-            - 4
-            - 4
-            num_res_blocks: 2
-            attn_resolutions: []
-            dropout: 0.0
-        decoder_config:
-          target: sgm.modules.autoencoding.temporal_ae.VideoDecoder
-          params:
-            attn_type: vanilla
-            double_z: true
-            z_channels: 4
-            resolution: 256
-            in_channels: 3
-            out_ch: 3
-            ch: 128
-            ch_mult:
-            - 1
-            - 2
-            - 4
-            - 4
-            num_res_blocks: 2
-            attn_resolutions: []
-            dropout: 0.0
-            video_kernel_size:
-            - 3
-            - 1
-            - 1
-    sampler_config:
-      target: sgm.modules.diffusionmodules.sampling.EulerEDMSampler
-      params:
-        num_steps: 30
-        discretization_config:
-          target: sgm.modules.diffusionmodules.discretizer.EDMDiscretization
-          params:
-            sigma_max: 700.0
-        guider_config:
-          target: sgm.modules.diffusionmodules.guiders.LinearPredictionGuider
-          params:
-            max_scale: 2.5
-            min_scale: 1.0
-            num_frames: 24
-    loss_fn_config:
-      target: sgm.modules.diffusionmodules.loss.StandardDiffusionLoss
-      params:
-        batch2model_keys:
-        - num_video_frames
-        - image_only_indicator
-        loss_weighting_config:
-          target: sgm.modules.diffusionmodules.loss_weighting.EDMWeighting
-          params:
-            sigma_data: 1.0
-        sigma_sampler_config:
-          target: sgm.modules.diffusionmodules.sigma_sampling.EDMSampling
-          params:
-            p_mean: 0.3
-            p_std: 1.2
-data:
-  target: sgm.data.objaverse.ObjaverseSpiralDataset
-  params:
-    root_dir: /mnt/mfs/zilong.chen/Downloads/objaverse-ndd-samples
-    random_front: true
-    batch_size: 2
-    num_workers: 16
-    cond_aug_mean: -0.0

configs/instant-mesh-base.yaml

@@ -0,0 +1,22 @@
+model_config:
+  target: src.models.lrm_mesh.InstantMesh
+  params:
+    encoder_feat_dim: 768
+    encoder_freeze: false
+    encoder_model_name: facebook/dino-vitb16
+    transformer_dim: 1024
+    transformer_layers: 12
+    transformer_heads: 16
+    triplane_low_res: 32
+    triplane_high_res: 64
+    triplane_dim: 40
+    rendering_samples_per_ray: 96
+    grid_res: 128
+    grid_scale: 2.1
+
+
+infer_config:
+  unet_path: ckpts/diffusion_pytorch_model.bin
+  model_path: ckpts/instant_mesh_base.ckpt
+  texture_resolution: 1024
+  render_resolution: 512

configs/instant-mesh-large.yaml

@@ -0,0 +1,22 @@
+model_config:
+  target: src.models.lrm_mesh.InstantMesh
+  params:
+    encoder_feat_dim: 768
+    encoder_freeze: false
+    encoder_model_name: facebook/dino-vitb16
+    transformer_dim: 1024
+    transformer_layers: 16
+    transformer_heads: 16
+    triplane_low_res: 32
+    triplane_high_res: 64
+    triplane_dim: 80
+    rendering_samples_per_ray: 128
+    grid_res: 128
+    grid_scale: 2.1
+
+
+infer_config:
+  unet_path: ckpts/diffusion_pytorch_model.bin
+  model_path: ckpts/instant_mesh_large.ckpt
+  texture_resolution: 1024
+  render_resolution: 512

configs/instant-nerf-base.yaml

@@ -0,0 +1,21 @@
+model_config:
+  target: src.models.lrm.InstantNeRF
+  params:
+    encoder_feat_dim: 768
+    encoder_freeze: false
+    encoder_model_name: facebook/dino-vitb16
+    transformer_dim: 1024
+    transformer_layers: 12
+    transformer_heads: 16
+    triplane_low_res: 32
+    triplane_high_res: 64
+    triplane_dim: 40
+    rendering_samples_per_ray: 96
+
+
+infer_config:
+  unet_path: ckpts/diffusion_pytorch_model.bin
+  model_path: ckpts/instant_nerf_base.ckpt
+  mesh_threshold: 10.0
+  mesh_resolution: 256
+  render_resolution: 384

configs/instant-nerf-large.yaml

@@ -0,0 +1,21 @@
+model_config:
+  target: src.models.lrm.InstantNeRF
+  params:
+    encoder_feat_dim: 768
+    encoder_freeze: false
+    encoder_model_name: facebook/dino-vitb16
+    transformer_dim: 1024
+    transformer_layers: 16
+    transformer_heads: 16
+    triplane_low_res: 32
+    triplane_high_res: 64
+    triplane_dim: 80
+    rendering_samples_per_ray: 128
+
+
+infer_config:
+  unet_path: ckpts/diffusion_pytorch_model.bin
+  model_path: ckpts/instant_nerf_large.ckpt
+  mesh_threshold: 10.0
+  mesh_resolution: 256
+  render_resolution: 384