add files

README.md

@@ -1,3 +1,47 @@
----
-license: apache-2.0
----
+# Official code for SINDER: Repairing the Singular Defects of DINOv2 (ECCV 2024 Oral)
+
+[![🦢 - Paper](https://img.shields.io/badge/🦢-Paper-red)](https://arxiv.org/abs/2407.16826)
+
+![SINDER](./resources/high_norm.jpg)
+
+## Citation
+
+```bibtex
+@InProceedings{Haoqi_2024_ECCV,
+author = {Wang, Haoqi and Zhang, Tong and Salzmann, Mathieu},
+title = {{SINDER}: Repairing the Singular Defects of DINOv2},
+booktitle = {Proceedings of the European Conference on Computer Vision (ECCV)},
+month = {September},
+year = {2024}
+}
+```
+
+## Install
+
+```bash
+conda env create -f environment.yml
+conda activate sinder
+pip install -e .
+```
+
+## Train
+
+Put ImageNet-1K dataset at `data/imagenet` folder.
+
+```bash
+./main.py
+```
+
+## Visualize
+
+```bash
+# visualize original dinov2
+./visualize.py resources/example.jpg
+
+# visualize sinder, ckpt download link below
+./visualize.py resources/example.jpg --checkpoint path/to/sinder.pth
+```
+
+## Checkpoints
+
+[Google Drive](https://drive.google.com/file/d/1g0Aq5qXYuMmVrN9-gGwC9ybwlCDFAw-l/view?usp=sharing)

__init__.py

@@ -0,0 +1,17 @@
+from .data import load_data, load_visual_data
+from .neighbor_loss import get_neighbor_loss
+from .repair import replace_back, replace_linear_addition_noqk
+from .singular_defect import singular_defect_directions
+from .utils import get_tokens, load_model, pca_array
+
+__all__ = [
+    'singular_defect_directions',
+    'get_neighbor_loss',
+    'pca_array',
+    'get_tokens',
+    'load_data',
+    'load_visual_data',
+    'replace_back',
+    'replace_linear_addition_noqk',
+    'load_model',
+]

data.py

@@ -0,0 +1,130 @@
+from glob import glob
+
+import torchvision.transforms as transforms
+from PIL import Image
+from torch.utils.data import Dataset
+
+
+def make_transform(
+    smaller_edge_size: int, patch_size, center_crop=False, max_edge_size=812
+) -> transforms.Compose:
+    IMAGENET_DEFAULT_MEAN = (0.485, 0.456, 0.406)
+    IMAGENET_DEFAULT_STD = (0.229, 0.224, 0.225)
+    interpolation_mode = transforms.InterpolationMode.BICUBIC
+    assert smaller_edge_size > 0
+
+    if center_crop:
+        return transforms.Compose(
+            [
+                transforms.Resize(
+                    size=smaller_edge_size,
+                    interpolation=interpolation_mode,
+                    antialias=True,
+                ),
+                transforms.CenterCrop(smaller_edge_size),
+                transforms.ToTensor(),
+                transforms.Normalize(
+                    mean=IMAGENET_DEFAULT_MEAN, std=IMAGENET_DEFAULT_STD
+                ),
+                transforms.Lambda(
+                    lambda img: img[
+                        :,
+                        : min(
+                            max_edge_size,
+                            (img.shape[1] - img.shape[1] % patch_size),
+                        ),
+                        : min(
+                            max_edge_size,
+                            (img.shape[2] - img.shape[2] % patch_size),
+                        ),
+                    ]
+                ),
+            ]
+        )
+    else:
+        return transforms.Compose(
+            [
+                transforms.Resize(
+                    size=smaller_edge_size,
+                    interpolation=interpolation_mode,
+                    antialias=True,
+                ),
+                transforms.ToTensor(),
+                transforms.Normalize(
+                    mean=IMAGENET_DEFAULT_MEAN, std=IMAGENET_DEFAULT_STD
+                ),
+                transforms.Lambda(
+                    lambda img: img[
+                        :,
+                        : min(
+                            max_edge_size,
+                            (img.shape[1] - img.shape[1] % patch_size),
+                        ),
+                        : min(
+                            max_edge_size,
+                            (img.shape[2] - img.shape[2] % patch_size),
+                        ),
+                    ]
+                ),
+            ]
+        )
+
+
+class VisualDataset(Dataset):
+    def __init__(self, transform, imgs=None):
+        self.transform = transform
+        if imgs is None:
+            self.files = [
+                'resources/example.jpg',
+                'resources/villa.png',
+                'resources/000000037740.jpg',
+                'resources/000000064359.jpg',
+                'resources/000000066635.jpg',
+                'resources/000000078420.jpg',
+            ]
+        else:
+            self.files = imgs
+
+    def __len__(self):
+        return len(self.files)
+
+    def __getitem__(self, index):
+        img = self.files[index]
+        img = Image.open(img).convert('RGB')
+        if self.transform:
+            img = self.transform(img)
+        return img
+
+
+class ImageNetDataset(Dataset):
+    def __init__(self, transform, num_train_max=1000000):
+        self.transform = transform
+        self.files = glob('data/imagenet/train/*/*.JPEG')
+        step = len(self.files) // num_train_max
+        self.files = self.files[::step]
+
+    def __len__(self):
+        return len(self.files)
+
+    def __getitem__(self, index):
+        img = Image.open(self.files[index]).convert('RGB')
+        img = self.transform(img)
+        return img
+
+
+def load_data(args, model):
+    transform = make_transform(
+        args.resolution, model.patch_size, center_crop=True
+    )
+    dataset = ImageNetDataset(
+        transform=transform, num_train_max=args.num_train_max
+    )
+    return dataset
+
+
+def load_visual_data(args, model):
+    transform = make_transform(
+        args.visual_size, model.patch_size, max_edge_size=1792
+    )
+    dataset = VisualDataset(transform=transform, imgs=vars(args).get('imgs'))
+    return dataset

environment.yml

@@ -0,0 +1,285 @@
+name: sinder
+channels:
+  - pytorch
+  - nvidia
+  - conda-forge
+dependencies:
+  - _libgcc_mutex=0.1
+  - _openmp_mutex=4.5
+  - alsa-lib=1.2.12
+  - aom=3.9.1
+  - asttokens=2.4.1
+  - black=22.1.0
+  - blas=2.116
+  - blas-devel=3.9.0
+  - blue=0.9.1
+  - brotli-python=1.1.0
+  - bzip2=1.0.8
+  - c-ares=1.33.1
+  - ca-certificates=2024.8.30
+  - cairo=1.18.0
+  - certifi=2024.8.30
+  - cffi=1.16.0
+  - cfgv=3.3.1
+  - charset-normalizer=3.3.2
+  - click=8.1.7
+  - colorama=0.4.6
+  - cuda-cudart=12.1.105
+  - cuda-cupti=12.1.105
+  - cuda-libraries=12.1.0
+  - cuda-nvrtc=12.1.105
+  - cuda-nvtx=12.1.105
+  - cuda-opencl=12.5.39
+  - cuda-runtime=12.1.0
+  - cuda-version=12.5
+  - dataclasses=0.8
+  - dav1d=1.2.1
+  - dbus=1.13.6
+  - decorator=5.1.1
+  - distlib=0.3.8
+  - double-conversion=3.3.0
+  - exceptiongroup=1.2.2
+  - executing=2.0.1
+  - expat=2.6.2
+  - ffmpeg=6.1.2
+  - filelock=3.15.4
+  - flake8=4.0.1
+  - font-ttf-dejavu-sans-mono=2.37
+  - font-ttf-inconsolata=3.000
+  - font-ttf-source-code-pro=2.038
+  - font-ttf-ubuntu=0.83
+  - fontconfig=2.14.2
+  - fonts-conda-ecosystem=1
+  - fonts-conda-forge=1
+  - freeglut=3.2.2
+  - freetype=2.12.1
+  - fribidi=1.0.10
+  - gettext=0.22.5
+  - gettext-tools=0.22.5
+  - gmp=6.3.0
+  - gmpy2=2.1.5
+  - gnutls=3.7.9
+  - graphite2=1.3.13
+  - h2=4.1.0
+  - harfbuzz=9.0.0
+  - hdf5=1.14.3
+  - hpack=4.0.0
+  - hyperframe=6.0.1
+  - icu=75.1
+  - identify=2.6.0
+  - idna=3.7
+  - imath=3.1.12
+  - importlib-metadata=8.5.0
+  - ipdb=0.13.13
+  - ipython=8.26.0
+  - jasper=4.2.4
+  - jedi=0.19.1
+  - jinja2=3.1.4
+  - joblib=1.4.2
+  - keyutils=1.6.1
+  - krb5=1.21.3
+  - lame=3.100
+  - lcms2=2.16
+  - ld_impl_linux-64=2.40
+  - lerc=4.0.0
+  - libabseil=20240116.2
+  - libaec=1.1.3
+  - libasprintf=0.22.5
+  - libasprintf-devel=0.22.5
+  - libass=0.17.3
+  - libblas=3.9.0
+  - libcblas=3.9.0
+  - libclang-cpp18.1=18.1.8
+  - libclang13=19.1.0
+  - libcublas=12.1.0.26
+  - libcufft=11.0.2.4
+  - libcufile=1.10.1.7
+  - libcups=2.3.3
+  - libcurand=10.3.6.82
+  - libcurl=8.10.1
+  - libcusolver=11.4.4.55
+  - libcusparse=12.0.2.55
+  - libdeflate=1.21
+  - libdrm=2.4.122
+  - libedit=3.1.20191231
+  - libegl=1.7.0
+  - libev=4.33
+  - libexpat=2.6.2
+  - libffi=3.4.2
+  - libgcc=14.1.0
+  - libgcc-ng=14.1.0
+  - libgettextpo=0.22.5
+  - libgettextpo-devel=0.22.5
+  - libgfortran=14.1.0
+  - libgfortran-ng=14.1.0
+  - libgfortran5=14.1.0
+  - libgl=1.7.0
+  - libglib=2.80.3
+  - libglu=9.0.0
+  - libglvnd=1.7.0
+  - libglx=1.7.0
+  - libgomp=14.1.0
+  - libhwloc=2.11.1
+  - libiconv=1.17
+  - libidn2=2.3.7
+  - libjpeg-turbo=3.0.0
+  - liblapack=3.9.0
+  - liblapacke=3.9.0
+  - libllvm18=18.1.8
+  - libllvm19=19.1.0
+  - libnghttp2=1.58.0
+  - libnpp=12.0.2.50
+  - libnsl=2.0.1
+  - libnvjitlink=12.1.105
+  - libnvjpeg=12.1.1.14
+  - libopencv=4.10.0
+  - libopenvino=2024.4.0
+  - libopenvino-auto-batch-plugin=2024.4.0
+  - libopenvino-auto-plugin=2024.4.0
+  - libopenvino-hetero-plugin=2024.4.0
+  - libopenvino-intel-cpu-plugin=2024.4.0
+  - libopenvino-intel-gpu-plugin=2024.4.0
+  - libopenvino-intel-npu-plugin=2024.4.0
+  - libopenvino-ir-frontend=2024.4.0
+  - libopenvino-onnx-frontend=2024.4.0
+  - libopenvino-paddle-frontend=2024.4.0
+  - libopenvino-pytorch-frontend=2024.4.0
+  - libopenvino-tensorflow-frontend=2024.4.0
+  - libopenvino-tensorflow-lite-frontend=2024.4.0
+  - libopus=1.3.1
+  - libpciaccess=0.18
+  - libpng=1.6.44
+  - libpq=16.4
+  - libprotobuf=4.25.3
+  - libsqlite=3.46.0
+  - libssh2=1.11.0
+  - libstdcxx=14.1.0
+  - libstdcxx-ng=14.1.0
+  - libtasn1=4.19.0
+  - libtiff=4.7.0
+  - libunistring=0.9.10
+  - libuuid=2.38.1
+  - libva=2.22.0
+  - libvpx=1.14.1
+  - libwebp-base=1.4.0
+  - libxcb=1.16
+  - libxcrypt=4.4.36
+  - libxkbcommon=1.7.0
+  - libxml2=2.12.7
+  - libzlib=1.3.1
+  - llvm-openmp=15.0.7
+  - markupsafe=2.1.5
+  - matplotlib-inline=0.1.7
+  - mccabe=0.6.1
+  - mkl=2022.1.0
+  - mkl-devel=2022.1.0
+  - mkl-include=2022.1.0
+  - mpc=1.3.1
+  - mpfr=4.2.1
+  - mpmath=1.3.0
+  - mypy_extensions=1.0.0
+  - mysql-common=9.0.1
+  - mysql-libs=9.0.1
+  - ncurses=6.5
+  - nettle=3.9.1
+  - networkx=3.3
+  - nodeenv=1.9.1
+  - numpy=2.0.0
+  - ocl-icd=2.3.2
+  - opencv=4.10.0
+  - openexr=3.2.2
+  - openh264=2.4.1
+  - openjpeg=2.5.2
+  - openssl=3.3.2
+  - p11-kit=0.24.1
+  - parso=0.8.4
+  - pathspec=0.12.1
+  - pcre2=10.44
+  - pexpect=4.9.0
+  - pickleshare=0.7.5
+  - pillow=10.4.0
+  - pip=24.0
+  - pixman=0.43.2
+  - platformdirs=4.2.2
+  - pre-commit=3.7.1
+  - prompt-toolkit=3.0.47
+  - pthread-stubs=0.4
+  - ptyprocess=0.7.0
+  - pugixml=1.14
+  - pure_eval=0.2.2
+  - py-opencv=4.10.0
+  - pycodestyle=2.8.0
+  - pycparser=2.22
+  - pyflakes=2.4.0
+  - pygments=2.18.0
+  - pysocks=1.7.1
+  - python=3.10.14
+  - python_abi=3.10
+  - pytorch=2.3.1
+  - pytorch-cuda=12.1
+  - pytorch-mutex=1.0
+  - pyyaml=6.0.1
+  - qt6-main=6.7.2
+  - readline=8.2
+  - requests=2.32.3
+  - scikit-learn=1.5.1
+  - scipy=1.14.0
+  - setuptools=71.0.4
+  - six=1.16.0
+  - snappy=1.2.1
+  - stack_data=0.6.2
+  - svt-av1=2.2.1
+  - sympy=1.13.0
+  - tbb=2021.13.0
+  - threadpoolctl=3.5.0
+  - tk=8.6.13
+  - toml=0.10.2
+  - tomli=2.0.1
+  - torchaudio=2.3.1
+  - torchtriton=2.3.1
+  - torchvision=0.18.1
+  - tqdm=4.66.5
+  - traitlets=5.14.3
+  - typed-ast=1.5.5
+  - typing_extensions=4.12.2
+  - tzdata=2024a
+  - ukkonen=1.0.1
+  - urllib3=2.2.2
+  - virtualenv=20.26.3
+  - wayland=1.23.0
+  - wayland-protocols=1.36
+  - wcwidth=0.2.13
+  - wheel=0.43.0
+  - x264=1!164.3095
+  - x265=3.5
+  - xcb-util=0.4.1
+  - xcb-util-cursor=0.1.5
+  - xcb-util-image=0.4.0
+  - xcb-util-keysyms=0.4.1
+  - xcb-util-renderutil=0.3.10
+  - xcb-util-wm=0.4.2
+  - xkeyboard-config=2.42
+  - xorg-fixesproto=5.0
+  - xorg-inputproto=2.3.2
+  - xorg-kbproto=1.0.7
+  - xorg-libice=1.1.1
+  - xorg-libsm=1.2.4
+  - xorg-libx11=1.8.9
+  - xorg-libxau=1.0.11
+  - xorg-libxdmcp=1.1.3
+  - xorg-libxext=1.3.4
+  - xorg-libxfixes=5.0.3
+  - xorg-libxi=1.7.10
+  - xorg-libxrender=0.9.11
+  - xorg-libxtst=1.2.5
+  - xorg-libxxf86vm=1.1.5
+  - xorg-recordproto=1.14.2
+  - xorg-renderproto=0.11.1
+  - xorg-xextproto=7.3.0
+  - xorg-xproto=7.0.31
+  - xz=5.2.6
+  - yaml=0.2.5
+  - zipp=3.20.2
+  - zlib=1.3.1
+  - zstandard=0.23.0
+  - zstd=1.5.6

hub/__init__.py

@@ -0,0 +1,4 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the Apache License, Version 2.0
+# found in the LICENSE file in the root directory of this source tree.

hub/backbones.py

@@ -0,0 +1,201 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the Apache License, Version 2.0
+# found in the LICENSE file in the root directory of this source tree.
+
+from enum import Enum
+from typing import Union
+
+import torch
+
+from .utils import _DINOV2_BASE_URL, _make_dinov2_model_name
+
+
+class Weights(Enum):
+    LVD142M = 'LVD142M'
+
+
+def _make_dinov2_model(
+    *,
+    arch_name: str = 'vit_large',
+    img_size: int = 518,
+    patch_size: int = 14,
+    init_values: float = 1.0,
+    ffn_layer: str = 'mlp',
+    block_chunks: int = 0,
+    num_register_tokens: int = 0,
+    interpolate_antialias: bool = False,
+    interpolate_offset: float = 0.1,
+    pretrained: bool = True,
+    weights: Union[Weights, str] = Weights.LVD142M,
+    **kwargs,
+):
+    from ..models.dinov2 import vision_transformer as vits
+
+    if isinstance(weights, str):
+        try:
+            weights = Weights[weights]
+        except KeyError:
+            raise AssertionError(f'Unsupported weights: {weights}')
+
+    model_base_name = _make_dinov2_model_name(arch_name, patch_size)
+    vit_kwargs = dict(
+        img_size=img_size,
+        patch_size=patch_size,
+        init_values=init_values,
+        ffn_layer=ffn_layer,
+        block_chunks=block_chunks,
+        num_register_tokens=num_register_tokens,
+        interpolate_antialias=interpolate_antialias,
+        interpolate_offset=interpolate_offset,
+    )
+    vit_kwargs.update(**kwargs)
+    model = vits.__dict__[arch_name](**vit_kwargs)
+
+    if pretrained:
+        model_full_name = _make_dinov2_model_name(
+            arch_name, patch_size, num_register_tokens
+        )
+        url = (
+            _DINOV2_BASE_URL
+            + f'/{model_base_name}/{model_full_name}_pretrain.pth'
+        )
+        state_dict = torch.hub.load_state_dict_from_url(
+            url, map_location='cpu'
+        )
+        model.load_state_dict(state_dict, strict=True)
+
+    return model
+
+
+def dinov2_vits14(
+    *,
+    pretrained: bool = True,
+    weights: Union[Weights, str] = Weights.LVD142M,
+    **kwargs,
+):
+    """DINOv2 ViT-S/14 model (optionally) pretrained on the LVD-142M
+    dataset."""
+    return _make_dinov2_model(
+        arch_name='vit_small', pretrained=pretrained, weights=weights, **kwargs
+    )
+
+
+def dinov2_vitb14(
+    *,
+    pretrained: bool = True,
+    weights: Union[Weights, str] = Weights.LVD142M,
+    **kwargs,
+):
+    """DINOv2 ViT-B/14 model (optionally) pretrained on the LVD-142M
+    dataset."""
+    return _make_dinov2_model(
+        arch_name='vit_base', pretrained=pretrained, weights=weights, **kwargs
+    )
+
+
+def dinov2_vitl14(
+    *,
+    pretrained: bool = True,
+    weights: Union[Weights, str] = Weights.LVD142M,
+    **kwargs,
+):
+    """DINOv2 ViT-L/14 model (optionally) pretrained on the LVD-142M
+    dataset."""
+    return _make_dinov2_model(
+        arch_name='vit_large', pretrained=pretrained, weights=weights, **kwargs
+    )
+
+
+def dinov2_vitg14(
+    *,
+    pretrained: bool = True,
+    weights: Union[Weights, str] = Weights.LVD142M,
+    **kwargs,
+):
+    """DINOv2 ViT-g/14 model (optionally) pretrained on the LVD-142M
+    dataset."""
+    return _make_dinov2_model(
+        arch_name='vit_giant2',
+        ffn_layer='swiglufused',
+        weights=weights,
+        pretrained=pretrained,
+        **kwargs,
+    )
+
+
+def dinov2_vits14_reg(
+    *,
+    pretrained: bool = True,
+    weights: Union[Weights, str] = Weights.LVD142M,
+    **kwargs,
+):
+    """DINOv2 ViT-S/14 model with registers (optionally) pretrained on the
+    LVD-142M dataset."""
+    return _make_dinov2_model(
+        arch_name='vit_small',
+        pretrained=pretrained,
+        weights=weights,
+        num_register_tokens=4,
+        interpolate_antialias=True,
+        interpolate_offset=0.0,
+        **kwargs,
+    )
+
+
+def dinov2_vitb14_reg(
+    *,
+    pretrained: bool = True,
+    weights: Union[Weights, str] = Weights.LVD142M,
+    **kwargs,
+):
+    """DINOv2 ViT-B/14 model with registers (optionally) pretrained on the
+    LVD-142M dataset."""
+    return _make_dinov2_model(
+        arch_name='vit_base',
+        pretrained=pretrained,
+        weights=weights,
+        num_register_tokens=4,
+        interpolate_antialias=True,
+        interpolate_offset=0.0,
+        **kwargs,
+    )
+
+
+def dinov2_vitl14_reg(
+    *,
+    pretrained: bool = True,
+    weights: Union[Weights, str] = Weights.LVD142M,
+    **kwargs,
+):
+    """DINOv2 ViT-L/14 model with registers (optionally) pretrained on the
+    LVD-142M dataset."""
+    return _make_dinov2_model(
+        arch_name='vit_large',
+        pretrained=pretrained,
+        weights=weights,
+        num_register_tokens=4,
+        interpolate_antialias=True,
+        interpolate_offset=0.0,
+        **kwargs,
+    )
+
+
+def dinov2_vitg14_reg(
+    *,
+    pretrained: bool = True,
+    weights: Union[Weights, str] = Weights.LVD142M,
+    **kwargs,
+):
+    """DINOv2 ViT-g/14 model with registers (optionally) pretrained on the
+    LVD-142M dataset."""
+    return _make_dinov2_model(
+        arch_name='vit_giant2',
+        ffn_layer='swiglufused',
+        weights=weights,
+        pretrained=pretrained,
+        num_register_tokens=4,
+        interpolate_antialias=True,
+        interpolate_offset=0.0,
+        **kwargs,
+    )

hub/utils.py

@@ -0,0 +1,46 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the Apache License, Version 2.0
+# found in the LICENSE file in the root directory of this source tree.
+
+import itertools
+import math
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+_DINOV2_BASE_URL = 'https://dl.fbaipublicfiles.com/dinov2'
+
+
+def _make_dinov2_model_name(
+    arch_name: str, patch_size: int, num_register_tokens: int = 0
+) -> str:
+    compact_arch_name = arch_name.replace('_', '')[:4]
+    registers_suffix = (
+        f'_reg{num_register_tokens}' if num_register_tokens else ''
+    )
+    return f'dinov2_{compact_arch_name}{patch_size}{registers_suffix}'
+
+
+class CenterPadding(nn.Module):
+    def __init__(self, multiple):
+        super().__init__()
+        self.multiple = multiple
+
+    def _get_pad(self, size):
+        new_size = math.ceil(size / self.multiple) * self.multiple
+        pad_size = new_size - size
+        pad_size_left = pad_size // 2
+        pad_size_right = pad_size - pad_size_left
+        return pad_size_left, pad_size_right
+
+    @torch.inference_mode()
+    def forward(self, x):
+        pads = list(
+            itertools.chain.from_iterable(
+                self._get_pad(m) for m in x.shape[:1:-1]
+            )
+        )
+        output = F.pad(x, pads)
+        return output

hubconf.py

@@ -0,0 +1,29 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the Apache License, Version 2.0
+# found in the LICENSE file in the root directory of this source tree.
+
+
+from sinder.hub.backbones import (
+    dinov2_vitb14,
+    dinov2_vitb14_reg,
+    dinov2_vitg14,
+    dinov2_vitg14_reg,
+    dinov2_vitl14,
+    dinov2_vitl14_reg,
+    dinov2_vits14,
+    dinov2_vits14_reg,
+)
+
+dependencies = ['torch']
+
+__all__ = [
+    'dinov2_vitb14',
+    'dinov2_vitb14_reg',
+    'dinov2_vitg14',
+    'dinov2_vitg14_reg',
+    'dinov2_vitl14',
+    'dinov2_vitl14_reg',
+    'dinov2_vits14',
+    'dinov2_vits14_reg',
+]

main.py

@@ -0,0 +1,238 @@
+#!/usr/bin/env python
+import argparse
+import os
+import sys
+from pathlib import Path
+
+import numpy as np
+import torch
+from PIL import Image
+from tqdm import tqdm
+
+from sinder import (
+    get_neighbor_loss,
+    get_tokens,
+    load_data,
+    load_model,
+    load_visual_data,
+    pca_array,
+    replace_back,
+    replace_linear_addition_noqk,
+)
+
+os.environ['XFORMERS_DISABLED'] = '1'
+torch.set_float32_matmul_precision('high')
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description='Beautify network')
+    parser.add_argument(
+        '--model', type=str, default='dinov2_vitg14', help='config file'
+    )
+    parser.add_argument('--work_dir', type=str, default='results')
+    parser.add_argument('--resolution', type=int, default=518)
+    parser.add_argument('--lr', type=float, default=0.005)
+    parser.add_argument('--max_iter', type=int, default=30000)
+    parser.add_argument('--num_train_max', type=int, default=30000)
+    parser.add_argument('--mask_thr', type=float, default=4)
+    parser.add_argument('--skip_less_than', type=int, default=3)
+    parser.add_argument('--visual_size', type=int, default=448 * 2)
+    parser.add_argument('--kernel', type=int, default=3)
+    parser.add_argument('--save_at_skip', type=int, nargs='+', default=[75])
+    parser.add_argument('--limit_layers', type=int, default=10)
+
+    args = parser.parse_args()
+    return args
+
+
+def prepare_train(args, model):
+    model.train()
+
+    all_params = []
+    for name, param in model.named_parameters():
+        param.requires_grad = False
+
+    replace_linear_addition_noqk(model, 'model')
+    for name, param in model.named_parameters():
+        if '.epsilon' in name and param.requires_grad is True:
+            all_params.append(param)
+
+    grad_params = []
+    for name, param in model.named_parameters():
+        if param.requires_grad:
+            grad_params.append(name)
+
+    assert len(grad_params) == len(all_params)
+    print(len(grad_params), grad_params)
+    print(len(all_params), all_params)
+    optimizer = torch.optim.SGD(
+        all_params,
+        lr=args.lr,
+        momentum=0.9,
+    )
+
+    return optimizer
+
+
+def save_model(args, model):
+    print('save model')
+    model.eval()
+
+    replace_back(model, 'model')
+
+    torch.save(model.state_dict(), args.folder / 'model.pt')
+
+
+def train(args, model, dataset, optimizer, visual_dataset):
+    print('training')
+    skip_history = [False] * 1000
+    model.train()
+
+    for global_iter in tqdm(range(args.max_iter)):
+        img = dataset[global_iter % len(dataset)]
+        H = img.shape[1] // model.patch_size
+        W = img.shape[2] // model.patch_size
+        density = np.array(skip_history[-1000:]).astype(float).mean()
+        print(f'{global_iter=} {W=} {H=} {density=:.2f}')
+
+        for percent in args.save_at_skip:
+            if percent / 100 <= density:
+                print(f'save checkpoint at {density=}')
+                args.save_at_skip.remove(percent)
+                torch.save(model, args.folder / f'checkpoint_p{percent}.pth')
+        if len(args.save_at_skip) == 0:
+            break
+
+        model.zero_grad()
+
+        model.train()
+        with torch.enable_grad():
+            image_batch = img.unsqueeze(0).cuda()
+            result = get_neighbor_loss(
+                model,
+                image_batch,
+                skip_less_than=args.skip_less_than,
+                mask_thr=args.mask_thr,
+                kernel=args.kernel,
+            )
+
+        if result is None:
+            skip_history.append(True)
+            print('no loss, skip')
+        else:
+            skip_history.append(False)
+            (
+                layer,
+                loss,
+                I,
+                J,
+                T,
+                alpha,
+                mask_angle,
+                x_token,
+            ) = result
+
+            print(
+                f'{global_iter=}, {layer=}, {density=}, {alpha=:.2f}, {len(I)=}, '
+                f'{loss.item()=:.2f}'
+            )
+
+            if torch.isnan(loss).any():
+                print('nan loss, skip')
+                continue
+            loss.backward()
+
+            # set some grad to 0
+            if args.limit_layers:
+                with torch.no_grad():
+                    for t in range(layer - args.limit_layers + 1):
+                        for p in model.blocks[t].parameters():
+                            p.grad = None
+
+            has_nan = False
+            for name, param in model.named_parameters():
+                if param.grad is not None and torch.isnan(param.grad).any():
+                    print(f'nan grad at {name}, skip')
+                    has_nan = True
+            if has_nan:
+                continue
+            optimizer.step()
+
+            # visualize
+            if global_iter % 100 == 0:
+                try:
+                    print(f'visualization at {global_iter=}')
+                    pca_img = pca_array(x_token)
+                    pca_img.save(args.folder / 'pca.png')
+                    mask_img = Image.fromarray(
+                        (mask_angle * 255)
+                        .detach()
+                        .cpu()
+                        .numpy()
+                        .astype(np.uint8)
+                    ).resize((W * 7, H * 7), resample=Image.NEAREST)
+                    mask_img.save(args.folder / 'mask.png')
+                    Image.fromarray(
+                        (
+                            (
+                                img.permute((1, 2, 0)).cpu().numpy() * 0.22
+                                + 0.45
+                            )
+                            * 255
+                        )
+                        .clip(0, 255)
+                        .astype(np.uint8)
+                    ).save(args.folder / 'img.png')
+                    if global_iter % 1000 == 0:
+                        pca_img.save(args.folder / f'{global_iter:05}_pca.png')
+                        mask_img.save(
+                            args.folder / f'{global_iter:05}_mask.png'
+                        )
+                        Image.fromarray(
+                            (
+                                (
+                                    img.permute((1, 2, 0)).cpu().numpy() * 0.22
+                                    + 0.45
+                                )
+                                * 255
+                            )
+                            .clip(0, 255)
+                            .astype(np.uint8)
+                        ).save(args.folder / f'{global_iter:05}_img.png')
+                except Exception as e:
+                    print(e)
+
+                for d in range(len(visual_dataset)):
+                    visual_image = visual_dataset[d]
+                    visual_tokens_all = get_tokens(model, visual_image)
+                    visual_tokens, visual_tokens_cls = zip(*visual_tokens_all)
+                    pca_img = pca_array(visual_tokens[-1])
+                    pca_img.save(args.folder / f'{d}_pca.png')
+                    if global_iter % 500 == 0:
+                        pca_img.save(
+                            args.folder / f'{global_iter:05}_{d}_pca.png'
+                        )
+
+    torch.save(model, args.folder / 'checkpoint.pth')
+
+
+def main():
+    print('Start beautify')
+    args = parse_args()
+
+    name = f'res{args.resolution}_lr{args.lr}_{args.num_train_max}_skipless{args.skip_less_than}_maskthr{args.mask_thr}_limit{args.limit_layers}_ker{args.kernel}'
+    args.folder = Path(args.work_dir) / name
+    os.makedirs(args.folder, exist_ok=True)
+    print(args)
+    print(' '.join(sys.argv))
+    print(f'work dir {args.folder}')
+    model = load_model(args.model)
+    dataset = load_data(args, model)
+    visual_dataset = load_visual_data(args, model)
+    optimizer = prepare_train(args, model)
+    train(args, model, dataset, optimizer, visual_dataset)
+    save_model(args, model)
+
+
+if __name__ == '__main__':
+    main()

models/dinov2/__init__.py

@@ -0,0 +1,46 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the Apache License, Version 2.0
+# found in the LICENSE file in the root directory of this source tree.
+
+import logging
+
+from . import vision_transformer as vits
+
+logger = logging.getLogger('dinov2')
+
+
+def build_model(args, only_teacher=False, img_size=224):
+    args.arch = args.arch.removesuffix('_memeff')
+    if 'vit' in args.arch:
+        vit_kwargs = dict(
+            img_size=img_size,
+            patch_size=args.patch_size,
+            init_values=args.layerscale,
+            ffn_layer=args.ffn_layer,
+            block_chunks=args.block_chunks,
+            qkv_bias=args.qkv_bias,
+            proj_bias=args.proj_bias,
+            ffn_bias=args.ffn_bias,
+            num_register_tokens=args.num_register_tokens,
+            interpolate_offset=args.interpolate_offset,
+            interpolate_antialias=args.interpolate_antialias,
+        )
+        teacher = vits.__dict__[args.arch](**vit_kwargs)
+        if only_teacher:
+            return teacher, teacher.embed_dim
+        student = vits.__dict__[args.arch](
+            **vit_kwargs,
+            drop_path_rate=args.drop_path_rate,
+            drop_path_uniform=args.drop_path_uniform,
+        )
+        embed_dim = student.embed_dim
+    return student, teacher, embed_dim
+
+
+def build_model_from_cfg(cfg, only_teacher=False):
+    return build_model(
+        cfg.student,
+        only_teacher=only_teacher,
+        img_size=cfg.crops.global_crops_size,
+    )

models/dinov2/layers/__init__.py

@@ -0,0 +1,11 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the Apache License, Version 2.0
+# found in the LICENSE file in the root directory of this source tree.
+
+from .attention import MemEffAttention
+from .block import NestedTensorBlock
+from .dino_head import DINOHead
+from .mlp import Mlp
+from .patch_embed import PatchEmbed
+from .swiglu_ffn import SwiGLUFFN, SwiGLUFFNFused

models/dinov2/layers/attention.py

@@ -0,0 +1,91 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the Apache License, Version 2.0
+# found in the LICENSE file in the root directory of this source tree.
+
+# References:
+#   https://github.com/facebookresearch/dino/blob/master/vision_transformer.py
+#   https://github.com/rwightman/pytorch-image-models/tree/master/timm/models/vision_transformer.py
+
+import logging
+import os
+import warnings
+
+from torch import Tensor, nn
+
+logger = logging.getLogger('dinov2')
+
+
+XFORMERS_ENABLED = os.environ.get('XFORMERS_DISABLED') is None
+try:
+    if XFORMERS_ENABLED:
+        from xformers.ops import memory_efficient_attention, unbind
+
+        XFORMERS_AVAILABLE = True
+        warnings.warn('xFormers is available (Attention)')
+    else:
+        raise ImportError
+except ImportError:
+    XFORMERS_AVAILABLE = False
+
+
+class Attention(nn.Module):
+    def __init__(
+        self,
+        dim: int,
+        num_heads: int = 8,
+        qkv_bias: bool = False,
+        proj_bias: bool = True,
+        attn_drop: float = 0.0,
+        proj_drop: float = 0.0,
+    ) -> None:
+        super().__init__()
+        self.num_heads = num_heads
+        head_dim = dim // num_heads
+        self.scale = head_dim**-0.5
+
+        self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
+        self.attn_drop = nn.Dropout(attn_drop)
+        self.proj = nn.Linear(dim, dim, bias=proj_bias)
+        self.proj_drop = nn.Dropout(proj_drop)
+
+    def forward(self, x: Tensor) -> Tensor:
+        B, N, C = x.shape
+        qkv = (
+            self.qkv(x)
+            .reshape(B, N, 3, self.num_heads, C // self.num_heads)
+            .permute(2, 0, 3, 1, 4)
+        )
+
+        q, k, v = qkv[0] * self.scale, qkv[1], qkv[2]
+        attn = q @ k.transpose(-2, -1)
+
+        attn = attn.softmax(dim=-1)
+        attn = self.attn_drop(attn)
+
+        x = (attn @ v).transpose(1, 2).reshape(B, N, C)
+        x = self.proj(x)
+        x = self.proj_drop(x)
+        return x
+
+
+class MemEffAttention(Attention):
+    def forward(self, x: Tensor, attn_bias=None) -> Tensor:
+        if not XFORMERS_AVAILABLE:
+            if attn_bias is not None:
+                raise AssertionError(
+                    'xFormers is required for using nested tensors'
+                )
+            return super().forward(x)
+
+        B, N, C = x.shape
+        qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads)
+
+        q, k, v = unbind(qkv, 2)
+
+        x = memory_efficient_attention(q, k, v, attn_bias=attn_bias)
+        x = x.reshape([B, N, C])
+
+        x = self.proj(x)
+        x = self.proj_drop(x)
+        return x

models/dinov2/layers/block.py

@@ -0,0 +1,304 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the Apache License, Version 2.0
+# found in the LICENSE file in the root directory of this source tree.
+
+# References:
+#   https://github.com/facebookresearch/dino/blob/master/vision_transformer.py
+#   https://github.com/rwightman/pytorch-image-models/tree/master/timm/layers/patch_embed.py
+
+import logging
+import os
+import warnings
+from typing import Any, Callable, Dict, List, Tuple
+
+import torch
+from torch import Tensor, nn
+
+from .attention import Attention, MemEffAttention
+from .drop_path import DropPath
+from .layer_scale import LayerScale
+from .mlp import Mlp
+
+logger = logging.getLogger('dinov2')
+
+
+XFORMERS_ENABLED = os.environ.get('XFORMERS_DISABLED') is None
+try:
+    if XFORMERS_ENABLED:
+        from xformers.ops import fmha, index_select_cat, scaled_index_add
+
+        XFORMERS_AVAILABLE = True
+        warnings.warn('xFormers is available (Block)')
+    else:
+        raise ImportError
+except ImportError:
+    XFORMERS_AVAILABLE = False
+
+
+class Block(nn.Module):
+    def __init__(
+        self,
+        dim: int,
+        num_heads: int,
+        mlp_ratio: float = 4.0,
+        qkv_bias: bool = False,
+        proj_bias: bool = True,
+        ffn_bias: bool = True,
+        drop: float = 0.0,
+        attn_drop: float = 0.0,
+        init_values=None,
+        drop_path: float = 0.0,
+        act_layer: Callable[..., nn.Module] = nn.GELU,
+        norm_layer: Callable[..., nn.Module] = nn.LayerNorm,
+        attn_class: Callable[..., nn.Module] = Attention,
+        ffn_layer: Callable[..., nn.Module] = Mlp,
+    ) -> None:
+        super().__init__()
+        # print(f"biases: qkv: {qkv_bias}, proj: {proj_bias}, ffn: {ffn_bias}")
+        self.norm1 = norm_layer(dim)
+        self.attn = attn_class(
+            dim,
+            num_heads=num_heads,
+            qkv_bias=qkv_bias,
+            proj_bias=proj_bias,
+            attn_drop=attn_drop,
+            proj_drop=drop,
+        )
+        self.ls1 = (
+            LayerScale(dim, init_values=init_values)
+            if init_values
+            else nn.Identity()
+        )
+        self.drop_path1 = (
+            DropPath(drop_path) if drop_path > 0.0 else nn.Identity()
+        )
+
+        self.norm2 = norm_layer(dim)
+        mlp_hidden_dim = int(dim * mlp_ratio)
+        self.mlp = ffn_layer(
+            in_features=dim,
+            hidden_features=mlp_hidden_dim,
+            act_layer=act_layer,
+            drop=drop,
+            bias=ffn_bias,
+        )
+        self.ls2 = (
+            LayerScale(dim, init_values=init_values)
+            if init_values
+            else nn.Identity()
+        )
+        self.drop_path2 = (
+            DropPath(drop_path) if drop_path > 0.0 else nn.Identity()
+        )
+
+        self.sample_drop_ratio = drop_path
+
+    def forward(self, x: Tensor) -> Tensor:
+        def attn_residual_func(x: Tensor) -> Tensor:
+            return self.ls1(self.attn(self.norm1(x)))
+
+        def ffn_residual_func(x: Tensor) -> Tensor:
+            return self.ls2(self.mlp(self.norm2(x)))
+
+        if self.training and self.sample_drop_ratio > 0.1:
+            # the overhead is compensated only for a drop path rate larger than 0.1
+            x = drop_add_residual_stochastic_depth(
+                x,
+                residual_func=attn_residual_func,
+                sample_drop_ratio=self.sample_drop_ratio,
+            )
+            x = drop_add_residual_stochastic_depth(
+                x,
+                residual_func=ffn_residual_func,
+                sample_drop_ratio=self.sample_drop_ratio,
+            )
+        elif self.training and self.sample_drop_ratio > 0.0:
+            x = x + self.drop_path1(attn_residual_func(x))
+            x = x + self.drop_path1(ffn_residual_func(x))  # FIXME: drop_path2
+        else:
+            x = x + attn_residual_func(x)
+            x = x + ffn_residual_func(x)
+        return x
+
+
+def drop_add_residual_stochastic_depth(
+    x: Tensor,
+    residual_func: Callable[[Tensor], Tensor],
+    sample_drop_ratio: float = 0.0,
+) -> Tensor:
+    # 1) extract subset using permutation
+    b, n, d = x.shape
+    sample_subset_size = max(int(b * (1 - sample_drop_ratio)), 1)
+    brange = (torch.randperm(b, device=x.device))[:sample_subset_size]
+    x_subset = x[brange]
+
+    # 2) apply residual_func to get residual
+    residual = residual_func(x_subset)
+
+    x_flat = x.flatten(1)
+    residual = residual.flatten(1)
+
+    residual_scale_factor = b / sample_subset_size
+
+    # 3) add the residual
+    x_plus_residual = torch.index_add(
+        x_flat,
+        0,
+        brange,
+        residual.to(dtype=x.dtype),
+        alpha=residual_scale_factor,
+    )
+    return x_plus_residual.view_as(x)
+
+
+def get_branges_scales(x, sample_drop_ratio=0.0):
+    b, n, d = x.shape
+    sample_subset_size = max(int(b * (1 - sample_drop_ratio)), 1)
+    brange = (torch.randperm(b, device=x.device))[:sample_subset_size]
+    residual_scale_factor = b / sample_subset_size
+    return brange, residual_scale_factor
+
+
+def add_residual(
+    x, brange, residual, residual_scale_factor, scaling_vector=None
+):
+    if scaling_vector is None:
+        x_flat = x.flatten(1)
+        residual = residual.flatten(1)
+        x_plus_residual = torch.index_add(
+            x_flat,
+            0,
+            brange,
+            residual.to(dtype=x.dtype),
+            alpha=residual_scale_factor,
+        )
+    else:
+        x_plus_residual = scaled_index_add(
+            x,
+            brange,
+            residual.to(dtype=x.dtype),
+            scaling=scaling_vector,
+            alpha=residual_scale_factor,
+        )
+    return x_plus_residual
+
+
+attn_bias_cache: Dict[Tuple, Any] = {}
+
+
+def get_attn_bias_and_cat(x_list, branges=None):
+    """This will perform the index select, cat the tensors, and provide the
+    attn_bias from cache."""
+    batch_sizes = (
+        [b.shape[0] for b in branges]
+        if branges is not None
+        else [x.shape[0] for x in x_list]
+    )
+    all_shapes = tuple((b, x.shape[1]) for b, x in zip(batch_sizes, x_list))
+    if all_shapes not in attn_bias_cache.keys():
+        seqlens = []
+        for b, x in zip(batch_sizes, x_list):
+            for _ in range(b):
+                seqlens.append(x.shape[1])
+        attn_bias = fmha.BlockDiagonalMask.from_seqlens(seqlens)
+        attn_bias._batch_sizes = batch_sizes
+        attn_bias_cache[all_shapes] = attn_bias
+
+    if branges is not None:
+        cat_tensors = index_select_cat(
+            [x.flatten(1) for x in x_list], branges
+        ).view(1, -1, x_list[0].shape[-1])
+    else:
+        tensors_bs1 = tuple(x.reshape([1, -1, *x.shape[2:]]) for x in x_list)
+        cat_tensors = torch.cat(tensors_bs1, dim=1)
+
+    return attn_bias_cache[all_shapes], cat_tensors
+
+
+def drop_add_residual_stochastic_depth_list(
+    x_list: List[Tensor],
+    residual_func: Callable[[Tensor, Any], Tensor],
+    sample_drop_ratio: float = 0.0,
+    scaling_vector=None,
+) -> Tensor:
+    # 1) generate random set of indices for dropping samples in the batch
+    branges_scales = [
+        get_branges_scales(x, sample_drop_ratio=sample_drop_ratio)
+        for x in x_list
+    ]
+    branges = [s[0] for s in branges_scales]
+    residual_scale_factors = [s[1] for s in branges_scales]
+
+    # 2) get attention bias and index+concat the tensors
+    attn_bias, x_cat = get_attn_bias_and_cat(x_list, branges)
+
+    # 3) apply residual_func to get residual, and split the result
+    residual_list = attn_bias.split(residual_func(x_cat, attn_bias=attn_bias))  # type: ignore
+
+    outputs = []
+    for x, brange, residual, residual_scale_factor in zip(
+        x_list, branges, residual_list, residual_scale_factors
+    ):
+        outputs.append(
+            add_residual(
+                x, brange, residual, residual_scale_factor, scaling_vector
+            ).view_as(x)
+        )
+    return outputs
+
+
+class NestedTensorBlock(Block):
+    def forward_nested(self, x_list: List[Tensor]) -> List[Tensor]:
+        """x_list contains a list of tensors to nest together and run."""
+        assert isinstance(self.attn, MemEffAttention)
+
+        if self.training and self.sample_drop_ratio > 0.0:
+
+            def attn_residual_func(x: Tensor, attn_bias=None) -> Tensor:
+                return self.attn(self.norm1(x), attn_bias=attn_bias)
+
+            def ffn_residual_func(x: Tensor, attn_bias=None) -> Tensor:
+                return self.mlp(self.norm2(x))
+
+            x_list = drop_add_residual_stochastic_depth_list(
+                x_list,
+                residual_func=attn_residual_func,
+                sample_drop_ratio=self.sample_drop_ratio,
+                scaling_vector=self.ls1.gamma
+                if isinstance(self.ls1, LayerScale)
+                else None,
+            )
+            x_list = drop_add_residual_stochastic_depth_list(
+                x_list,
+                residual_func=ffn_residual_func,
+                sample_drop_ratio=self.sample_drop_ratio,
+                scaling_vector=self.ls2.gamma
+                if isinstance(self.ls1, LayerScale)
+                else None,
+            )
+            return x_list
+        else:
+
+            def attn_residual_func(x: Tensor, attn_bias=None) -> Tensor:
+                return self.ls1(self.attn(self.norm1(x), attn_bias=attn_bias))
+
+            def ffn_residual_func(x: Tensor, attn_bias=None) -> Tensor:
+                return self.ls2(self.mlp(self.norm2(x)))
+
+            attn_bias, x = get_attn_bias_and_cat(x_list)
+            x = x + attn_residual_func(x, attn_bias=attn_bias)
+            x = x + ffn_residual_func(x)
+            return attn_bias.split(x)
+
+    def forward(self, x_or_x_list):
+        if isinstance(x_or_x_list, Tensor):
+            return super().forward(x_or_x_list)
+        elif isinstance(x_or_x_list, list):
+            if not XFORMERS_AVAILABLE:
+                raise AssertionError(
+                    'xFormers is required for using nested tensors'
+                )
+            return self.forward_nested(x_or_x_list)
+        else:
+            raise AssertionError

models/dinov2/layers/dino_head.py

@@ -0,0 +1,69 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the Apache License, Version 2.0
+# found in the LICENSE file in the root directory of this source tree.
+
+import torch
+import torch.nn as nn
+from torch.nn.init import trunc_normal_
+from torch.nn.utils import weight_norm
+
+
+class DINOHead(nn.Module):
+    def __init__(
+        self,
+        in_dim,
+        out_dim,
+        use_bn=False,
+        nlayers=3,
+        hidden_dim=2048,
+        bottleneck_dim=256,
+        mlp_bias=True,
+    ):
+        super().__init__()
+        nlayers = max(nlayers, 1)
+        self.mlp = _build_mlp(
+            nlayers,
+            in_dim,
+            bottleneck_dim,
+            hidden_dim=hidden_dim,
+            use_bn=use_bn,
+            bias=mlp_bias,
+        )
+        self.apply(self._init_weights)
+        self.last_layer = weight_norm(
+            nn.Linear(bottleneck_dim, out_dim, bias=False)
+        )
+        self.last_layer.weight_g.data.fill_(1)
+
+    def _init_weights(self, m):
+        if isinstance(m, nn.Linear):
+            trunc_normal_(m.weight, std=0.02)
+            if isinstance(m, nn.Linear) and m.bias is not None:
+                nn.init.constant_(m.bias, 0)
+
+    def forward(self, x):
+        x = self.mlp(x)
+        eps = 1e-6 if x.dtype == torch.float16 else 1e-12
+        x = nn.functional.normalize(x, dim=-1, p=2, eps=eps)
+        x = self.last_layer(x)
+        return x
+
+
+def _build_mlp(
+    nlayers, in_dim, bottleneck_dim, hidden_dim=None, use_bn=False, bias=True
+):
+    if nlayers == 1:
+        return nn.Linear(in_dim, bottleneck_dim, bias=bias)
+    else:
+        layers = [nn.Linear(in_dim, hidden_dim, bias=bias)]
+        if use_bn:
+            layers.append(nn.BatchNorm1d(hidden_dim))
+        layers.append(nn.GELU())
+        for _ in range(nlayers - 2):
+            layers.append(nn.Linear(hidden_dim, hidden_dim, bias=bias))
+            if use_bn:
+                layers.append(nn.BatchNorm1d(hidden_dim))
+            layers.append(nn.GELU())
+        layers.append(nn.Linear(hidden_dim, bottleneck_dim, bias=bias))
+        return nn.Sequential(*layers)

models/dinov2/layers/drop_path.py

@@ -0,0 +1,37 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the Apache License, Version 2.0
+# found in the LICENSE file in the root directory of this source tree.
+
+# References:
+#   https://github.com/facebookresearch/dino/blob/master/vision_transformer.py
+#   https://github.com/rwightman/pytorch-image-models/tree/master/timm/layers/drop.py
+
+
+from torch import nn
+
+
+def drop_path(x, drop_prob: float = 0.0, training: bool = False):
+    if drop_prob == 0.0 or not training:
+        return x
+    keep_prob = 1 - drop_prob
+    shape = (x.shape[0],) + (1,) * (
+        x.ndim - 1
+    )  # work with diff dim tensors, not just 2D ConvNets
+    random_tensor = x.new_empty(shape).bernoulli_(keep_prob)
+    if keep_prob > 0.0:
+        random_tensor.div_(keep_prob)
+    output = x * random_tensor
+    return output
+
+
+class DropPath(nn.Module):
+    """Drop paths (Stochastic Depth) per sample (when applied in main path of
+    residual blocks)."""
+
+    def __init__(self, drop_prob=None):
+        super().__init__()
+        self.drop_prob = drop_prob
+
+    def forward(self, x):
+        return drop_path(x, self.drop_prob, self.training)

models/dinov2/layers/layer_scale.py

@@ -0,0 +1,26 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the Apache License, Version 2.0
+# found in the LICENSE file in the root directory of this source tree.
+
+# Modified from: https://github.com/huggingface/pytorch-image-models/blob/main/timm/models/vision_transformer.py#L103-L110
+
+from typing import Union
+
+import torch
+from torch import Tensor, nn
+
+
+class LayerScale(nn.Module):
+    def __init__(
+        self,
+        dim: int,
+        init_values: Union[float, Tensor] = 1e-5,
+        inplace: bool = False,
+    ) -> None:
+        super().__init__()
+        self.inplace = inplace
+        self.gamma = nn.Parameter(init_values * torch.ones(dim))
+
+    def forward(self, x: Tensor) -> Tensor:
+        return x.mul_(self.gamma) if self.inplace else x * self.gamma

models/dinov2/layers/mlp.py

@@ -0,0 +1,40 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the Apache License, Version 2.0
+# found in the LICENSE file in the root directory of this source tree.
+
+# References:
+#   https://github.com/facebookresearch/dino/blob/master/vision_transformer.py
+#   https://github.com/rwightman/pytorch-image-models/tree/master/timm/layers/mlp.py
+
+
+from typing import Callable, Optional
+
+from torch import Tensor, nn
+
+
+class Mlp(nn.Module):
+    def __init__(
+        self,
+        in_features: int,
+        hidden_features: Optional[int] = None,
+        out_features: Optional[int] = None,
+        act_layer: Callable[..., nn.Module] = nn.GELU,
+        drop: float = 0.0,
+        bias: bool = True,
+    ) -> None:
+        super().__init__()
+        out_features = out_features or in_features
+        hidden_features = hidden_features or in_features
+        self.fc1 = nn.Linear(in_features, hidden_features, bias=bias)
+        self.act = act_layer()
+        self.fc2 = nn.Linear(hidden_features, out_features, bias=bias)
+        self.drop = nn.Dropout(drop)
+
+    def forward(self, x: Tensor) -> Tensor:
+        x = self.fc1(x)
+        x = self.act(x)
+        x = self.drop(x)
+        x = self.fc2(x)
+        x = self.drop(x)
+        return x

models/dinov2/layers/patch_embed.py

@@ -0,0 +1,100 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the Apache License, Version 2.0
+# found in the LICENSE file in the root directory of this source tree.
+
+# References:
+#   https://github.com/facebookresearch/dino/blob/master/vision_transformer.py
+#   https://github.com/rwightman/pytorch-image-models/tree/master/timm/layers/patch_embed.py
+
+from typing import Callable, Optional, Tuple, Union
+
+import torch.nn as nn
+from torch import Tensor
+
+
+def make_2tuple(x):
+    if isinstance(x, tuple):
+        assert len(x) == 2
+        return x
+
+    assert isinstance(x, int)
+    return (x, x)
+
+
+class PatchEmbed(nn.Module):
+    """
+    2D image to patch embedding: (B,C,H,W) -> (B,N,D)
+
+    Args:
+        img_size: Image size.
+        patch_size: Patch token size.
+        in_chans: Number of input image channels.
+        embed_dim: Number of linear projection output channels.
+        norm_layer: Normalization layer.
+    """
+
+    def __init__(
+        self,
+        img_size: Union[int, Tuple[int, int]] = 224,
+        patch_size: Union[int, Tuple[int, int]] = 16,
+        in_chans: int = 3,
+        embed_dim: int = 768,
+        norm_layer: Optional[Callable] = None,
+        flatten_embedding: bool = True,
+    ) -> None:
+        super().__init__()
+
+        image_HW = make_2tuple(img_size)
+        patch_HW = make_2tuple(patch_size)
+        patch_grid_size = (
+            image_HW[0] // patch_HW[0],
+            image_HW[1] // patch_HW[1],
+        )
+
+        self.img_size = image_HW
+        self.patch_size = patch_HW
+        self.patches_resolution = patch_grid_size
+        self.num_patches = patch_grid_size[0] * patch_grid_size[1]
+
+        self.in_chans = in_chans
+        self.embed_dim = embed_dim
+
+        self.flatten_embedding = flatten_embedding
+
+        self.proj = nn.Conv2d(
+            in_chans, embed_dim, kernel_size=patch_HW, stride=patch_HW
+        )
+        self.norm = norm_layer(embed_dim) if norm_layer else nn.Identity()
+
+    def forward(self, x: Tensor) -> Tensor:
+        _, _, H, W = x.shape
+        patch_H, patch_W = self.patch_size
+
+        assert (
+            H % patch_H == 0
+        ), f'Input image height {H} is not a multiple of patch height {patch_H}'
+        assert (
+            W % patch_W == 0
+        ), f'Input image width {W} is not a multiple of patch width: {patch_W}'
+
+        x = self.proj(x)  # B C H W
+        H, W = x.size(2), x.size(3)
+        x = x.flatten(2).transpose(1, 2)  # B HW C
+        x = self.norm(x)
+        if not self.flatten_embedding:
+            x = x.reshape(-1, H, W, self.embed_dim)  # B H W C
+        return x
+
+    def flops(self) -> float:
+        Ho, Wo = self.patches_resolution
+        flops = (
+            Ho
+            * Wo
+            * self.embed_dim
+            * self.in_chans
+            * (self.patch_size[0] * self.patch_size[1])
+        )
+        if self.norm is not None:
+            flops += Ho * Wo * self.embed_dim
+        return flops

models/dinov2/layers/swiglu_ffn.py

@@ -0,0 +1,69 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the Apache License, Version 2.0
+# found in the LICENSE file in the root directory of this source tree.
+
+import os
+import warnings
+from typing import Callable, Optional
+
+import torch.nn.functional as F
+from torch import Tensor, nn
+
+
+class SwiGLUFFN(nn.Module):
+    def __init__(
+        self,
+        in_features: int,
+        hidden_features: Optional[int] = None,
+        out_features: Optional[int] = None,
+        act_layer: Callable[..., nn.Module] = None,
+        drop: float = 0.0,
+        bias: bool = True,
+    ) -> None:
+        super().__init__()
+        out_features = out_features or in_features
+        hidden_features = hidden_features or in_features
+        self.w12 = nn.Linear(in_features, 2 * hidden_features, bias=bias)
+        self.w3 = nn.Linear(hidden_features, out_features, bias=bias)
+
+    def forward(self, x: Tensor) -> Tensor:
+        x12 = self.w12(x)
+        x1, x2 = x12.chunk(2, dim=-1)
+        hidden = F.silu(x1) * x2
+        return self.w3(hidden)
+
+
+XFORMERS_ENABLED = os.environ.get('XFORMERS_DISABLED') is None
+try:
+    if XFORMERS_ENABLED:
+        from xformers.ops import SwiGLU
+
+        XFORMERS_AVAILABLE = True
+        warnings.warn('xFormers is available (SwiGLU)')
+    else:
+        raise ImportError
+except ImportError:
+    SwiGLU = SwiGLUFFN
+    XFORMERS_AVAILABLE = False
+
+
+class SwiGLUFFNFused(SwiGLU):
+    def __init__(
+        self,
+        in_features: int,
+        hidden_features: Optional[int] = None,
+        out_features: Optional[int] = None,
+        act_layer: Callable[..., nn.Module] = None,
+        drop: float = 0.0,
+        bias: bool = True,
+    ) -> None:
+        out_features = out_features or in_features
+        hidden_features = hidden_features or in_features
+        hidden_features = (int(hidden_features * 2 / 3) + 7) // 8 * 8
+        super().__init__(
+            in_features=in_features,
+            hidden_features=hidden_features,
+            out_features=out_features,
+            bias=bias,
+        )

models/dinov2/vision_transformer.py

@@ -0,0 +1,454 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the Apache License, Version 2.0
+# found in the LICENSE file in the root directory of this source tree.
+
+# References:
+#   https://github.com/facebookresearch/dino/blob/main/vision_transformer.py
+#   https://github.com/rwightman/pytorch-image-models/tree/master/timm/models/vision_transformer.py
+
+import logging
+import math
+from functools import partial
+from typing import Callable, Sequence, Tuple, Union
+
+import torch
+import torch.nn as nn
+import torch.utils.checkpoint
+from torch.nn.init import trunc_normal_
+
+from .layers import MemEffAttention, Mlp
+from .layers import NestedTensorBlock as Block
+from .layers import PatchEmbed, SwiGLUFFNFused
+
+logger = logging.getLogger('dinov2')
+
+
+def named_apply(
+    fn: Callable,
+    module: nn.Module,
+    name='',
+    depth_first=True,
+    include_root=False,
+) -> nn.Module:
+    if not depth_first and include_root:
+        fn(module=module, name=name)
+    for child_name, child_module in module.named_children():
+        child_name = '.'.join((name, child_name)) if name else child_name
+        named_apply(
+            fn=fn,
+            module=child_module,
+            name=child_name,
+            depth_first=depth_first,
+            include_root=True,
+        )
+    if depth_first and include_root:
+        fn(module=module, name=name)
+    return module
+
+
+class BlockChunk(nn.ModuleList):
+    def forward(self, x):
+        for b in self:
+            x = b(x)
+        return x
+
+
+class DinoVisionTransformer(nn.Module):
+    def __init__(
+        self,
+        img_size=224,
+        patch_size=16,
+        in_chans=3,
+        embed_dim=768,
+        depth=12,
+        num_heads=12,
+        mlp_ratio=4.0,
+        qkv_bias=True,
+        ffn_bias=True,
+        proj_bias=True,
+        drop_path_rate=0.0,
+        drop_path_uniform=False,
+        init_values=None,  # for layerscale: None or 0 => no layerscale
+        embed_layer=PatchEmbed,
+        act_layer=nn.GELU,
+        block_fn=Block,
+        ffn_layer='mlp',
+        block_chunks=1,
+        num_register_tokens=0,
+        interpolate_antialias=False,
+        interpolate_offset=0.1,
+    ):
+        """
+        Args:
+            img_size (int, tuple): input image size
+            patch_size (int, tuple): patch size
+            in_chans (int): number of input channels
+            embed_dim (int): embedding dimension
+            depth (int): depth of transformer
+            num_heads (int): number of attention heads
+            mlp_ratio (int): ratio of mlp hidden dim to embedding dim
+            qkv_bias (bool): enable bias for qkv if True
+            proj_bias (bool): enable bias for proj in attn if True
+            ffn_bias (bool): enable bias for ffn if True
+            drop_path_rate (float): stochastic depth rate
+            drop_path_uniform (bool): apply uniform drop rate across blocks
+            weight_init (str): weight init scheme
+            init_values (float): layer-scale init values
+            embed_layer (nn.Module): patch embedding layer
+            act_layer (nn.Module): MLP activation layer
+            block_fn (nn.Module): transformer block class
+            ffn_layer (str): "mlp", "swiglu", "swiglufused" or "identity"
+            block_chunks: (int) split block sequence into block_chunks units for FSDP wrap
+            num_register_tokens: (int) number of extra cls tokens (so-called "registers")
+            interpolate_antialias: (str) flag to apply anti-aliasing when interpolating positional embeddings
+            interpolate_offset: (float) work-around offset to apply when interpolating positional embeddings
+        """
+        super().__init__()
+        norm_layer = partial(nn.LayerNorm, eps=1e-6)
+
+        self.num_features = (
+            self.embed_dim
+        ) = embed_dim  # num_features for consistency with other models
+        self.num_tokens = 1
+        self.n_blocks = depth
+        self.num_heads = num_heads
+        self.patch_size = patch_size
+        self.num_register_tokens = num_register_tokens
+        self.interpolate_antialias = interpolate_antialias
+        self.interpolate_offset = interpolate_offset
+
+        self.patch_embed = embed_layer(
+            img_size=img_size,
+            patch_size=patch_size,
+            in_chans=in_chans,
+            embed_dim=embed_dim,
+        )
+        num_patches = self.patch_embed.num_patches
+
+        self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim))
+        self.pos_embed = nn.Parameter(
+            torch.zeros(1, num_patches + self.num_tokens, embed_dim)
+        )
+        assert num_register_tokens >= 0
+        self.register_tokens = (
+            nn.Parameter(torch.zeros(1, num_register_tokens, embed_dim))
+            if num_register_tokens
+            else None
+        )
+
+        if drop_path_uniform is True:
+            dpr = [drop_path_rate] * depth
+        else:
+            dpr = [
+                x.item() for x in torch.linspace(0, drop_path_rate, depth)
+            ]  # stochastic depth decay rule
+
+        if ffn_layer == 'mlp':
+            logger.info('using MLP layer as FFN')
+            ffn_layer = Mlp
+        elif ffn_layer == 'swiglufused' or ffn_layer == 'swiglu':
+            logger.info('using SwiGLU layer as FFN')
+            ffn_layer = SwiGLUFFNFused
+        elif ffn_layer == 'identity':
+            logger.info('using Identity layer as FFN')
+
+            def f(*args, **kwargs):
+                return nn.Identity()
+
+            ffn_layer = f
+        else:
+            raise NotImplementedError
+
+        blocks_list = [
+            block_fn(
+                dim=embed_dim,
+                num_heads=num_heads,
+                mlp_ratio=mlp_ratio,
+                qkv_bias=qkv_bias,
+                proj_bias=proj_bias,
+                ffn_bias=ffn_bias,
+                drop_path=dpr[i],
+                norm_layer=norm_layer,
+                act_layer=act_layer,
+                ffn_layer=ffn_layer,
+                init_values=init_values,
+            )
+            for i in range(depth)
+        ]
+        if block_chunks > 0:
+            self.chunked_blocks = True
+            chunked_blocks = []
+            chunksize = depth // block_chunks
+            for i in range(0, depth, chunksize):
+                # this is to keep the block index consistent if we chunk the block list
+                chunked_blocks.append(
+                    [nn.Identity()] * i + blocks_list[i : i + chunksize]
+                )
+            self.blocks = nn.ModuleList(
+                [BlockChunk(p) for p in chunked_blocks]
+            )
+        else:
+            self.chunked_blocks = False
+            self.blocks = nn.ModuleList(blocks_list)
+
+        self.norm = norm_layer(embed_dim)
+        self.head = nn.Identity()
+
+        self.mask_token = nn.Parameter(torch.zeros(1, embed_dim))
+
+        self.init_weights()
+
+    def init_weights(self):
+        trunc_normal_(self.pos_embed, std=0.02)
+        nn.init.normal_(self.cls_token, std=1e-6)
+        if self.register_tokens is not None:
+            nn.init.normal_(self.register_tokens, std=1e-6)
+        named_apply(init_weights_vit_timm, self)
+
+    def interpolate_pos_encoding(self, x, w, h):
+        previous_dtype = x.dtype
+        npatch = x.shape[1] - 1
+        N = self.pos_embed.shape[1] - 1
+        if npatch == N and w == h:
+            return self.pos_embed
+        pos_embed = self.pos_embed.float()
+        class_pos_embed = pos_embed[:, 0]
+        patch_pos_embed = pos_embed[:, 1:]
+        dim = x.shape[-1]
+        w0 = w // self.patch_size
+        h0 = h // self.patch_size
+        M = int(
+            math.sqrt(N)
+        )  # Recover the number of patches in each dimension
+        assert N == M * M
+        kwargs = {}
+        if self.interpolate_offset:
+            # Historical kludge: add a small number to avoid floating point error in the interpolation, see https://github.com/facebookresearch/dino/issues/8
+            # Note: still needed for backward-compatibility, the underlying operators are using both output size and scale factors
+            sx = float(w0 + self.interpolate_offset) / M
+            sy = float(h0 + self.interpolate_offset) / M
+            kwargs['scale_factor'] = (sx, sy)
+        else:
+            # Simply specify an output size instead of a scale factor
+            kwargs['size'] = (w0, h0)
+        patch_pos_embed = nn.functional.interpolate(
+            patch_pos_embed.reshape(1, M, M, dim).permute(0, 3, 1, 2),
+            mode='bicubic',
+            antialias=self.interpolate_antialias,
+            **kwargs,
+        )
+        assert (w0, h0) == patch_pos_embed.shape[-2:]
+        patch_pos_embed = patch_pos_embed.permute(0, 2, 3, 1).view(1, -1, dim)
+        return torch.cat(
+            (class_pos_embed.unsqueeze(0), patch_pos_embed), dim=1
+        ).to(previous_dtype)
+
+    def prepare_tokens_with_masks(self, x, masks=None):
+        B, nc, w, h = x.shape
+        x = self.patch_embed(x)
+        if masks is not None:
+            x = torch.where(
+                masks.unsqueeze(-1),
+                self.mask_token.to(x.dtype).unsqueeze(0),
+                x,
+            )
+
+        x = torch.cat((self.cls_token.expand(x.shape[0], -1, -1), x), dim=1)
+        x = x + self.interpolate_pos_encoding(x, w, h)
+
+        if self.register_tokens is not None:
+            x = torch.cat(
+                (
+                    x[:, :1],
+                    self.register_tokens.expand(x.shape[0], -1, -1),
+                    x[:, 1:],
+                ),
+                dim=1,
+            )
+
+        return x
+
+    def forward_features_list(self, x_list, masks_list):
+        x = [
+            self.prepare_tokens_with_masks(x, masks)
+            for x, masks in zip(x_list, masks_list)
+        ]
+        for blk in self.blocks:
+            x = blk(x)
+
+        all_x = x
+        output = []
+        for x, masks in zip(all_x, masks_list):
+            x_norm = self.norm(x)
+            output.append(
+                {
+                    'x_norm_clstoken': x_norm[:, 0],
+                    'x_norm_regtokens': x_norm[
+                        :, 1 : self.num_register_tokens + 1
+                    ],
+                    'x_norm_patchtokens': x_norm[
+                        :, self.num_register_tokens + 1 :
+                    ],
+                    'x_prenorm': x,
+                    'masks': masks,
+                }
+            )
+        return output
+
+    def forward_features(self, x, masks=None):
+        if isinstance(x, list):
+            return self.forward_features_list(x, masks)
+
+        x = self.prepare_tokens_with_masks(x, masks)
+
+        for blk in self.blocks:
+            x = blk(x)
+
+        x_norm = self.norm(x)
+        return {
+            'x_norm_clstoken': x_norm[:, 0],
+            'x_norm_regtokens': x_norm[:, 1 : self.num_register_tokens + 1],
+            'x_norm_patchtokens': x_norm[:, self.num_register_tokens + 1 :],
+            'x_prenorm': x,
+            'masks': masks,
+        }
+
+    def _get_intermediate_layers_not_chunked(self, x, n=1):
+        x = self.prepare_tokens_with_masks(x)
+        # If n is an int, take the n last blocks. If it's a list, take them
+        output, total_block_len = [], len(self.blocks)
+        blocks_to_take = (
+            range(total_block_len - n, total_block_len)
+            if isinstance(n, int)
+            else n
+        )
+        for i, blk in enumerate(self.blocks):
+            x = blk(x)
+            if i in blocks_to_take:
+                output.append(x)
+        assert len(output) == len(
+            blocks_to_take
+        ), f'only {len(output)} / {len(blocks_to_take)} blocks found'
+        return output
+
+    def _get_intermediate_layers_chunked(self, x, n=1):
+        x = self.prepare_tokens_with_masks(x)
+        output, i, total_block_len = [], 0, len(self.blocks[-1])
+        # If n is an int, take the n last blocks. If it's a list, take them
+        blocks_to_take = (
+            range(total_block_len - n, total_block_len)
+            if isinstance(n, int)
+            else n
+        )
+        for block_chunk in self.blocks:
+            for blk in block_chunk[i:]:  # Passing the nn.Identity()
+                x = blk(x)
+                if i in blocks_to_take:
+                    output.append(x)
+                i += 1
+        assert len(output) == len(
+            blocks_to_take
+        ), f'only {len(output)} / {len(blocks_to_take)} blocks found'
+        return output
+
+    def get_intermediate_layers(
+        self,
+        x: torch.Tensor,
+        n: Union[int, Sequence] = 1,  # Layers or n last layers to take
+        reshape: bool = False,
+        return_class_token: bool = False,
+        norm=True,
+    ) -> Tuple[Union[torch.Tensor, Tuple[torch.Tensor]]]:
+        if self.chunked_blocks:
+            outputs = self._get_intermediate_layers_chunked(x, n)
+        else:
+            outputs = self._get_intermediate_layers_not_chunked(x, n)
+        if norm:
+            outputs = [self.norm(out) for out in outputs]
+        class_tokens = [out[:, 0] for out in outputs]
+        outputs = [out[:, 1 + self.num_register_tokens :] for out in outputs]
+        if reshape:
+            B, _, w, h = x.shape
+            outputs = [
+                out.reshape(B, w // self.patch_size, h // self.patch_size, -1)
+                .permute(0, 3, 1, 2)
+                .contiguous()
+                for out in outputs
+            ]
+        if return_class_token:
+            return tuple(zip(outputs, class_tokens))
+        return tuple(outputs)
+
+    def forward(self, *args, is_training=False, **kwargs):
+        ret = self.forward_features(*args, **kwargs)
+        if is_training:
+            return ret
+        else:
+            return self.head(ret['x_norm_clstoken'])
+
+
+def init_weights_vit_timm(module: nn.Module, name: str = ''):
+    """ViT weight initialization, original timm impl (for reproducibility)"""
+    if isinstance(module, nn.Linear):
+        trunc_normal_(module.weight, std=0.02)
+        if module.bias is not None:
+            nn.init.zeros_(module.bias)
+
+
+def vit_small(patch_size=16, num_register_tokens=0, **kwargs):
+    model = DinoVisionTransformer(
+        patch_size=patch_size,
+        embed_dim=384,
+        depth=12,
+        num_heads=6,
+        mlp_ratio=4,
+        block_fn=partial(Block, attn_class=MemEffAttention),
+        num_register_tokens=num_register_tokens,
+        **kwargs,
+    )
+    return model
+
+
+def vit_base(patch_size=16, num_register_tokens=0, **kwargs):
+    model = DinoVisionTransformer(
+        patch_size=patch_size,
+        embed_dim=768,
+        depth=12,
+        num_heads=12,
+        mlp_ratio=4,
+        block_fn=partial(Block, attn_class=MemEffAttention),
+        num_register_tokens=num_register_tokens,
+        **kwargs,
+    )
+    return model
+
+
+def vit_large(patch_size=16, num_register_tokens=0, **kwargs):
+    model = DinoVisionTransformer(
+        patch_size=patch_size,
+        embed_dim=1024,
+        depth=24,
+        num_heads=16,
+        mlp_ratio=4,
+        block_fn=partial(Block, attn_class=MemEffAttention),
+        num_register_tokens=num_register_tokens,
+        **kwargs,
+    )
+    return model
+
+
+def vit_giant2(patch_size=16, num_register_tokens=0, **kwargs):
+    """Close to ViT-giant, with embed-dim 1536 and 24 heads => embed-dim per
+    head 64."""
+    model = DinoVisionTransformer(
+        patch_size=patch_size,
+        embed_dim=1536,
+        depth=40,
+        num_heads=24,
+        mlp_ratio=4,
+        block_fn=partial(Block, attn_class=MemEffAttention),
+        num_register_tokens=num_register_tokens,
+        **kwargs,
+    )
+    return model

neighbor_loss.py

@@ -0,0 +1,137 @@
+import torch
+from torch.nn.functional import normalize
+
+
+def check_anomaly_theoretical(
+    x,
+    H,
+    W,
+    anomaly_dir=None,
+    temperature=0.1,
+    mask_thr=0.001,
+    kernel=3,
+):
+    x_token = x[:, 1:]
+    B = x.shape[0]
+    assert B == 1
+    x_token = x_token.reshape(H, W, -1).contiguous()
+
+    with torch.no_grad():
+        feature = normalize(x_token, dim=-1)
+        direction = normalize(anomaly_dir, dim=-1)
+
+        logits = -(feature * direction).sum(dim=-1).abs()
+        prob = torch.exp(logits / temperature)
+
+        assert kernel in (3, 5)
+        pad = kernel // 2
+
+        w = prob.unfold(0, kernel, 1).unfold(1, kernel, 1)
+        w = w / w.sum(dim=(-1, -2), keepdims=True)
+
+        if kernel == 3:
+            gaussian = (
+                torch.FloatTensor(
+                    [
+                        1 / 16,
+                        1 / 8,
+                        1 / 16,
+                        1 / 8,
+                        1 / 4,
+                        1 / 8,
+                        1 / 16,
+                        1 / 8,
+                        1 / 16,
+                    ]
+                )
+                .to(w.device)
+                .reshape(1, 1, 3, 3)
+            )
+        elif kernel == 5:
+            gaussian = (
+                torch.tensor(
+                    [
+                        [1, 4, 7, 4, 1],
+                        [4, 16, 26, 16, 4],
+                        [7, 26, 41, 26, 7],
+                        [4, 16, 26, 16, 4],
+                        [1, 4, 7, 4, 1],
+                    ]
+                )
+                .float()
+                .to(w.device)
+                / 273
+            )
+
+        w2 = w * gaussian
+
+        w2 = w2 / w2.sum(dim=(-1, -2), keepdims=True)
+
+        T = x_token.unfold(0, kernel, 1).unfold(1, kernel, 1)
+        T = (T * w2[:, :, None].to(T.device)).sum(dim=(-1, -2))
+
+        mask_full = logits < logits.mean() - mask_thr * logits.std()
+        mask_full[:pad, :] = False
+        mask_full[:, :pad] = False
+        mask_full[-pad:, :] = False
+        mask_full[:, -pad:] = False
+        index_tensor = torch.nonzero(mask_full.flatten()).flatten()
+        if len(index_tensor) == 0:
+            return None
+        rows = index_tensor // W
+        cols = index_tensor % W
+
+        alpha = x_token[pad:-pad, pad:-pad].norm(dim=-1).mean()
+
+    loss_neighbor = (
+        (x_token[rows, cols] - T[rows - pad, cols - pad]).norm(dim=-1)
+    ).mean() / alpha
+
+    return loss_neighbor, rows, cols, T, alpha, mask_full, x_token
+
+
+def get_neighbor_loss(
+    model,
+    x,
+    skip_less_than=1,
+    mask_thr=0.001,
+    kernel=3,
+):
+    H = x.shape[2]
+    W = x.shape[3]
+    x = model.prepare_tokens_with_masks(x)
+
+    for i, blk in enumerate(model.blocks):
+        x = blk(x)
+        assert len(model.singular_defects) > 0
+        result = check_anomaly_theoretical(
+            x,
+            H // model.patch_size,
+            W // model.patch_size,
+            model.singular_defects[i],
+            mask_thr=mask_thr,
+            kernel=kernel,
+        )
+        if result is not None:
+            (
+                loss_neighbor,
+                rows,
+                cols,
+                T,
+                alpha,
+                mask_angle,
+                x_token,
+            ) = result
+            if len(rows) >= skip_less_than:
+                assert not torch.isnan(loss_neighbor).any()
+                return (
+                    i,
+                    loss_neighbor,
+                    rows,
+                    cols,
+                    T,
+                    alpha,
+                    mask_angle,
+                    x_token,
+                )
+    return None

pyproject.toml

@@ -0,0 +1,17 @@
+[build-system]
+requires = ["setuptools>=68", "setuptools_scm[toml]>=8"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "sinder"
+requires-python = ">=3.8"
+dynamic = ["version"]
+dependencies = [
+    # Add runtime dependencies here
+]
+
+# Enables the usage of setuptools_scm
+[tool.setuptools_scm]
+
+[tool.setuptools]
+py-modules = []

repair.py

@@ -0,0 +1,147 @@
+import torch
+
+
+class SVDLinearAddition(torch.nn.Module):
+    def __init__(self, linear, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        w, b = linear.weight, linear.bias
+
+        self.bias = b
+
+        with torch.no_grad():
+            U, S, Vt = torch.linalg.svd(w, full_matrices=False)
+            self.U = torch.nn.Parameter(U)
+            self.S = torch.nn.Parameter(S)
+            self.Vt = torch.nn.Parameter(Vt)
+            self.epsilon = torch.nn.Parameter(torch.zeros_like(S))
+
+    def forward(self, x):
+        w = self.U @ torch.diag(self.S + self.epsilon) @ self.Vt
+        x = torch.nn.functional.linear(x, w, self.bias)
+        return x
+
+
+class SVDLinearAdditionQKV(torch.nn.Module):
+    def __init__(self, linear, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        w, b = linear.weight, linear.bias
+
+        self.bias = b
+
+        with torch.no_grad():
+            c = w.shape[0] // 3
+            q = w[:c]
+            k = w[c : 2 * c]
+            v = w[2 * c :]
+            U_q, S_q, Vt_q = torch.linalg.svd(q, full_matrices=False)
+            self.U_q = torch.nn.Parameter(U_q)
+            self.S_q = torch.nn.Parameter(S_q)
+            self.Vt_q = torch.nn.Parameter(Vt_q)
+            self.epsilon_q = torch.nn.Parameter(torch.zeros_like(S_q))
+            U_k, S_k, Vt_k = torch.linalg.svd(k, full_matrices=False)
+            self.U_k = torch.nn.Parameter(U_k)
+            self.S_k = torch.nn.Parameter(S_k)
+            self.Vt_k = torch.nn.Parameter(Vt_k)
+            self.epsilon_k = torch.nn.Parameter(torch.zeros_like(S_k))
+            U_v, S_v, Vt_v = torch.linalg.svd(v, full_matrices=False)
+            self.U_v = torch.nn.Parameter(U_v)
+            self.S_v = torch.nn.Parameter(S_v)
+            self.Vt_v = torch.nn.Parameter(Vt_v)
+            self.epsilon_v = torch.nn.Parameter(torch.zeros_like(S_v))
+
+    def forward(self, x):
+        w = torch.concatenate(
+            (
+                self.U_q @ torch.diag(self.S_q + self.epsilon_q) @ self.Vt_q,
+                self.U_k @ torch.diag(self.S_k + self.epsilon_k) @ self.Vt_k,
+                self.U_v @ torch.diag(self.S_v + self.epsilon_v) @ self.Vt_v,
+            )
+        )
+        x = torch.nn.functional.linear(x, w, self.bias)
+        return x
+
+
+def replace_linear_addition_noqk(module, name):
+    for attr_str in dir(module):
+        target_attr = getattr(module, attr_str)
+        if type(target_attr) == torch.nn.Linear:
+            if 'qkv' in attr_str:
+                print('replaced: ', name, attr_str)
+                svd_linear_qkv = SVDLinearAdditionQKV(target_attr)
+                svd_linear_qkv.U_q.requires_grad = False
+                svd_linear_qkv.U_k.requires_grad = False
+                svd_linear_qkv.U_v.requires_grad = False
+                svd_linear_qkv.S_q.requires_grad = False
+                svd_linear_qkv.S_k.requires_grad = False
+                svd_linear_qkv.S_v.requires_grad = False
+                svd_linear_qkv.Vt_q.requires_grad = False
+                svd_linear_qkv.Vt_k.requires_grad = False
+                svd_linear_qkv.Vt_v.requires_grad = False
+                svd_linear_qkv.epsilon_q.requires_grad = False
+                svd_linear_qkv.epsilon_k.requires_grad = False
+                svd_linear_qkv.epsilon_v.requires_grad = True
+                svd_linear_qkv.bias.requires_grad = False
+                setattr(module, attr_str, svd_linear_qkv)
+            else:
+                print('replaced: ', name, attr_str)
+                svd_linear = SVDLinearAddition(target_attr)
+                svd_linear.U.requires_grad = False
+                svd_linear.S.requires_grad = False
+                svd_linear.Vt.requires_grad = False
+                svd_linear.bias.requires_grad = False
+                svd_linear.epsilon.requires_grad = True
+                setattr(module, attr_str, svd_linear)
+
+    for name, immediate_child_module in module.named_children():
+        replace_linear_addition_noqk(immediate_child_module, name)
+
+
+def replace_back(module, name):
+    for attr_str in dir(module):
+        target_attr = getattr(module, attr_str)
+
+        if type(target_attr) == SVDLinearAddition:
+            print('replaced back: ', name, attr_str)
+            with torch.no_grad():
+                linear = torch.nn.Linear(
+                    target_attr.Vt.shape[1],
+                    target_attr.U.shape[0],
+                    device=target_attr.U.device,
+                )
+                linear.weight.add_(
+                    target_attr.U
+                    @ torch.diag(target_attr.S + target_attr.epsilon)
+                    @ target_attr.Vt
+                    - linear.weight
+                )
+                linear.bias.add_(target_attr.bias - linear.bias)
+
+            setattr(module, attr_str, linear)
+
+        elif type(target_attr) == SVDLinearAdditionQKV:
+            print('replaced back: ', name, attr_str)
+            with torch.no_grad():
+                w = torch.concatenate(
+                    (
+                        target_attr.U_q
+                        @ torch.diag(target_attr.S_q + target_attr.epsilon_q)
+                        @ target_attr.Vt_q,
+                        target_attr.U_k
+                        @ torch.diag(target_attr.S_k + target_attr.epsilon_k)
+                        @ target_attr.Vt_k,
+                        target_attr.U_v
+                        @ torch.diag(target_attr.S_v + target_attr.epsilon_v)
+                        @ target_attr.Vt_v,
+                    )
+                )
+                linear = torch.nn.Linear(
+                    w.shape[1], w.shape[0], device=target_attr.U_q.device
+                )
+                linear.weight.add_(w - linear.weight)
+                linear.bias.add_(target_attr.bias - linear.bias)
+
+            setattr(module, attr_str, linear)
+
+    # iterate through immediate child modules. Note, the recursion is done by our code no need to use named_modules()
+    for name, immediate_child_module in module.named_children():
+        replace_back(immediate_child_module, name)

setup.cfg

@@ -0,0 +1,23 @@
+[bdist_wheel]
+universal=1
+
+[yapf]
+based_on_style = pep8
+blank_line_before_nested_class_or_def = true
+split_before_expression_after_opening_paren = true
+allow_split_before_dict_value = False
+split_penalty_import_names=0
+SPLIT_PENALTY_AFTER_OPENING_BRACKET=0
+
+[isort]
+line_length = 79
+extra_standard_library = pkg_resources,setuptools,logging,os,warnings,abc
+known_first_party =
+known_third_party = addict,cv2,matplotlib,numpy,onnx,packaging,pytest,pytorch_sphinx_theme,scipy,sphinx,torch,torchvision,yaml,yapf
+no_lines_before = STDLIB,LOCALFOLDER
+default_section = THIRDPARTY
+profile = black
+
+[flake8]
+extend-ignore = E501,E203,E722,E266,E402,E251
+exclude = .git,__pycache__

singular_defect.py

@@ -0,0 +1,149 @@
+# compute singular defect directions
+import torch
+import torch.nn.functional as F
+
+
+def anomaly_dir_attn(
+    blk,
+    identity=False,
+    bias=False,
+    centered=False,
+    homogeneous=False,
+):
+    with torch.no_grad():
+        N = blk.ls1.gamma.shape[0]
+        dev = blk.ls1.gamma.device
+
+        A4 = torch.diag(blk.ls1.gamma)
+        A3 = blk.attn.proj.weight
+        B3 = blk.attn.proj.bias
+        A2 = blk.attn.qkv.weight.chunk(3, dim=0)[-1]
+        B2 = blk.attn.qkv.bias.chunk(3, dim=0)[-1]
+        A1 = torch.diag(blk.norm1.weight)
+        B1 = blk.norm1.bias
+        A0 = (torch.eye(N) - 1 / N * torch.ones(N, N)).to(dev)
+        A = A4 @ A3 @ A2 @ A1
+
+        if centered:
+            A = A @ A0
+        B = A4 @ (A3 @ (A2 @ B1)) + A4 @ (A3 @ B2) + A4 @ B3
+
+        if bias:
+            A = torch.cat((A, B[:, None]), dim=1)
+            if homogeneous:
+                onehot = torch.cat(
+                    (torch.zeros_like(B), torch.ones(1).to(dev))
+                )
+                A = torch.cat((A, onehot[None]), dim=0)
+
+        if identity:
+            iden = torch.eye(N).to(dev)
+            A[:N, :N] += iden
+        u, _, _ = torch.linalg.svd(A)
+
+    return u[:N, 0], A, B
+
+
+def w12(blk, x):
+    with torch.no_grad():
+        x1, x2 = blk.mlp.w12(x).chunk(2, dim=-1)
+    return F.silu(x1) * x2
+
+
+def anomaly_dir_mlp_ls(
+    blk,
+    identity=False,
+    bias=False,
+    centered=False,
+    homogeneous=False,
+    bias_ls=False,
+):
+    with torch.no_grad():
+        N = blk.ls2.gamma.shape[0]
+        M = blk.mlp.w3.weight.shape[1]
+        dev = blk.ls2.gamma.device
+
+        A4 = torch.diag(blk.ls2.gamma)
+        A3 = blk.mlp.w3.weight
+        B3 = blk.mlp.w3.bias
+
+        X = torch.randn(100000, N, device=dev)
+        Y = w12(blk, X)
+        if bias_ls:
+            X_one = torch.cat((X, torch.ones(100000, 1).to(dev)), dim=1)
+        else:
+            X_one = X
+        sol = torch.linalg.lstsq(X_one, Y)
+        if bias_ls:
+            A2 = sol.solution.T[:, :-1]
+            B2 = sol.solution.T[:, -1]
+        else:
+            A2 = sol.solution.T
+            B2 = torch.zeros(M).to(dev)
+
+        A1 = torch.diag(blk.norm2.weight)
+        B1 = blk.norm2.bias
+        A0 = (torch.eye(N) - 1 / N * torch.ones(N, N)).to(dev)
+        A = A4 @ A3 @ A2 @ A1
+
+        if centered:
+            A = A @ A0
+        B = A4 @ (A3 @ (A2 @ B1)) + A4 @ (A3 @ B2) + A4 @ B3
+
+        if bias:
+            A = torch.cat((A, B[:, None]), dim=1)
+            if homogeneous:
+                onehot = torch.cat(
+                    (torch.zeros_like(B), torch.ones(1).to(dev))
+                )
+                A = torch.cat((A, onehot[None]), dim=0)
+
+        if identity:
+            iden = torch.eye(N).to(dev)
+            A[:N, :N] += iden
+        u, s, vt = torch.linalg.svd(A)
+
+    return u[:N, 0], A, B
+
+
+def anomaly_dir(blk, homogeneous=False):
+    _, A, b = anomaly_dir_attn(
+        blk,
+        identity=True,
+        bias=homogeneous,
+        centered=True,
+        homogeneous=homogeneous,
+    )
+    _, C, d = anomaly_dir_mlp_ls(
+        blk,
+        identity=True,
+        bias=homogeneous,
+        bias_ls=False,
+        centered=True,
+        homogeneous=homogeneous,
+    )
+
+    with torch.no_grad():
+        N = b.shape[0]
+        AA = C @ A
+        if homogeneous:
+            BB = 0
+        else:
+            BB = C @ b + d
+        u, _, _ = torch.linalg.svd(AA)
+
+    return u[:N, 0], AA, BB
+
+
+def singular_defect_directions(model):
+    accumulative_anomalies = []
+    anomaly_dab = [anomaly_dir(blk) for blk in model.blocks]
+    anomaly_as = [dab[1] for dab in anomaly_dab]
+
+    with torch.no_grad():
+        aaa = torch.eye(anomaly_as[0].shape[0]).to(anomaly_as[0])
+        for a in anomaly_as:
+            aaa = a @ aaa
+            u, _, _ = torch.linalg.svd(aaa)
+            accumulative_anomalies.append(u[:, 0])
+    return accumulative_anomalies

utils.py

@@ -0,0 +1,70 @@
+from pathlib import Path
+
+import torch
+from PIL import Image
+from sklearn.decomposition import PCA
+
+import sinder
+from .singular_defect import singular_defect_directions
+
+
+def pca_array(tokens, whiten=False):
+    h, w, c = tokens.shape
+    tokens = tokens.detach().cpu()
+
+    pca = PCA(n_components=3, whiten=whiten)
+    pca.fit(tokens.reshape(-1, c))
+    projected_tokens = pca.transform(tokens.reshape(-1, c))
+
+    t = torch.tensor(projected_tokens)
+    t_min = t.min(dim=0, keepdim=True).values
+    t_max = t.max(dim=0, keepdim=True).values
+    normalized_t = (t - t_min) / (t_max - t_min)
+
+    array = (normalized_t * 255).byte().numpy()
+    array = array.reshape(h, w, 3)
+
+    return Image.fromarray(array).resize((w * 7, h * 7), 0)
+
+
+def get_tokens(model, image, blocks=1):
+    model.eval()
+    with torch.no_grad():
+        image_batch = image.unsqueeze(0).cuda()
+        image_batch = image_batch.cuda()
+        H = image_batch.shape[2]
+        W = image_batch.shape[3]
+        print(f'{W=} {H=}')
+        tokens = model.get_intermediate_layers(
+            image_batch, blocks, return_class_token=True, norm=False
+        )
+        tokens = [
+            (
+                t.reshape(
+                    (H // model.patch_size, W // model.patch_size, t.size(-1))
+                ),
+                tc,
+            )
+            for t, tc in tokens
+        ]
+
+    return tokens
+
+
+def load_model(model_name, checkpoint=None):
+    print(f'using {model_name} model')
+    model = torch.hub.load(
+        repo_or_dir=Path(sinder.__file__).parent.parent,
+        source='local',
+        model=model_name,
+    )
+    if checkpoint is not None:
+        states = torch.load(checkpoint, map_location='cpu')
+        model.load_state_dict(states, strict=False)
+    model = model.cuda()
+    model.eval()
+    model.interpolate_antialias = True
+    model.singular_defects = singular_defect_directions(model)
+    print(f'model loaded. patch size: {model.patch_size}')
+
+    return model

visualize.py

@@ -0,0 +1,82 @@
+#!/usr/bin/env python
+import argparse
+import os
+import sys
+from pathlib import Path
+import cv2
+
+import numpy as np
+from PIL import Image
+from tqdm import tqdm
+
+from sinder import (
+    get_tokens,
+    load_model,
+    load_visual_data,
+    pca_array,
+)
+
+os.environ['XFORMERS_DISABLED'] = '1'
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description='Visualize')
+    parser.add_argument(
+        'imgs', nargs='+', type=str, help='path to image/images'
+    )
+    parser.add_argument(
+        '--model', type=str, default='dinov2_vitg14', help='model name'
+    )
+    parser.add_argument('--workdir', type=str, default='visualize')
+    parser.add_argument(
+        '--checkpoint',
+        type=str,
+        default=None,
+        help='Path to checkpoint. Default is None, which loads the official pretrained weights',
+    )
+    parser.add_argument(
+        '--visual_size',
+        type=int,
+        default=518,
+        help='short side size of input image',
+    )
+
+    args = parser.parse_args()
+    return args
+
+
+def visualize(args, model, visual_dataset):
+    model.eval()
+
+    for d in tqdm(range(len(visual_dataset))):
+        visual_image = visual_dataset[d]
+        visual_tokens_all = get_tokens(model, visual_image)
+        visual_tokens, visual_tokens_cls = zip(*visual_tokens_all)
+        filename = Path(visual_dataset.files[d]).stem
+
+        t = visual_tokens[-1].detach().cpu()
+        h, w, c = t.shape
+        norm = ((t.norm(dim=-1) / t.norm(dim=-1).max()) * 255).byte().numpy()
+        norm_img = Image.fromarray(norm).resize((w * 14, h * 14), 0)
+        norm = cv2.applyColorMap(np.array(norm_img), cv2.COLORMAP_JET)
+        cv2.imwrite(args.folder / f'{filename}_norm.png', norm)
+
+        pca_img = pca_array(visual_tokens[-1])
+        pca_img.save(args.folder / f'{filename}_pca.png')
+
+
+def main():
+    args = parse_args()
+
+    args.folder = Path(args.workdir).expanduser()
+    os.makedirs(args.folder, exist_ok=True)
+    print(args)
+    print(' '.join(sys.argv))
+
+    model = load_model(args.model, args.checkpoint)
+    visual_dataset = load_visual_data(args, model)
+    visualize(args, model, visual_dataset)
+
+
+if __name__ == '__main__':
+    main()