init

CSD/CSD/loss_utils.py

@@ -0,0 +1,96 @@
+import torchvision.transforms as transforms
+import torchvision.transforms.functional as F
+import numpy as np
+import torch
+from torch import nn
+np.random.seed(0)
+
+
+class GaussianBlur(object):
+    """blur a single image on CPU"""
+    def __init__(self, kernel_size):
+        radias = kernel_size // 2
+        kernel_size = radias * 2 + 1
+        self.blur_h = nn.Conv2d(3, 3, kernel_size=(kernel_size, 1),
+                                stride=1, padding=0, bias=False, groups=3)
+        self.blur_v = nn.Conv2d(3, 3, kernel_size=(1, kernel_size),
+                                stride=1, padding=0, bias=False, groups=3)
+        self.k = kernel_size
+        self.r = radias
+
+        self.blur = nn.Sequential(
+            nn.ReflectionPad2d(radias),
+            self.blur_h,
+            self.blur_v
+        )
+
+        self.pil_to_tensor = transforms.ToTensor()
+        self.tensor_to_pil = transforms.ToPILImage()
+
+    def __call__(self, img):
+        img = self.pil_to_tensor(img).unsqueeze(0)
+
+        sigma = np.random.uniform(0.1, 2.0)
+        x = np.arange(-self.r, self.r + 1)
+        x = np.exp(-np.power(x, 2) / (2 * sigma * sigma))
+        x = x / x.sum()
+        x = torch.from_numpy(x).view(1, -1).repeat(3, 1)
+
+        self.blur_h.weight.data.copy_(x.view(3, 1, self.k, 1))
+        self.blur_v.weight.data.copy_(x.view(3, 1, 1, self.k))
+
+        with torch.no_grad():
+            img = self.blur(img)
+            img = img.squeeze()
+
+        img = self.tensor_to_pil(img)
+
+        return 
+    
+
+s=1
+size = 224
+
+normalize = transforms.Normalize((0.48145466, 0.4578275, 0.40821073), (0.26862954, 0.26130258, 0.27577711))
+
+transforms_branch0 = transforms.Compose([
+                transforms.Resize(size=size, interpolation=F.InterpolationMode.BICUBIC),
+                transforms.CenterCrop(size),
+                transforms.ToTensor(),
+                normalize,
+            ])
+
+transforms_branch1 = transforms.Compose([
+                transforms.RandomResizedCrop(size, interpolation=F.InterpolationMode.BICUBIC),
+                transforms.RandomHorizontalFlip(),
+                transforms.RandomVerticalFlip(p=0.3), 
+                transforms.RandomRotation(degrees=np.random.choice([0,90,180,270])),
+                transforms.ToTensor(),
+                normalize,
+            ])
+
+color_jitter = transforms.ColorJitter(0.8 * s, 0.8 * s, 0.8 * s, 0.2 * s)
+transforms_branch2 = transforms.Compose([
+                                        # transforms.RandomResizedCrop(size=size, interpolation=F.InterpolationMode.BICUBIC),
+                                        transforms.Resize(size=size, interpolation=F.InterpolationMode.BICUBIC),
+                                        transforms.CenterCrop(size),
+                                        transforms.RandomHorizontalFlip(),
+                                            transforms.RandomApply([transforms.ColorJitter(brightness=0.5, contrast=0.5,
+                                            saturation=0.5,hue=0.1)
+                                            ], p=0.6),
+                                        transforms.RandomApply([transforms.RandomInvert(),transforms.RandomGrayscale(), transforms.GaussianBlur(kernel_size=(5,5), sigma=(0.1, 4))], p=0.8),
+                                        # GaussianBlur(kernel_size=int(0.1 * size)),
+                                        transforms.ToTensor(),
+                                        normalize
+                                        ])
+
+
+class ContrastiveTransformations(object):
+
+    def __init__(self, transforms_b0, transforms_b1,transforms_b2):
+        self.transforms_b0 = transforms_b0
+        self.transforms_b1 = transforms_b1
+        self.transforms_b2 = transforms_b2
+        
+    def __call__(self, x):
+        return [self.transforms_b0(x), self.transforms_b1(x), self.transforms_b2(x)]

CSD/CSD/losses.py

@@ -0,0 +1,99 @@
+"""
+Author: Yonglong Tian (yonglong@mit.edu)
+Date: May 07, 2020
+Code from https://github.com/HobbitLong/SupContrast/blob/master/losses.py
+"""
+from __future__ import print_function
+
+import torch
+import torch.nn as nn
+
+
+class SupConLoss(nn.Module):
+    """Supervised Contrastive Learning: https://arxiv.org/pdf/2004.11362.pdf.
+    It also supports the unsupervised contrastive loss in SimCLR"""
+    def __init__(self, temperature=0.07, contrast_mode='all',
+                 base_temperature=1.0):
+        super(SupConLoss, self).__init__()
+        self.temperature = temperature
+        self.contrast_mode = contrast_mode
+        self.base_temperature = base_temperature
+
+    def forward(self, features, labels=None, mask=None):
+        """Compute loss for model. If both `labels` and `mask` are None,
+        it degenerates to SimCLR unsupervised loss:
+        https://arxiv.org/pdf/2002.05709.pdf
+
+        Args:
+            features: hidden vector of shape [bsz, n_views, ...].
+            labels: ground truth of shape [bsz].
+            mask: contrastive mask of shape [bsz, bsz], mask_{i,j}=1 if sample j
+                has the same class as sample i. Can be asymmetric.
+        Returns:
+            A loss scalar.
+        """
+        device = (torch.device('cuda')
+                  if features.is_cuda
+                  else torch.device('cpu'))
+
+        if len(features.shape) < 3:
+            raise ValueError('`features` needs to be [bsz, n_views, ...],'
+                             'at least 3 dimensions are required')
+        if len(features.shape) > 3:
+            features = features.view(features.shape[0], features.shape[1], -1)
+
+        batch_size = features.shape[0]
+        if labels is not None and mask is not None:
+            raise ValueError('Cannot define both `labels` and `mask`')
+        elif labels is None and mask is None:
+            mask = torch.eye(batch_size, dtype=torch.float32).to(device)
+        elif labels is not None:
+            labels = labels.contiguous().view(-1, 1)
+            if labels.shape[0] != batch_size:
+                raise ValueError('Num of labels does not match num of features')
+            mask = torch.eq(labels, labels.T).float().to(device)
+        else:
+            mask = mask.float().to(device)
+
+        contrast_count = features.shape[1]
+        contrast_feature = torch.cat(torch.unbind(features, dim=1), dim=0)
+        if self.contrast_mode == 'one':
+            anchor_feature = features[:, 0]
+            anchor_count = 1
+        elif self.contrast_mode == 'all':
+            anchor_feature = contrast_feature
+            anchor_count = contrast_count
+        else:
+            raise ValueError('Unknown mode: {}'.format(self.contrast_mode))
+
+        anchor_dot_contrast = torch.div(
+            torch.matmul(anchor_feature, contrast_feature.T),
+            self.temperature)
+        # for numerical stability
+        logits_max, _ = torch.max(anchor_dot_contrast, dim=1, keepdim=True)
+        logits = anchor_dot_contrast - logits_max.detach()
+
+        # tile mask
+        mask = mask.repeat(anchor_count, contrast_count)
+        # mask-out self-contrast cases
+        logits_mask = torch.scatter(
+            torch.ones_like(mask),
+            1,
+            torch.arange(batch_size * anchor_count).view(-1, 1).to(device),
+            0
+        )
+        mask = mask * logits_mask
+
+        # compute log_prob
+        exp_logits = torch.exp(logits) * logits_mask
+        log_prob =  logits - torch.log(exp_logits.sum(1, keepdim=True) + 1e-6) # NOTE: modified based on https://github.com/HobbitLong/SupContrast/issues/104
+
+        # compute mean of log-likelihood over positive, adding a small value in case mask row is 0
+        mean_log_prob_pos = (mask * log_prob).sum(1) / (mask.sum(1) + 1e-6)
+
+        # loss
+        loss = - (self.temperature / self.base_temperature) * mean_log_prob_pos
+        loss = loss.view(anchor_count, batch_size).mean()
+
+        return loss
+    

CSD/CSD/model.py

@@ -0,0 +1,108 @@
+import torch
+import torch.nn as nn
+import clip
+import copy
+from torch.autograd import Function
+
+
+from .utils import convert_weights_float
+
+
+class ReverseLayerF(Function):
+
+    @staticmethod
+    def forward(ctx, x, alpha):
+        ctx.alpha = alpha
+
+        return x.view_as(x)
+
+    @staticmethod
+    def backward(ctx, grad_output):
+        output = grad_output.neg() * ctx.alpha
+
+        return output, None
+
+
+## taken from https://github.com/moein-shariatnia/OpenAI-CLIP/blob/master/modules.py
+class ProjectionHead(nn.Module):
+    def __init__(
+        self,
+        embedding_dim,
+        projection_dim,
+        dropout=0
+    ):
+        super().__init__()
+        self.projection = nn.Linear(embedding_dim, projection_dim)
+        self.gelu = nn.GELU()
+        self.fc = nn.Linear(projection_dim, projection_dim)
+        self.dropout = nn.Dropout(dropout)
+        self.layer_norm = nn.LayerNorm(projection_dim)
+    
+    def forward(self, x):
+        projected = self.projection(x)
+        x = self.gelu(projected)
+        x = self.fc(x)
+        x = self.dropout(x)
+        x = x + projected
+        x = self.layer_norm(x)
+        return x
+
+
+def init_weights(m): # TODO: do we need init for layernorm?
+    if isinstance(m, nn.Linear):
+        torch.nn.init.xavier_uniform_(m.weight)
+        if m.bias is not None:
+            nn.init.normal_(m.bias, std=1e-6)
+
+
+class CSD_CLIP(nn.Module):
+    """backbone + projection head"""
+    def __init__(self, name='vit_large',content_proj_head='default'):
+        super(CSD_CLIP, self).__init__()
+        self.content_proj_head = content_proj_head
+        if name == 'vit_large':
+            clipmodel, _ = clip.load("ViT-L/14")
+            self.backbone = clipmodel.visual
+            self.embedding_dim = 1024
+        elif name == 'vit_base':
+            clipmodel, _ = clip.load("ViT-B/16")
+            self.backbone = clipmodel.visual
+            self.embedding_dim = 768 
+            self.feat_dim = 512
+        else:
+            raise Exception('This model is not implemented')
+
+        convert_weights_float(self.backbone)
+        self.last_layer_style = copy.deepcopy(self.backbone.proj)
+        if content_proj_head == 'custom':
+            self.last_layer_content = ProjectionHead(self.embedding_dim,self.feat_dim)
+            self.last_layer_content.apply(init_weights)
+            
+        else:
+            self.last_layer_content = copy.deepcopy(self.backbone.proj)
+
+        self.backbone.proj = None
+
+    @property
+    def dtype(self):
+        return self.backbone.conv1.weight.dtype
+
+    def forward(self, input_data, alpha=None):
+        
+        feature = self.backbone(input_data)
+
+        if alpha is not None:
+            reverse_feature = ReverseLayerF.apply(feature, alpha)
+        else:
+            reverse_feature = feature
+
+        style_output = feature @ self.last_layer_style
+        style_output = nn.functional.normalize(style_output, dim=1, p=2)
+
+        # if alpha is not None:
+        if self.content_proj_head == 'custom':
+            content_output =  self.last_layer_content(reverse_feature)
+        else:
+            content_output = reverse_feature @ self.last_layer_content
+        content_output = nn.functional.normalize(content_output, dim=1, p=2)
+        return feature, content_output, style_output

CSD/CSD/train_csd.py

@@ -0,0 +1,496 @@
+#!/usr/bin/env python
+
+import argparse
+import json
+import math
+import os
+import pathlib
+import sys
+import time
+import datetime
+import numpy as np
+import copy
+import torch
+import torch.backends.cudnn as cudnn
+import torch.nn as nn
+import torch.nn.parallel
+import torch.optim
+import torch.utils.data
+import torch.utils.data.distributed
+from pathlib import Path
+
+sys.path.insert(0, str(pathlib.Path(__file__).parent.resolve()))
+
+from CSD import utils
+from data.wikiart import WikiArtTrain
+from data.laion import LAION, LAIONDedup
+from CSD.loss_utils import ContrastiveTransformations, transforms_branch0, transforms_branch1, transforms_branch2
+from CSD.model import CSD_CLIP
+from CSD.losses import SupConLoss
+
+
+def get_args_parser():
+
+    parser = argparse.ArgumentParser('CSD', add_help=False)
+
+    # Model
+    parser.add_argument("-a","--arch",default='vit_base', type=str)
+
+    # Data
+    parser.add_argument('--train_set', default='wikiart',  # 'wikiart' or 'laion'
+                    help='Wiki art data path')
+    parser.add_argument('--train_path', required=True,
+                        help='Wiki art data path')
+    parser.add_argument('--train_anno_path',
+                        default='-projects/diffusion_rep/data/laion_style_subset',
+                        help='Annotation dir,  used only for LAION')
+    parser.add_argument("--min_images_per_label", default=1, type=int, 
+                        help="minimum images for a label (used only for laion)")
+    parser.add_argument("--max_images_per_label", default=100000, type=int, 
+                        help="minimum images for a label (used only for laion)")
+
+    parser.add_argument('--eval_set', default='wikiart',  # 'domainnet' or 'wikiart'
+                        help='Wiki art data path')
+    parser.add_argument('--eval_path',required=True,
+                        help='Path to query dataset.')
+    parser.add_argument("--maxsize", default=512, type=int, 
+                        help="maximum size of the val dataset to be used")
+
+    # Optimization
+    parser.add_argument( "--use_fp16", action="store_true",
+                        help="use fp16")
+    parser.add_argument( "--use_distributed_loss", action="store_true",
+                        help="use distributed loss")
+    parser.add_argument('--clip_grad', type=float, default=3.0,
+                        help="""Maximal parameter gradient norm if using 
+                        gradient clipping. Clipping with norm .3 ~ 1.0 can
+                        help optimization for larger ViT architectures.
+                        0 for disabling.""")
+    parser.add_argument("--iters", default=100000, type=int,  # default: eval only
+                        help="number of total iterations to run")
+    parser.add_argument("-b", "--batch_size_per_gpu", default=64, type=int,
+                        help="batch size per GPU (default: 64)")
+    parser.add_argument("--lr", "--learning_rate", default=0.003, type=float,
+                        help="learning rate", dest="lr",)
+    parser.add_argument("--lr_bb", "--learning_rate_bb", default=0.0001, type=float,
+                        help="learning rat for backbone", dest="lr_bb",)
+    parser.add_argument("--wd", "--weight_decay", default=1e-4, type=float,
+                        help="weight decay (default: 1e-4)", dest="weight_decay")
+    parser.add_argument("--warmup_iters", default=30000, type=int,
+                        help="Number of iterations for the linear learning-rate warm up.")
+    parser.add_argument('--min_lr', type=float, default=1e-6, help="""Target LR at the
+                        end of optimization. We use a cosine LR schedule with linear warmup.""")
+    parser.add_argument('--lr_scheduler_type', type=str, default='constant_with_warmup')                   
+    parser.add_argument('--freeze_last_layer', default=0, type=int,
+                        help="""Number of iterations during which we keep the
+                        output layer fixed. Typically doing so during 
+                        first few iters helps training. Try increasing this 
+                        value if the loss does not decrease.""")
+
+    parser.add_argument('--content_proj_head', type=str, default='default')
+    parser.add_argument('--lambda_s', default=1, type=float, help='Weighting on style loss')
+    parser.add_argument('--lambda_c', default=0, type=float, help='Weighting on content loss')
+    parser.add_argument('--lam_sup', default=5, type=float, help='Supervised style loss lambda')
+    parser.add_argument('--temp', default=0.1, type=float, help='contrastive temperature')
+
+    parser.add_argument('--clamp_content_loss', default=None, type=float, help='Clipping the content loss')
+    parser.add_argument( "--non_adv_train", action="store_true",
+                        help="dont train content adversarially, use neg of content loss")
+    parser.add_argument('--eval_embed', type=str, default='head', help='which embeddings to use in evaluation')
+    parser.add_argument('--style_loss_type', type=str, default='SupCon', help='which loss function for style loss computation')
+    # Logging Params
+    parser.add_argument('--output_dir', required=True, type=str, help='Path to save logs and checkpoints.')
+    parser.add_argument('--print_freq', default=100, type=int, help='Print the logs every x iterations.')
+    parser.add_argument('--saveckp_freq', default=5000, type=int, help='Save checkpoint every x iterations.')
+    parser.add_argument('--eval_freq', default=5000, type=int, help='Eval the model every x iterations.')
+    parser.add_argument('--eval_k', type=int, nargs='+', default=[1, 5, 100], help='eval map and recall at these k values.')
+
+    # Misc
+    parser.add_argument("--resume_if_available", action="store_true")
+    parser.add_argument("--seed", default=42, type=int,
+                        help="seed for initializing training. ")
+    parser.add_argument("-j", "--workers", default=4, type=int,
+                        help="number of data loading workers (default: 32)")
+    parser.add_argument("--rank", default=-1, type=int,
+                        help="node rank for distributed training")
+    parser.add_argument("--dist_url", default="env://",
+                        help="url used to set up distributed training")
+    parser.add_argument("--local_rank", default=0, type=int,
+                        help="Please ignore and do not set this argument.")
+    return parser
+
+
+def sample_infinite_data(loader, seed=0):
+    rng = torch.Generator()
+    rng.manual_seed(seed)
+    BIG_NUMBER = 9999999999999
+    while True:
+        # Randomize dataloader indices before every epoch:
+        try:  # Only relevant for distributed sampler:
+            shuffle_seed = torch.randint(0, BIG_NUMBER, (1,), generator=rng).item()
+            loader.sampler.set_epoch(shuffle_seed)
+        except AttributeError:
+            pass
+        for batch in loader:
+            yield batch
+
+
+def main():
+    parser = argparse.ArgumentParser('CSD', parents=[get_args_parser()])
+    args = parser.parse_args()
+
+    if args.non_adv_train:
+        assert args.clamp_content_loss is not None, 'You have to clamp content loss in non-adv style of training'
+    utils.init_distributed_mode(args)
+    if args.seed is not None:
+        utils.fix_random_seeds(args.seed)
+
+    print("\n".join("%s: %s" % (k, str(v)) for k, v in sorted(dict(vars(args)).items())))
+    cudnn.benchmark = True
+
+    # ======================= setup logging =======================
+    if utils.is_main_process() and args.iters > 0:
+        os.makedirs(args.output_dir, exist_ok=True)
+
+    # ======================= preparing data =======================
+    if args.lambda_c < 1e-3:
+        train_transforms = ContrastiveTransformations(transforms_branch1, transforms_branch1, transforms_branch2)
+    else:
+        train_transforms = ContrastiveTransformations(transforms_branch0, transforms_branch1, transforms_branch2)
+
+    if args.train_set == 'wikiart':
+        train_dataset = WikiArtTrain(
+            args.train_path, 'database',
+            transform=train_transforms)
+    elif args.train_set == 'laion':
+        train_dataset = LAION(
+            args.train_path, args.train_anno_path,
+            min_images_per_label=args.min_images_per_label,
+            max_images_per_label=args.max_images_per_label,
+            transform=train_transforms)
+    elif args.train_set == 'laion_dedup':
+        train_dataset = LAIONDedup(
+            args.train_path, args.train_anno_path,
+            transform=train_transforms)
+    else:
+        raise NotImplementedError
+
+    train_sampler = torch.utils.data.DistributedSampler(train_dataset, shuffle=True)
+    train_loader = torch.utils.data.DataLoader(
+        train_dataset, batch_size=args.batch_size_per_gpu, drop_last=True,
+        num_workers=args.workers, pin_memory=True, sampler=train_sampler)
+    train_loader = sample_infinite_data(train_loader, args.seed)
+
+    if args.eval_set == 'wikiart':
+        vq_dataset = WikiArtTrain(
+            args.eval_path, 'query', transform=transforms_branch0, maxsize=args.maxsize)
+        vidx_dataset = WikiArtTrain(
+            args.eval_path, 'database', transform=transforms_branch0, maxsize=8*args.maxsize)
+
+    vq_loader = torch.utils.data.DataLoader(
+        vq_dataset, batch_size=2*args.batch_size_per_gpu, drop_last=True,
+        num_workers=min(args.workers, 2), pin_memory=True, shuffle=False)
+    vidx_loader = torch.utils.data.DataLoader(
+        vidx_dataset, batch_size=2*args.batch_size_per_gpu, drop_last=True,
+        num_workers=min(args.workers, 2), pin_memory=True, shuffle=False)
+    print(f"Data loaded: there are {len(train_dataset)} train images.")
+    print(f"Data loaded: there are {len(vq_dataset)} query and {len(vidx_dataset)} index images.")
+
+    # ======================= building model =======================
+    model = CSD_CLIP(args.arch, args.content_proj_head) # TODO: projection dim into hyperparam
+    model = model.cuda()
+    # synchronize batch norms (if any)
+    if utils.has_batchnorms(model):
+        model = nn.SyncBatchNorm.convert_sync_batchnorm(model)
+
+    if args.distributed:
+        model = nn.parallel.DistributedDataParallel(model, device_ids=[args.gpu])
+        model_without_ddp = model.module
+    else:
+        model_without_ddp = model
+
+    print(f"Model built with {args.arch} network.")
+
+    # ======================= setup loss and optimizers =======================
+    loss_content = SupConLoss(temperature=args.temp) # TODO: Do we want 2 diff 
+    loss_style = SupConLoss(temperature=args.temp)
+
+    params_groups = utils.get_params_groups(model_without_ddp.backbone)
+    # lr is set by scheduler
+    opt_bb = torch.optim.SGD(
+        params_groups, lr=0, momentum=0.9, weight_decay=args.weight_decay)
+
+    if args.content_proj_head != 'default':
+        opt_proj = torch.optim.SGD(
+            [{'params': model_without_ddp.last_layer_style},
+            {'params': model_without_ddp.last_layer_content.parameters()},],
+            # [model_without_ddp.last_layer_style, *model_without_ddp.last_layer_content.parameters()],
+            lr=0, momentum=0.9, weight_decay=0, # we do not apply weight decay
+        )
+    else:
+        opt_proj = torch.optim.SGD(
+            [model_without_ddp.last_layer_style, model_without_ddp.last_layer_content],
+            lr=0, momentum=0.9, weight_decay=0, # we do not apply weight decay
+        )
+
+    fp16_scaler = None
+    if args.use_fp16:
+        fp16_scaler = torch.cuda.amp.GradScaler()
+
+    # ======================= init schedulers =======================
+    if args.lr_scheduler_type =='cosine':
+        lr_schedule_bb = utils.cosine_scheduler(
+            args.lr_bb * (args.batch_size_per_gpu * utils.get_world_size()) / 256.,  # linear scaling rule
+            min(args.min_lr, args.lr_bb),
+            max(args.iters, 1), warmup_iters=min(args.warmup_iters, args.iters)
+        )
+
+        lr_schedule_proj = utils.cosine_scheduler(
+            args.lr * (args.batch_size_per_gpu * utils.get_world_size()) / 256.,  # linear scaling rule
+            min(args.min_lr, args.lr),
+            max(args.iters, 1), warmup_iters=min(args.warmup_iters, args.iters)
+        )
+    elif args.lr_scheduler_type =='constant_with_warmup':
+        lr_schedule_bb = utils.constant_with_warmup_scheduler(
+            args.lr_bb * (args.batch_size_per_gpu * utils.get_world_size()) / 256.,  # linear scaling rule
+            max(args.iters, 1), warmup_iters=min(args.warmup_iters, args.iters),
+        )
+
+        lr_schedule_proj = utils.constant_with_warmup_scheduler(
+            args.lr * (args.batch_size_per_gpu * utils.get_world_size()) / 256.,  # linear scaling rule
+            max(args.iters, 1), warmup_iters=min(args.warmup_iters, args.iters),
+        )
+    else:
+        print('Using constant LR for training')
+        lr_schedule_bb = utils.constant_with_warmup_scheduler(
+            args.lr_bb * (args.batch_size_per_gpu * utils.get_world_size()) / 256.,  # linear scaling rule
+            max(args.iters, 1), warmup_iters=0,
+        )
+
+        lr_schedule_proj = utils.constant_with_warmup_scheduler(
+            args.lr * (args.batch_size_per_gpu * utils.get_world_size()) / 256.,  # linear scaling rule
+            max(args.iters, 1), warmup_iters=0,
+        )
+
+    print(f"Loss, optimizer and schedulers ready.")
+
+    # ======================= optionally resume training =======================
+    to_restore = {"iter": 0}
+    if args.resume_if_available:
+        if not args.output_dir.endswith(".pth"):
+            ckpt_path = os.path.join(args.output_dir, "checkpoint.pth")
+        else:
+            ckpt_path = args.output_dir
+        utils.restart_from_checkpoint(
+            ckpt_path,
+            run_variables=to_restore,
+            model_state_dict=model,
+            opt_bb=opt_bb,
+            opt_proj=opt_proj,
+            fp16_scaler=fp16_scaler,
+        )
+        print(f"Start iter: {to_restore['iter']}")
+    start_iter = to_restore["iter"]
+    save_dict = None
+    print("Running eval before training!")
+    val_stats = evaluate(model, vq_loader, vidx_loader, fp16_scaler is not None, args.eval_k, args.eval_embed)
+    if start_iter >= args.iters:
+        print(f"Start iter {start_iter} >= Max iters {args.iters} training!")
+        return
+
+    start_time = time.time()
+    print("Starting CSD training !")
+    metric_logger = utils.MetricLogger(delimiter="  ", max_len=args.iters)
+    header = 'Iter:'
+    
+    #TODO: Check if we need to set model to train mode
+    model.eval()
+    for iter, batch in enumerate(metric_logger.log_every(train_loader, 100, header)):
+        # ======================= training =======================
+
+        if iter < start_iter:
+            continue
+
+        if iter >= args.iters:
+            break
+
+        # update learning rates according to their schedule
+        # it = len(train_loader) * epoch + it  # global training iteration
+        p = float(iter) / args.iters
+
+        for param_group in opt_bb.param_groups:
+            param_group["lr"] = lr_schedule_bb[iter]
+
+        for param_group in opt_proj.param_groups:
+            param_group["lr"] = lr_schedule_proj[iter]
+        if args.non_adv_train:
+            alpha = None
+        else:
+            alpha = 2. / (1. + np.exp(-10 * p)) - 1
+        images, artists, *_ = batch
+        if args.lambda_c < 1e-3:
+            images = torch.cat([images[0],images[1]], dim=0)
+        else:
+            images = torch.cat(images, dim=0)
+
+        # import torchvision
+        # torchvision.utils.save_image(images,'./temp.png')
+        images= images.cuda(non_blocking=True)
+        artists = artists.cuda(non_blocking=True).float()
+
+        with torch.cuda.amp.autocast(fp16_scaler is not None):
+            _ , content_output, style_output = model(images, alpha)
+
+            # Normalize the output features for each image
+            content_output = nn.functional.normalize(content_output, dim=1, p=2)
+            style_output = nn.functional.normalize(style_output, dim=1, p=2)
+
+            # Split the output features for each image and its views
+            style_output  = utils.split_reshape(style_output, args.batch_size_per_gpu, [0, 1])
+            content_output  = utils.split_reshape(content_output, args.batch_size_per_gpu, [0, -1])
+
+            # Gather tensors from all GPUs
+            if args.use_distributed_loss:
+                style_output = torch.cat(utils.GatherLayer.apply(style_output), dim=0)
+                content_output = torch.cat(utils.GatherLayer.apply(content_output), dim=0)
+
+            # Compute content loss (SimCLR loss, doesn't use labels)
+            loss_c = loss_content(content_output)
+            if args.clamp_content_loss is not None:
+                loss_c = loss_c.clamp(max = args.clamp_content_loss)
+                if args.non_adv_train:
+                    loss_c = -1 * loss_c
+
+            # Compute style loss
+            if args.use_distributed_loss:
+                artists = torch.cat(utils.GatherLayer.apply(artists), dim=0)
+
+            label_mask = artists @ artists.t()
+            if args.style_loss_type == 'SimClr':
+                loss_s_ssl = loss_style(style_output)
+                loss_s_sup = torch.Tensor([0]).to(model.device)
+            elif args.style_loss_type == 'OnlySup':
+                loss_s_ssl = torch.Tensor([0]).to(model.device)
+                loss_s_sup = loss_style(style_output[:, 0:1, :], mask=label_mask)
+            else:
+                loss_s_sup = loss_style(style_output[:, 0:1, :], mask=label_mask)
+                loss_s_ssl = loss_style(style_output)
+
+            loss_s = args.lam_sup*loss_s_sup + loss_s_ssl
+
+        loss = args.lambda_c * loss_c + args.lambda_s * loss_s
+
+        if not math.isfinite(loss.item()):
+            print("Loss is {}, stopping training".format(loss.item()))
+            sys.exit(1)
+
+        opt_bb.zero_grad()
+        opt_proj.zero_grad()
+        param_norms = None
+        if fp16_scaler is None:
+            loss.backward()
+            if args.clip_grad:
+                param_norms = utils.clip_gradients(model, args.clip_grad)
+            utils.cancel_gradients_last_layer(iter, model, args.freeze_last_layer)
+            opt_bb.step()
+            opt_proj.step()
+        else:
+            fp16_scaler.scale(loss).backward()
+            if args.clip_grad:
+                fp16_scaler.unscale_(opt_bb)  # unscale the gradients of optimizer's assigned params in-place
+                fp16_scaler.unscale_(opt_proj)
+                param_norms = utils.clip_gradients(model, args.clip_grad)
+            utils.cancel_gradients_last_layer(iter, model, args.freeze_last_layer)
+            fp16_scaler.step(opt_bb)
+            fp16_scaler.step(opt_proj)
+            fp16_scaler.update()
+
+        # logging
+        torch.cuda.synchronize()
+        metric_logger.update(loss=loss.item())
+        metric_logger.update(content_loss=loss_c.item())
+        metric_logger.update(style_loss=loss_s.item())
+        metric_logger.update(style_loss_sup=loss_s_sup.item())
+        metric_logger.update(style_loss_ssl=loss_s_ssl.item())
+        metric_logger.update(lr_bb=opt_bb.param_groups[0]["lr"])
+        # metric_logger.update(wd_bb=opt_bb.param_groups[0]["weight_decay"])
+        metric_logger.update(lr_proj=opt_proj.param_groups[0]["lr"])
+        # metric_logger.update(wd_proj=opt_proj.param_groups[0]["weight_decay"])
+
+        # ============ writing logs ... ============
+        save_dict = {
+            'model_state_dict': model.state_dict(),
+            'opt_bb': opt_bb.state_dict(),
+            'opt_proj': opt_proj.state_dict(),
+            'iter': iter+1,
+            'args': args,
+        }
+        if fp16_scaler is not None:
+            save_dict['fp16_scaler'] = fp16_scaler.state_dict()
+
+        if (iter+1) % args.saveckp_freq == 0:
+            utils.save_on_master(save_dict, os.path.join(args.output_dir, 'checkpoint.pth'))
+            utils.save_on_master(save_dict, os.path.join(args.output_dir, f'checkpoint{iter+1:08}.pth'))
+
+        train_stats = {k: meter.global_avg for k, meter in metric_logger.meters.items()}
+        log_stats = {**{f'train_{k}': v for k, v in train_stats.items()},
+                    'iter': iter+1}
+
+        if utils.is_main_process() and (iter+1) % args.print_freq == 0:
+            with (Path(args.output_dir) / "log.txt").open("a") as f:
+                f.write(json.dumps(log_stats) + "\n")
+
+        # Eval
+        if (iter+1) % args.eval_freq==0:
+            # gather the stats from all processes
+            metric_logger.synchronize_between_processes()
+            print("Averaged stats:", metric_logger)
+
+            val_stats = evaluate(model, vq_loader, vidx_loader, fp16_scaler is not None, args.eval_k, args.eval_embed)
+
+    if args.iters > 0 and save_dict is not None:
+        utils.save_on_master(save_dict, os.path.join(args.output_dir, 'checkpoint.pth'))
+
+    total_time = time.time() - start_time
+    total_time_str = str(datetime.timedelta(seconds=int(total_time)))
+    print('Training time {}'.format(total_time_str))
+
+
+def evaluate(model, vq_loader, vidx_loader, use_fp16=False, eval_k=[1, 5, 100], eval_embed='head'):
+    metric_logger = utils.MetricLogger(delimiter="  ")
+    # Valid loader is the query set
+    # Train loader is the search set
+    use_cuda = True
+    db_features = utils.extract_features(model, vidx_loader,use_cuda, use_fp16, eval_embed)
+    q_features = utils.extract_features(model, vq_loader, use_cuda, use_fp16, eval_embed)
+
+    # Aggregate style features across GPUs
+    if utils.get_rank() != 0:
+        return
+
+    # Find the nearest neighbor indices for each query
+    similarities = q_features @ db_features.T
+    similarities = torch.argsort(similarities, dim=1, descending=True).cpu()
+
+    # Map neighbor indices to labels (assuming one hot labels)
+    q_labels = vq_loader.dataset.labels
+    db_labels = vidx_loader.dataset.labels
+    gts = q_labels @ db_labels.T
+    #TODO: vectorize this
+    preds = np.array([gts[i][similarities[i]] for i in range(len(gts))])
+
+    # Compute metrics
+    for topk in eval_k:
+        mode_recall = utils.Metrics.get_recall_bin(copy.deepcopy(preds), topk)
+        mode_mrr = utils.Metrics.get_mrr_bin(copy.deepcopy(preds), topk)
+        mode_map = utils.Metrics.get_map_bin(copy.deepcopy(preds), topk)
+        # print(f'Recall@{topk}: {mode_recall:.2f}, mAP@{topk}: {mode_map:.2f}')
+        metric_logger.update(**{f'recall@{topk}': mode_recall, f'mAP@{topk}': mode_map, f'MRR@{topk}': mode_mrr})
+
+    # gather the stats from all processes
+    print("Averaged stats:", metric_logger)
+    return {k: meter.global_avg for k, meter in metric_logger.meters.items()}
+
+
+if __name__ == "__main__":
+    main()

CSD/CSD/utils.py

@@ -0,0 +1,853 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Misc functions.
+
+Mostly copy-paste from torchvision references or other public repos like DETR:
+https://github.com/facebookresearch/detr/blob/master/util/misc.py
+"""
+import os
+import sys
+import time
+import math
+import random
+import datetime
+import subprocess
+from collections import defaultdict, deque, OrderedDict
+
+import numpy as np
+import torch
+from torch import nn
+import torch.distributed as dist
+import warnings
+import argparse
+from PIL import ImageFilter, ImageOps
+
+
+class GaussianBlur(object):
+    """
+    Apply Gaussian Blur to the PIL image.
+    """
+
+    def __init__(self, p=0.5, radius_min=0.1, radius_max=2.):
+        self.prob = p
+        self.radius_min = radius_min
+        self.radius_max = radius_max
+
+    def __call__(self, img):
+        do_it = random.random() <= self.prob
+        if not do_it:
+            return img
+
+        return img.filter(
+            ImageFilter.GaussianBlur(
+                radius=random.uniform(self.radius_min, self.radius_max)
+            )
+        )
+
+
+class Solarization(object):
+    """
+    Apply Solarization to the PIL image.
+    """
+
+    def __init__(self, p):
+        self.p = p
+
+    def __call__(self, img):
+        if random.random() < self.p:
+            return ImageOps.solarize(img)
+        else:
+            return img
+
+
+def clip_gradients(model, clip):
+    norms = []
+    for name, p in model.named_parameters():
+        if p.grad is not None:
+            param_norm = p.grad.data.norm(2)
+            norms.append(param_norm.item())
+            clip_coef = clip / (param_norm + 1e-6)
+            if clip_coef < 1:
+                p.grad.data.mul_(clip_coef)
+    return norms
+
+
+def restart_from_checkpoint(ckp_path, run_variables=None, **kwargs):
+    """
+    Re-start from checkpoint
+    """
+    if not os.path.isfile(ckp_path):
+        return
+    print("Found checkpoint at {}".format(ckp_path))
+
+    # open checkpoint file
+    checkpoint = torch.load(ckp_path, map_location="cpu")
+
+    # key is what to look for in the checkpoint file
+    # value is the object to load
+    # example: {'state_dict': model}
+    for key, value in kwargs.items():
+        if key in checkpoint and value is not None:
+            try:
+                msg = value.load_state_dict(checkpoint[key], strict=False)
+                print("=> loaded '{}' from checkpoint '{}' with msg {}".format(key, ckp_path, msg))
+            except TypeError:
+                try:
+                    msg = value.load_state_dict(checkpoint[key])
+                    print("=> loaded '{}' from checkpoint: '{}'".format(key, ckp_path))
+                except ValueError:
+                    print("=> failed to load '{}' from checkpoint: '{}'".format(key, ckp_path))
+        else:
+            print("=> key '{}' not found in checkpoint: '{}'".format(key, ckp_path))
+
+    # re load variable important for the run
+    if run_variables is not None:
+        for var_name in run_variables:
+            if var_name in checkpoint:
+                run_variables[var_name] = checkpoint[var_name]
+
+
+def convert_state_dict(state_dict):
+    new_state_dict = OrderedDict()
+    for k, v in state_dict.items():
+        if k.startswith("module."):
+            k = k.replace("module.", "")
+        new_state_dict[k] = v
+    return new_state_dict
+
+
+def cosine_scheduler(base_value, final_value, iters, warmup_iters, start_warmup_value=0):
+    warmup_schedule = np.array([])
+
+    if warmup_iters > 0:
+        warmup_schedule = np.linspace(start_warmup_value, base_value, warmup_iters)
+
+    post_warmup_iters = np.arange(iters - warmup_iters)
+    schedule = final_value + 0.5 * (base_value - final_value) * (
+                1 + np.cos(np.pi * post_warmup_iters / len(post_warmup_iters)))
+
+    schedule = np.concatenate((warmup_schedule, schedule))
+    assert len(schedule) == iters
+    return schedule
+
+
+def constant_with_warmup_scheduler(base_value, iters, warmup_iters=0, start_warmup_value=0):
+    warmup_schedule = np.array([])
+
+    if warmup_iters > 0:
+        warmup_schedule = np.linspace(start_warmup_value, base_value, warmup_iters)
+
+    num_iters = iters - warmup_iters
+    schedule = np.array([base_value] * num_iters)
+
+    schedule = np.concatenate((warmup_schedule, schedule))
+    assert len(schedule) == iters
+    return schedule
+
+
+def bool_flag(s):
+    """
+    Parse boolean arguments from the command line.
+    """
+    FALSY_STRINGS = {"off", "false", "0"}
+    TRUTHY_STRINGS = {"on", "true", "1"}
+    if s.lower() in FALSY_STRINGS:
+        return False
+    elif s.lower() in TRUTHY_STRINGS:
+        return True
+    else:
+        raise argparse.ArgumentTypeError("invalid value for a boolean flag")
+
+
+def fix_random_seeds(seed=31):
+    """
+    Fix random seeds.
+    """
+    torch.manual_seed(seed)
+    torch.cuda.manual_seed_all(seed)
+    np.random.seed(seed)
+
+
+class SmoothedValue(object):
+    """Track a series of values and provide access to smoothed values over a
+    window or the global series average.
+    """
+
+    def __init__(self, window_size=20, fmt=None):
+        if fmt is None:
+            fmt = "{median:.6f} ({global_avg:.6f})"
+        self.deque = deque(maxlen=window_size)
+        self.total = 0.0
+        self.count = 0
+        self.fmt = fmt
+
+    def update(self, value, n=1):
+        self.deque.append(value)
+        self.count += n
+        self.total += value * n
+
+    def synchronize_between_processes(self):
+        """
+        Warning: does not synchronize the deque!
+        """
+        if not is_dist_avail_and_initialized():
+            return
+        t = torch.tensor([self.count, self.total], dtype=torch.float64, device='cuda')
+        dist.barrier()
+        dist.all_reduce(t)
+        t = t.tolist()
+        self.count = int(t[0])
+        self.total = t[1]
+
+    @property
+    def median(self):
+        d = torch.tensor(list(self.deque))
+        return d.median().item()
+
+    @property
+    def avg(self):
+        d = torch.tensor(list(self.deque), dtype=torch.float32)
+        return d.mean().item()
+
+    @property
+    def global_avg(self):
+        return self.total / self.count
+
+    @property
+    def max(self):
+        return max(self.deque)
+
+    @property
+    def value(self):
+        return self.deque[-1]
+
+    def __str__(self):
+        return self.fmt.format(
+            median=self.median,
+            avg=self.avg,
+            global_avg=self.global_avg,
+            max=self.max,
+            value=self.value)
+
+
+def reduce_dict(input_dict, average=True):
+    """
+    Args:
+        input_dict (dict): all the values will be reduced
+        average (bool): whether to do average or sum
+    Reduce the values in the dictionary from all processes so that all processes
+    have the averaged results. Returns a dict with the same fields as
+    input_dict, after reduction.
+    """
+    world_size = get_world_size()
+    if world_size < 2:
+        return input_dict
+    with torch.no_grad():
+        names = []
+        values = []
+        # sort the keys so that they are consistent across processes
+        for k in sorted(input_dict.keys()):
+            names.append(k)
+            values.append(input_dict[k])
+        values = torch.stack(values, dim=0)
+        dist.all_reduce(values)
+        if average:
+            values /= world_size
+        reduced_dict = {k: v for k, v in zip(names, values)}
+    return reduced_dict
+
+
+class MetricLogger(object):
+    def __init__(self, delimiter="\t", max_len=100):
+        self.meters = defaultdict(SmoothedValue)
+        self.delimiter = delimiter
+        self.max_len = max_len
+
+    def update(self, **kwargs):
+        for k, v in kwargs.items():
+            if isinstance(v, torch.Tensor):
+                v = v.item()
+            assert isinstance(v, (float, int))
+            self.meters[k].update(v)
+
+    def __getattr__(self, attr):
+        if attr in self.meters:
+            return self.meters[attr]
+        if attr in self.__dict__:
+            return self.__dict__[attr]
+        raise AttributeError("'{}' object has no attribute '{}'".format(
+            type(self).__name__, attr))
+
+    def __str__(self):
+        loss_str = []
+        for name, meter in self.meters.items():
+            loss_str.append(
+                "{}: {}".format(name, str(meter))
+            )
+        return self.delimiter.join(loss_str)
+
+    def synchronize_between_processes(self):
+        for meter in self.meters.values():
+            meter.synchronize_between_processes()
+
+    def add_meter(self, name, meter):
+        self.meters[name] = meter
+
+    def log_every(self, iterable, print_freq, header=None):
+        i = 0
+        if not header:
+            header = ''
+        start_time = time.time()
+        end = time.time()
+        iter_time = SmoothedValue(fmt='{avg:.6f}')
+        data_time = SmoothedValue(fmt='{avg:.6f}')
+        space_fmt = ':' + str(len(str(self.max_len))) + 'd'
+        if torch.cuda.is_available():
+            log_msg = self.delimiter.join([
+                header,
+                '[{0' + space_fmt + '}/{1}]',
+                'eta: {eta}',
+                '{meters}',
+                'time: {time}',
+                'data: {data}',
+                'max mem: {memory:.0f}'
+            ])
+        else:
+            log_msg = self.delimiter.join([
+                header,
+                '[{0' + space_fmt + '}/{1}]',
+                'eta: {eta}',
+                '{meters}',
+                'time: {time}',
+                'data: {data}'
+            ])
+        MB = 1024.0 * 1024.0
+        for obj in iterable:
+            data_time.update(time.time() - end)
+            yield obj
+            iter_time.update(time.time() - end)
+            if i % print_freq == 0 or i == self.max_len - 1:
+                eta_seconds = iter_time.global_avg * (self.max_len - i)
+                eta_string = str(datetime.timedelta(seconds=int(eta_seconds)))
+                if torch.cuda.is_available():
+                    print(log_msg.format(
+                        i, self.max_len,
+                        eta=eta_string,
+                        meters=str(self),
+                        time=str(iter_time), data=str(data_time),
+                        memory=torch.cuda.max_memory_allocated() / MB))
+                else:
+                    print(log_msg.format(
+                        i, self.max_len,
+                        eta=eta_string,
+                        meters=str(self),
+                        time=str(iter_time), data=str(data_time)))
+            i += 1
+            end = time.time()
+        total_time = time.time() - start_time
+        total_time_str = str(datetime.timedelta(seconds=int(total_time)))
+        print('{} Total time: {} ({:.6f} s / it)'.format(
+            header, total_time_str, total_time / self.max_len))
+
+
+def get_sha():
+    cwd = os.path.dirname(os.path.abspath(__file__))
+
+    def _run(command):
+        return subprocess.check_output(command, cwd=cwd).decode('ascii').strip()
+
+    sha = 'N/A'
+    diff = "clean"
+    branch = 'N/A'
+    try:
+        sha = _run(['git', 'rev-parse', 'HEAD'])
+        subprocess.check_output(['git', 'diff'], cwd=cwd)
+        diff = _run(['git', 'diff-index', 'HEAD'])
+        diff = "has uncommited changes" if diff else "clean"
+        branch = _run(['git', 'rev-parse', '--abbrev-ref', 'HEAD'])
+    except Exception:
+        pass
+    message = f"sha: {sha}, status: {diff}, branch: {branch}"
+    return message
+
+
+def is_dist_avail_and_initialized():
+    if not dist.is_available():
+        return False
+    if not dist.is_initialized():
+        return False
+    return True
+
+
+def get_world_size():
+    if not is_dist_avail_and_initialized():
+        return 1
+    return dist.get_world_size()
+
+
+def get_rank():
+    if not is_dist_avail_and_initialized():
+        return 0
+    return dist.get_rank()
+
+
+def is_main_process():
+    return get_rank() == 0
+
+
+def save_on_master(*args, **kwargs):
+    if is_main_process():
+        torch.save(*args, **kwargs)
+
+
+def setup_for_distributed(is_master):
+    """
+    This function disables printing when not in master process
+    """
+    import builtins as __builtin__
+    builtin_print = __builtin__.print
+
+    def print(*args, **kwargs):
+        force = kwargs.pop('force', False)
+        if is_master or force:
+            builtin_print(*args, **kwargs)
+
+    __builtin__.print = print
+
+
+def init_distributed_mode(args):
+    # launched with torch.distributed.launch
+    if 'RANK' in os.environ and 'WORLD_SIZE' in os.environ:
+        args.rank = int(os.environ["RANK"])
+        args.world_size = int(os.environ['WORLD_SIZE'])
+        args.gpu = int(os.environ['LOCAL_RANK'])
+    # launched with submitit on a slurm cluster
+    elif 'SLURM_PROCID' in os.environ:
+        args.rank = int(os.environ['SLURM_PROCID'])
+        args.gpu = args.rank % torch.cuda.device_count()
+    # launched naively with `python main_dino.py`
+    # we manually add MASTER_ADDR and MASTER_PORT to env variables
+    elif torch.cuda.is_available():
+        print('Will run the code on one GPU.')
+        args.rank, args.gpu, args.world_size = 0, 0, 1
+        os.environ['MASTER_ADDR'] = '127.0.0.1'
+        os.environ['MASTER_PORT'] = '29500'
+    else:
+        print('Does not support training without GPU.')
+        sys.exit(1)
+
+    if torch.cuda.device_count() > 0:
+        args.distributed = True
+    else:
+        args.distributed = False
+
+    dist.init_process_group(
+        backend="nccl",
+        init_method=args.dist_url,
+        world_size=args.world_size,
+        rank=args.rank,
+    )
+
+    torch.cuda.set_device(args.gpu)
+    print('| distributed init (rank {}): {}'.format(
+        args.rank, args.dist_url), flush=True)
+    dist.barrier()
+    setup_for_distributed(args.rank == 0)
+
+
+def accuracy(output, target, topk=(1,)):
+    """Computes the accuracy over the k top predictions for the specified values of k"""
+    maxk = max(topk)
+    batch_size = target.size(0)
+    _, pred = output.topk(maxk, 1, True, True)
+    pred = pred.t()
+    correct = pred.eq(target.reshape(1, -1).expand_as(pred))
+    return [correct[:k].reshape(-1).float().sum(0) * 100. / batch_size for k in topk]
+
+
+def _no_grad_trunc_normal_(tensor, mean, std, a, b):
+    # Cut & paste from PyTorch official master until it's in a few official releases - RW
+    # Method based on https://people.sc.fsu.edu/~jburkardt/presentations/truncated_normal.pdf
+    def norm_cdf(x):
+        # Computes standard normal cumulative distribution function
+        return (1. + math.erf(x / math.sqrt(2.))) / 2.
+
+    if (mean < a - 2 * std) or (mean > b + 2 * std):
+        warnings.warn("mean is more than 2 std from [a, b] in nn.init.trunc_normal_. "
+                      "The distribution of values may be incorrect.",
+                      stacklevel=2)
+
+    with torch.no_grad():
+        # Values are generated by using a truncated uniform distribution and
+        # then using the inverse CDF for the normal distribution.
+        # Get upper and lower cdf values
+        l = norm_cdf((a - mean) / std)
+        u = norm_cdf((b - mean) / std)
+
+        # Uniformly fill tensor with values from [l, u], then translate to
+        # [2l-1, 2u-1].
+        tensor.uniform_(2 * l - 1, 2 * u - 1)
+
+        # Use inverse cdf transform for normal distribution to get truncated
+        # standard normal
+        tensor.erfinv_()
+
+        # Transform to proper mean, std
+        tensor.mul_(std * math.sqrt(2.))
+        tensor.add_(mean)
+
+        # Clamp to ensure it's in the proper range
+        tensor.clamp_(min=a, max=b)
+        return tensor
+
+
+def trunc_normal_(tensor, mean=0., std=1., a=-2., b=2.):
+    # type: (Tensor, float, float, float, float) -> Tensor
+    return _no_grad_trunc_normal_(tensor, mean, std, a, b)
+
+
+class LARS(torch.optim.Optimizer):
+    """
+    Almost copy-paste from https://github.com/facebookresearch/barlowtwins/blob/main/main.py
+    """
+
+    def __init__(self, params, lr=0, weight_decay=0, momentum=0.9, eta=0.001,
+                 weight_decay_filter=None, lars_adaptation_filter=None):
+        defaults = dict(lr=lr, weight_decay=weight_decay, momentum=momentum,
+                        eta=eta, weight_decay_filter=weight_decay_filter,
+                        lars_adaptation_filter=lars_adaptation_filter)
+        super().__init__(params, defaults)
+
+    @torch.no_grad()
+    def step(self):
+        for g in self.param_groups:
+            for p in g['params']:
+                dp = p.grad
+
+                if dp is None:
+                    continue
+
+                if p.ndim != 1:
+                    dp = dp.add(p, alpha=g['weight_decay'])
+
+                if p.ndim != 1:
+                    param_norm = torch.norm(p)
+                    update_norm = torch.norm(dp)
+                    one = torch.ones_like(param_norm)
+                    q = torch.where(param_norm > 0.,
+                                    torch.where(update_norm > 0,
+                                                (g['eta'] * param_norm / update_norm), one), one)
+                    dp = dp.mul(q)
+
+                param_state = self.state[p]
+                if 'mu' not in param_state:
+                    param_state['mu'] = torch.zeros_like(p)
+                mu = param_state['mu']
+                mu.mul_(g['momentum']).add_(dp)
+
+                p.add_(mu, alpha=-g['lr'])
+
+
+class MultiCropWrapper(nn.Module):
+    """
+    Perform forward pass separately on each resolution input.
+    The inputs corresponding to a single resolution are clubbed and single
+    forward is run on the same resolution inputs. Hence we do several
+    forward passes = number of different resolutions used. We then
+    concatenate all the output features and run the head forward on these
+    concatenated features.
+    """
+
+    def __init__(self, backbone, head):
+        super(MultiCropWrapper, self).__init__()
+        # disable layers dedicated to ImageNet labels classification
+        backbone.fc, backbone.head = nn.Identity(), nn.Identity()
+        self.backbone = backbone
+        self.head = head
+
+    def forward(self, x):
+        # convert to list
+        if not isinstance(x, list):
+            x = [x]
+        idx_crops = torch.cumsum(torch.unique_consecutive(
+            torch.tensor([inp.shape[-1] for inp in x]),
+            return_counts=True,
+        )[1], 0)
+        start_idx, output = 0, torch.empty(0).to(x[0].device)
+        for end_idx in idx_crops:
+            _out = self.backbone(torch.cat(x[start_idx: end_idx]))
+            # The output is a tuple with XCiT model. See:
+            # https://github.com/facebookresearch/xcit/blob/master/xcit.py#L404-L405
+            if isinstance(_out, tuple):
+                _out = _out[0]
+            # accumulate outputs
+            output = torch.cat((output, _out))
+            start_idx = end_idx
+        # Run the head forward on the concatenated features.
+        return self.head(output)
+
+
+def get_params_groups(model):
+    regularized = []
+    not_regularized = []
+    for name, param in model.named_parameters():
+        if not param.requires_grad:
+            continue
+        # we do not regularize biases nor Norm parameters
+        if name.endswith(".bias") or len(param.shape) == 1:
+            not_regularized.append(param)
+        else:
+            regularized.append(param)
+    return [{'params': regularized}, {'params': not_regularized, 'weight_decay': 0.}]
+
+
+def cancel_gradients_last_layer(epoch, model, freeze_last_layer):
+    if epoch >= freeze_last_layer:
+        return
+    for n, p in model.named_parameters():
+        if "last_layer" in n:
+            p.grad = None
+
+
+def has_batchnorms(model):
+    bn_types = (nn.BatchNorm1d, nn.BatchNorm2d, nn.BatchNorm3d, nn.SyncBatchNorm)
+    for name, module in model.named_modules():
+        if isinstance(module, bn_types):
+            return True
+    return False
+
+
+#####
+def convert_weights_float(model: nn.Module):
+    """Convert applicable model parameters to fp32"""
+
+    def _convert_weights_to_fp32(l):
+        if isinstance(l, (nn.Conv1d, nn.Conv2d, nn.Linear)):
+            l.weight.data = l.weight.data.float()
+            if l.bias is not None:
+                l.bias.data = l.bias.data.float()
+
+        if isinstance(l, nn.MultiheadAttention):
+            for attr in [*[f"{s}_proj_weight" for s in ["in", "q", "k", "v"]], "in_proj_bias", "bias_k", "bias_v"]:
+                tensor = getattr(l, attr)
+                if tensor is not None:
+                    tensor.data = tensor.data.float()
+
+        for name in ["text_projection", "proj"]:
+            if hasattr(l, name):
+                attr = getattr(l, name)
+                if attr is not None:
+                    attr.data = attr.data.float()
+
+    model.apply(_convert_weights_to_fp32)
+
+
+def split_reshape(x, bs, combination=None):
+    n = len(x) // bs
+    assert n in [2, 3], "The num augs should be 2 or 3 in number"
+    f = torch.split(x, [bs] * n, dim=0)
+    if combination is None:
+        x_reshape = torch.cat([f[i].unsqueeze(1) for i in range(n)], dim=1)
+    else:
+        x_reshape = torch.cat([f[i].unsqueeze(1) for i in combination], dim=1)
+
+    # if repeatcase:
+    #     x_reshape = torch.cat([f1.unsqueeze(1), f1.unsqueeze(1)], dim=1)
+    return x_reshape
+
+
+class AverageMeter(object):
+    """Computes and stores the average and current value"""
+
+    def __init__(self, name, fmt=":f"):
+        self.name = name
+        self.fmt = fmt
+        self.reset()
+
+    def reset(self):
+        self.val = 0
+        self.avg = 0
+        self.sum = 0
+        self.count = 0
+
+    def update(self, val, n=1):
+        self.val = val
+        self.sum += val * n
+        self.count += n
+        self.avg = self.sum / self.count
+
+    def __str__(self):
+        fmtstr = "{name} {val" + self.fmt + "} ({avg" + self.fmt + "})"
+        return fmtstr.format(**self.__dict__)
+
+
+class ProgressMeter(object):
+    def __init__(self, num_batches, meters, prefix=""):
+        self.batch_fmtstr = self._get_batch_fmtstr(num_batches)
+        self.meters = meters
+        self.prefix = prefix
+
+    def display(self, batch):
+        entries = [self.prefix + self.batch_fmtstr.format(batch)]
+        entries += [str(meter) for meter in self.meters]
+        print("\t".join(entries))
+
+    def _get_batch_fmtstr(self, num_batches):
+        num_digits = len(str(num_batches // 1))
+        fmt = "{:" + str(num_digits) + "d}"
+        return "[" + fmt + "/" + fmt.format(num_batches) + "]"
+
+
+@torch.no_grad()
+def extract_features(model, data_loader, use_cuda=True, use_fp16=False, eval_embed='head'):
+    metric_logger = MetricLogger(delimiter="  ")
+    features = None
+    # count = 0
+    for samples, *_, index in metric_logger.log_every(data_loader, 100):
+        # print(f'At the index {index}')
+        samples = samples.cuda(non_blocking=True)
+        index = index.cuda(non_blocking=True)
+        if use_fp16:
+            with torch.cuda.amp.autocast():
+                bb_feats, cont_feats, style_feats = model(samples)
+
+                if eval_embed == 'backbone':
+                    feats = bb_feats.clone()
+                else:
+                    feats = style_feats.clone()
+
+        else:
+            bb_feats, cont_feats, style_feats = model(samples)
+            if eval_embed == 'backbone':
+                feats = bb_feats.clone()
+            else:
+                feats = style_feats.clone()
+        # init storage feature matrix
+        if dist.get_rank() == 0 and features is None:
+            features = torch.zeros(len(data_loader.dataset), feats.shape[-1], dtype=feats.dtype)
+            if use_cuda:
+                features = features.cuda(non_blocking=True)
+            print(f"Storing features into tensor of shape {features.shape}")
+
+        # get indexes from all processes
+        y_all = torch.empty(dist.get_world_size(), index.size(0), dtype=index.dtype, device=index.device)
+        y_l = list(y_all.unbind(0))
+        y_all_reduce = torch.distributed.all_gather(y_l, index, async_op=True)
+        y_all_reduce.wait()
+        index_all = torch.cat(y_l)
+
+        # share features between processes
+        feats_all = torch.empty(
+            dist.get_world_size(),
+            feats.size(0),
+            feats.size(1),
+            dtype=feats.dtype,
+            device=feats.device,
+        )
+        output_l = list(feats_all.unbind(0))
+        output_all_reduce = torch.distributed.all_gather(output_l, feats, async_op=True)
+        output_all_reduce.wait()
+
+        # update storage feature matrix
+        if dist.get_rank() == 0:
+            if use_cuda:
+                features.index_copy_(0, index_all, torch.cat(output_l))
+            else:
+                features.index_copy_(0, index_all.cpu(), torch.cat(output_l).cpu())
+    return features
+
+
+# Copy from https://github.com/learn2phoenix/dynamicDistances/blob/main/metrics/metrics.py
+class Metrics(object):
+    def __init__(self):
+        self.data = None
+
+    @staticmethod
+    def get_recall(preds, gts, topk=5):
+        preds = preds[:, :topk]
+        preds -= gts[:, None]
+        found = np.where(np.amin(np.absolute(preds), axis=1) == 0)[0]
+        return found.shape[0] / gts.shape[0]
+
+    @staticmethod
+    def get_mrr(preds, gts, topk=5):
+        preds = preds[:, :topk]
+        preds -= gts[:, None]
+        rows, cols = np.where(preds == 0)
+        _, unique_rows = np.unique(rows, return_index=True)
+        valid_cols = cols[unique_rows]
+        valid_cols += 1
+        return np.mean(1 / valid_cols)
+
+    @staticmethod
+    def get_map(preds, gts, topk=5):
+        preds = preds[:, :topk]
+        preds -= gts[:, None]
+        rows, cols = np.where(preds == 0)
+        _, unique_rows = np.unique(rows, return_index=True)
+        row_cols = np.split(cols, unique_rows)[1:]
+        row_cols = [np.hstack([x[0], np.diff(x), topk - x[-1]]) for x in row_cols]
+        row_cols = [np.pad(x, (0, topk + 1 - x.shape[0]), 'constant', constant_values=(0, 0)) for x in row_cols]
+        precision = np.asarray([np.repeat(np.arange(topk + 1), x) / np.arange(1, topk + 1) for x in row_cols])
+        return np.sum(np.mean(precision, axis=1)) / preds.shape[0]
+
+    @staticmethod
+    def get_recall_bin(preds, topk=5):
+        # preds is a binary matrix of size Q x K
+        preds = preds[:, :topk]
+        found = np.where(np.amax(preds, axis=1) == True)[0]
+        return found.shape[0] / preds.shape[0]
+
+    @staticmethod
+    def get_mrr_bin(preds, topk=5):
+        # preds is a binary matrix of size Q x K
+        preds = preds[:, :topk]
+        rows, cols = np.where(preds)
+        _, unique_rows = np.unique(rows, return_index=True)
+        valid_cols = cols[unique_rows]
+        valid_cols += 1
+        return np.mean(1 / valid_cols)
+
+    @staticmethod
+    def get_map_bin(preds, topk=5):
+        # preds is a binary matrix of size Q x K
+        preds = preds[:, :topk]
+        rows, cols = np.where(preds)
+        _, unique_rows = np.unique(rows, return_index=True)
+        row_cols = np.split(cols, unique_rows)[1:]
+        row_cols = [np.hstack([x[0], np.diff(x), topk - x[-1]]) for x in row_cols]
+        row_cols = [np.pad(x, (0, topk + 1 - x.shape[0]), 'constant', constant_values=(0, 0)) for x in row_cols]
+        precision = np.asarray([np.repeat(np.arange(topk + 1), x) / np.arange(1, topk + 1) for x in row_cols])
+        return np.sum(np.mean(precision, axis=1)) / preds.shape[0]
+
+
+class GatherLayer(torch.autograd.Function):
+    """Gather tensors from all process, supporting backward propagation.
+    """
+
+    @staticmethod
+    def forward(ctx, input):
+        ctx.save_for_backward(input)
+        output = [torch.zeros_like(input) \
+                  for _ in range(dist.get_world_size())]
+        dist.all_gather(output, input)
+        return tuple(output)
+
+    @staticmethod
+    def backward(ctx, *grads):
+        input, = ctx.saved_tensors
+        grad_out = torch.zeros_like(input)
+        grad_out[:] = grads[dist.get_rank()]
+        return grad_out

CSD/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 learn2phoenix
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

CSD/README.md

@@ -0,0 +1,91 @@
+# Measuring Style Similarity in Diffusion Models
+Check out the paper here - [arxiv](https://arxiv.org/abs/2404.01292).
+
+![alt text](github_teaser.jpg "Generations from Stable Diffusion and corresponding matches from LAION-Styles split")
+
+## Create and activate the environment
+
+```
+conda env create -f environment.yml
+conda activate style
+```
+
+## Download the pretrained weights for the CSD model
+
+Please download the CSD model (ViT-L) weights [here](https://drive.google.com/file/d/1FX0xs8p-C7Ob-h5Y4cUhTeOepHzXv_46/view?usp=sharing). 
+
+
+## Download the pretrained weights for the baseline models
+
+You need these only if you want to test the baseline numbers. For `CLIP` and `DINO`, pretrained weights will be downloaded automatically. For `SSCD` and `MoCo`, please download the weights
+from the links below and put them in `./pretrainedmodels` folder.
+
+* SSCD: [resnet50](https://dl.fbaipublicfiles.com/sscd-copy-detection/sscd_disc_mixup.torchscript.pt)
+* MoCO: [ViT-B](https://dl.fbaipublicfiles.com/moco-v3/vit-b-300ep/vit-b-300ep.pth.tar)
+
+
+
+## Download the WikiArt dataset
+WikiArt can be downloaded from [here](https://drive.google.com/file/d/1vTChp3nU5GQeLkPwotrybpUGUXj12BTK/view?usp=drivesdk0)
+or [here1](http://web.fsktm.um.edu.my/~cschan/source/ICIP2017/wikiart.zip)
+
+After dataset is downloaded please put `./wikiart.csv` in the parent directory of the dataset. The final directory structure should look like this:
+```
+path/to/WikiArt
+├── wikiart
+    ├── Abstract_Expressionism
+        ├── <filename>.jpg
+    ├── ...
+└── wikiart.csv
+```
+
+Also, make sure that you add a column `path` in the `wikiart.csv` file which contains the absolute path to the image.
+
+## Generate the embeddings
+
+Once WikiArt dataset is set up, you can generate the CSD embeddings by running the following command. Please adjust
+the `--data-dir` and `--embed_dir` accordingly. You should also adjust the batch size `--b` and number of workers `--j`
+according to your machine. The command to generate baseline embeddings is same, you just need to change the `--pt_style`
+with any of the following: `clip`, `dino`, `sscd`, `moco`.
+
+```angular2html
+python main_sim.py --dataset wikiart -a vit_large --pt_style csd --feattype normal --world-size 1 
+--dist-url tcp://localhost:6001 -b 128 -j 8 --embed_dir ./embeddings --data-dir <path to WikiArt dataset>
+--model_path <path to CSD weights>
+```
+
+## Evaluate
+Once you've generated the embeddings, run the following command:
+
+```angular2html
+python search.py --mode artist --dataset wikiart --chunked --query-chunk-dir <path to query embeddings above> 
+    --database-chunk-dir <path to database embeddings above> --topk 1 10 100 1000 --method IP --data-dir <path to WikiArt dataset>
+```
+
+## Train CSD on LAION-Styles
+
+You can also train style descriptors for your own datasets. A sample code for training on LAION-styles dataset is provided below.
+
+We have started to release the **Contra-Styles** (referred to as LAION-Styles in the paper) dataset. The dataset is available [here](https://huggingface.co/datasets/tomg-group-umd/ContraStyles)
+and will keep getting updated over the next few days as we are running profanity checks through NSFW and PhotoDNA. We will update here once the dataset has been completely uploaded.
+
+```
+export PYTHONPATH="$PWD:$PYTHONPATH"
+
+torchrun --standalone --nproc_per_node=4 CSD/train_csd.py --arch vit_base -j 8 -b 32 --maxsize 512 --resume_if_available --eval_k 1 10 100 --use_fp16 --use_distributed_loss --train_set laion_dedup --train_path <PATH to LAION-Styles> --eval_path <PATH to WikiArt/some val set>  --output_dir <PATH to save checkpoint>
+```
+
+## Pending items
+
+We will soon release the code to compute the artists' prototypical style representations and compute similarity score against any given generation. ETA end of June'24.
+
+## Cite us
+
+```
+@article{somepalli2024measuring,
+  title={Measuring Style Similarity in Diffusion Models},
+  author={Somepalli, Gowthami and Gupta, Anubhav and Gupta, Kamal and Palta, Shramay and Goldblum, Micah and Geiping, Jonas and Shrivastava, Abhinav and Goldstein, Tom},
+  journal={arXiv preprint arXiv:2404.01292},
+  year={2024}
+}
+```

CSD/artists_400.txt

@@ -0,0 +1,400 @@
+leonid afremov
+georges seurat
+amedeo modigliani
+Alan bean
+scott naismith
+antoine blanchard
+frederic remington
+Artgerm
+Greg Rutkowski
+mucha
+Alphonse Mucha
+WLOP
+Ilya Kuvshinov
+stanley artgerm
+Makoto Shinkai
+rossdraws
+James Jean
+Magali Villeneuve
+Donato Giancola
+loish
+Ruan Jia
+Studio Ghibli
+Beeple
+Ross Tran
+Beksinski
+Peter Mohrbacher
+Marc Simonetti
+Tom Bagshaw
+Craig Mullins
+Pixar
+Boris Vallejo
+joseph christian leyendecker
+Charlie Bowater
+Dan Mumford
+Moebius
+amano
+tomasz alen kopera
+Norman Rockwell
+Thomas Kinkade
+Giger
+Michael Garmash
+Victo Ngai
+Lois van Baarle
+Klimt
+RHADS
+Jordan Grimmer
+Bruce Pennington
+James Gurney
+disney
+Michael Whelan
+Frank Frazetta
+Tim Hildebrandt
+Raphael
+Caravaggio
+James Gilleard
+Simon Stalenhag
+Gil Elvgren
+Zdzislaw Beksinski
+Edward Hopper
+gaston bussiere
+Larry Elmore
+Fenghua Zhong
+Mike Mignola
+Karol Bak
+Francis Bacon
+Brom
+Syd Mead
+Gustav Klimt
+Jesper Ejsing
+Noah Bradley
+Steve McCurry
+Gustave Dore
+Atey Ghailan
+gustav dore
+Alex Ross
+Mark Brooks
+albert aublet
+Raphael Lacoste
+John Collier
+Alex Grey
+Rembrandt
+Eddie Mendoza
+Julie Bell
+Android Jones
+Miyazaki
+Greg Hildebrandt
+Gerald Brom
+Darek Zabrocki
+Titian
+Jim Burns
+Guy Denning
+Mattias Adolfsson
+Raymond Swanland
+Ernst
+Wes Anderson
+Feng Zhu
+Luis Royo
+Justin Gerard
+Hayao Miyazaki
+Takato Yamamoto
+Tyler Edlin
+Bob Eggleton
+Jakub Rozalski
+Tuomas Korpi
+Maxfield Parrish
+Kilian Eng
+Rebecca Guay
+ferdinand knab
+Jamie Wyeth
+John Berkey
+John Singer Sargent
+Rene Magritte
+Zaha Hadid
+Josan Gonzalez
+Shaddy Safadi
+Carl Spitzweg
+hajime sorayama
+Conrad Roset
+lovecraft
+Simon Stålenhag
+Masamune Shirow
+Leonardo da Vinci
+Gediminas Pranckevicius
+charles vess
+Jason Chan
+Anton Fadeev
+Albert Bierstadt
+hr giger
+Dali
+Roger Dean
+Zdzisław Beksiński
+Bayard Wu
+Ivan Shishkin
+Bob Ross
+Andreas Rocha
+Warhol
+Joe Fenton
+Hiroshi Yoshida
+Goro Fujita
+Cedric Peyravernay
+Jan van Eyck
+Ismail Inceoglu
+Ralph Horsley
+Andy Warhol
+Tomer Hanuka
+Jean Giraud
+Gustave Courbet
+Roberto Ferri
+Dustin Nguyen
+Mark Arian
+Louis Wain
+Ernst Haeckel
+Ivan Aivazovsky
+Salvador Dali
+Arthur Rackham
+Louis Comfort Tiffany
+Maciej Kuciara
+John Harris
+Andrew Wyeth
+Mark Ryden
+Junji Ito
+sung choi
+Alan Lee
+sylvain sarrailh
+Gaudi
+Max Ernst
+Filip Hodas
+Daarken
+Ralph McQuarrie
+Sailor Moon
+roger deakins
+Rosa Bonheur
+Brad Kunkle
+Lee Madgwick
+Caspar David Friedrich
+Alberto Vargas
+Chris Foss
+Alena Aenami
+Ian McQue
+Wadim Kashin
+Jean Delville
+Fra Angelico
+Peter Elson
+Martin Johnson Heade
+John Howe
+Anna Dittmann
+Zack Snyder
+Jim Lee
+Hieronymus Bosch
+Josephine Wall
+jessica rossier
+Michelangelo
+Michaelangelo
+Ryohei Hase
+Ilya Repin
+Annie Leibovitz
+Picasso
+Stephan Martiniere
+Frank Stella
+Eugene von Guerard
+Hokusai
+Alexander McQueen
+Tyler Jacobson
+Monet
+William Turner
+Van Gogh
+Anne Stokes
+Jeff Koons
+Frank Miller
+Anton Pieck
+Christopher Balaskas
+Ernst Fuchs
+Thomas Cole
+Carne Griffiths
+Mikhail Vrubel
+John William Waterhouse
+John William Godward
+Arcimboldo
+Vermeer
+Daniel Merriam
+James Paick
+Takashi Murakami
+Murakami
+Jan Matejko
+Banksy
+Cyril Rolando
+Amanda Sage
+Miho Hirano
+Eric Zener
+Remedios Varo
+Liam Wong
+Art Green
+Ed Roth
+Drew Struzan
+Jacek Yerka
+Kelly McKernan
+Raja Ravi Varma
+ashley wood
+Kandinsky
+Sam Spratt
+Rolf Armstrong
+Bauhaus
+Esao Andrews
+ESAO
+Richter
+Gertrude Abercrombie
+Yuumei
+Jack Kirby
+Victor Nizovtsev
+Roy Lichtenstein
+Lichtenstein
+Harumi Hironaka
+Paul Lehr
+Les Edwards
+Mike Winkelmann
+Dan Luvisi
+Art Frahm
+ridley scott
+Diego Rivera
+irakli nadar
+Dante Gabriel Rossetti
+Francisco Goya
+Evelyn De Morgan
+Frederic Edwin Church
+Frederick Edwin Church
+Jon Foster
+John Carpenter
+Giuseppe Arcimboldo
+Marcel Duchamp
+MC Escher
+Giorgio de Chirico
+Frans Hals
+Winslow Homer
+adrian ghenie
+Gerhard Richter
+Cecil Beaton
+Martine Johanna
+Tom Whalen
+Brian Froud
+Sandra Chevrier
+Vincent Van Gogh
+Yasutomo Oka
+Gregory Crewdson
+George Stubbs
+Eyvind Earle
+Gustave Baumann
+Yanjun Cheng
+Tran Nguyen
+Marina Abramović
+Cy Twombly
+Anselm Kiefer
+John James Audubon
+Chris Moore
+Hasui Kawase
+Scott Listfield
+Hugh Ferriss
+Claude Monet
+Jeff Easley
+Michael Komarck
+Jeremy Geddes
+Yves Tanguy
+Svetlin Velinov
+Lucian Freud
+Viktor Vasnetsov
+Gustave Doré
+Hikari Shimoda
+Edmund Dulac
+William Blake
+Thomas Eakins
+Frederic Church
+Gian Lorenzo Bernini
+Bill Sienkiewicz
+David Hockney
+Lucas Graciano
+national geographic
+Frida Kahlo
+Kahlo
+Jaime Jones
+Donald Judd
+Kawase Hasui
+Tim Okamura
+Anton Otto Fischer
+Tom Lovell
+Richard Hamilton
+Emiliano Ponzi
+Charles Marion Russell
+Ina Wong
+Adam Paquette
+Otto Dix
+Gabriel Dawe
+Mary Cassatt
+Arkhip Kuindzhi
+Jason Felix
+Piranesi
+Marianne North
+Peter Lindbergh
+Georges de La Tour
+Francis Picabia
+Kay Nielsen
+Sanford Robinson Gifford
+Hans Baluschek
+Audrey Kawasaki
+Mark Rothko
+Frank Auerbach
+Winston Churchill
+Cynthia Sheppard
+Chris Rahn
+Todd Lockwood
+Harry Clarke
+Coby Whitmore
+Margaret Keane
+Man Ray
+Hubert Robert
+Dorothea Tanning
+Ivan Bilibin
+Austin Osman Spare
+Paul Klee
+Frederic Leighton
+Alfonse Mucha
+Fernando Botero
+Marco Mazzoni
+Evgeny Lushpin
+John Atkinson Grimshaw
+Peter Paul Rubens
+Thomas Lawrence
+Yasar Vurdem
+Isaac Levitan
+Asher Brown Durand
+Yoann Lossel
+Henry Ossawa Tanner
+Bill Ward
+Jean Arp
+Jenny Saville
+Katsushika Hokusai
+Kim Keever
+Pablo Picasso
+Robert Delaunay
+Delaunay
+Chris Rallis
+Oleg Oprisco
+Anka Zhuravleva
+Walt Disney
+Tom Chambers
+Salvador Dalí
+Dalí
+Edward Gorey
+William Morris
+Takeshi Obata
+Juan Luna
+Christophe Vacher
+Grzegorz Rutkowski
+Tamara de Lempicka
+Tadao Ando
+Peter Gric
+sparth
+Leonora Carrington
+Mœbius
+Constant
+John Anster Fitzgerald
+Patrick Nagel

CSD/data/laion.py

@@ -0,0 +1,356 @@
+import os
+from PIL import Image
+from torch.utils.data import Dataset
+import numpy as np
+from tqdm import tqdm
+import pickle
+import vaex as vx
+
+
+def create_laion_cache(root_dir, anno_dir, keys=['artist', 'medium', 'movement']):
+    # -projects/diffusion_rep/data/laion_style_subset
+    # read all the picke files in the anno_dir
+    paths = []
+    labels = []  # list of lists since each image can have multiple labels
+    labels_to_index = {}  # dictionary that maps each label to an list of image indices
+
+    keys_offset = {k: 1000000 * i for i, k in enumerate(keys)}  # offset each key labels by a large number
+
+    str_to_list = lambda x, offset: [offset + int(a) for a in x.strip().split(',') if len(a) > 0]
+    for f in tqdm(os.listdir(anno_dir)):
+        if f.endswith('.pkl'):
+            with open(os.path.join(anno_dir, f), 'rb') as tmp:
+                ann = pickle.load(tmp)
+
+                for i, path in enumerate(ann['key']):
+                    cur_label = {
+                        k: str_to_list(ann[k][i], keys_offset[k])
+                        for k in keys
+                    }
+                    cur_label = sum(cur_label.values(), [])
+                    if len(cur_label) > 0:
+                        image_path = os.path.join(root_dir, 'data', path[:5], path + '.jpg')
+
+                        # if not os.path.exists(image_path):
+                        #     continue
+
+                        paths.append(image_path)
+                        labels.append(set(cur_label))
+                        for l in cur_label: labels_to_index.setdefault(l, []).append(i)
+
+    cache_path = os.path.join(anno_dir, '_'.join(keys) + '.cache')
+    with open(cache_path, 'wb') as tmp:
+        pickle.dump((paths, labels, labels_to_index), tmp)
+    return paths, labels, labels_to_index
+
+
+class LAION(Dataset):
+    def __init__(self, root_dir, anno_dir, split='database', transform=None,
+                 keys=['artist', 'medium', 'movement'],
+                 min_images_per_label=1, max_images_per_label=100000,
+                 num_queries_per_label=10, maxsize=None, model_type='dann'):
+        # -projects/diffusion_rep/data/laion_style_subset
+        self.root_dir = root_dir
+        self.transform = transform
+        self.model_type = model_type
+
+        # read all the picke files in the anno_dir
+        paths = []
+        labels = []  # list of lists since each image can have multiple labels
+        labels_to_index = {}  # dictionary that maps each label to an list of image indices
+
+        cache_path = os.path.join(anno_dir, '_'.join(keys) + '.cache')
+        if os.path.exists(cache_path):
+            with open(cache_path, 'rb') as tmp:
+                paths, labels, labels_to_index = pickle.load(tmp)
+        else:
+            paths, labels, labels_to_index = create_laion_cache(root_dir, anno_dir, keys)
+
+        maxout_labels = [l for l, v in labels_to_index.items() if len(v) > max_images_per_label]
+        maxout_labels.append('')  # Artificially add an empty label
+        print(f"Removing {len(maxout_labels)} tags with > {max_images_per_label} images")
+
+        minout_labels = [l for l, v in labels_to_index.items() if len(v) < min_images_per_label]
+        print(f"Removing {len(minout_labels)} tags with < {min_images_per_label} images")
+
+        # Get all possible tags
+        self.index_to_labels = list(set(labels_to_index.keys()) - set(maxout_labels) - set(minout_labels))
+        self.labels_to_index = {l: i for i, l in enumerate(self.index_to_labels)}
+
+        self.pathlist = []
+        self.labels = []
+        eye = np.eye(len(self.index_to_labels))
+        print("Filtering out labels")
+        for path, label in tqdm(zip(paths, labels)):
+            for l in maxout_labels:
+                if l in label:
+                    label.remove(l)
+
+            for l in minout_labels:
+                if l in label:
+                    label.remove(l)
+
+            if len(label) > 0:
+                self.pathlist.append(path)
+                cur_label = np.sum(eye[[self.labels_to_index[l] for l in label]], axis=0).astype(bool)
+                self.labels.append(cur_label)
+        self.labels = np.array(self.labels)
+
+        ## Split the dataset into index and query
+        keys_offset = {k: 1000000 * i for i, k in enumerate(keys)}
+        self.name_to_label = {}
+        for k in keys:
+            key_labels_path = os.path.join(
+                anno_dir, '../clip-interrogator/clip_interrogator/data',
+                k + "s_filtered_new.txt")
+            with open(os.path.join(key_labels_path)) as f:
+                for i, l in enumerate(f.readlines()):
+                    self.name_to_label[l.strip().replace("@", " ")] = keys_offset[k] + i
+
+        with open(os.path.join(anno_dir, 'top612_artists_shortlist_400.txt'), 'r') as f:
+            q_names = [l.lower().strip() for l in f.readlines()]
+            q_labels = [self.name_to_label[n] for n in q_names]
+            q_index = [self.labels_to_index[l] for l in q_labels]
+
+        query_ind = np.unique(np.concatenate(
+            [np.where(self.labels[:, i])[0][:num_queries_per_label]
+             for i in q_index]))
+
+        if split == "database":
+            self.pathlist = [self.pathlist[i] for i in range(len(self.pathlist)) if i not in query_ind]
+            self.labels = np.delete(self.labels, query_ind, axis=0)
+        else:
+            self.pathlist = [self.pathlist[i] for i in query_ind]
+            self.labels = self.labels[query_ind]
+
+        self.namelist = list(map(lambda x: x.split('/')[-1], self.pathlist))
+        # Select maxsize number of images
+        if maxsize is not None:
+            ind = np.random.randint(0, len(self.pathlist), maxsize)
+            self.pathlist = [self.pathlist[i] for i in ind]
+            self.labels = self.labels[ind]
+            self.namelist = [self.namelist[i] for i in ind]
+
+    def __len__(self):
+        return len(self.pathlist)
+
+    def __getitem__(self, idx):
+        img_loc = self.pathlist[idx]
+        image = Image.open(img_loc).convert("RGB")
+
+        if self.transform:
+            images = self.transform(image)
+
+        style = self.labels[idx]
+        if self.model_type == 'dann':
+            return images, style, idx
+        else:
+            return images, idx
+
+
+def create_laion_dedup_cache(dedup_dir):
+    # -projects/diffusion_rep/data/laion_style_subset/dedup_info
+    keys = None
+    labels = None
+    rejects = None
+    matching_info = None
+
+    files = [f for f in os.listdir(dedup_dir) if f.endswith('.parquet')]
+    for f in tqdm(sorted(files, key=lambda x: int(x.split('_')[2]))):
+        # Load dedup info
+        df = vx.open(os.path.join(dedup_dir, f))
+        if keys is None:
+            keys = df['name'].tolist()
+
+        # Updating reject information
+        cur_reject = df['matched'].to_numpy()
+        if rejects is not None:
+            rejects += cur_reject
+        else:
+            rejects = cur_reject
+
+        # Load labels
+        cur_labels = np.load(os.path.join(dedup_dir, f.replace('parquet', 'npz').replace('val_db', 'multilabel')))
+        cur_labels = cur_labels["arr_0"]
+        if labels is not None:
+            labels += cur_labels
+        else:
+            labels = cur_labels
+
+        # Load matching info
+        cur_matching_info = pickle.load(
+            open(os.path.join(dedup_dir, f.replace('parquet', 'pkl').replace('val_db', 'matching_info')), 'rb'))
+        if matching_info is not None:
+            matching_info.extend(cur_matching_info)
+        else:
+            matching_info = cur_matching_info
+
+    # Propagating labels
+    for i in tqdm(range(len(matching_info) - 1, -1, -1)):
+        labels[i] += np.sum(labels[matching_info[i], :], axis=0, dtype=bool)
+
+    cache_path = os.path.join(dedup_dir, 'joined.cache')
+    with open(cache_path, 'wb') as tmp:
+        pickle.dump((keys, labels, rejects), tmp)
+    return keys, labels, rejects
+
+
+class LAIONDedup(Dataset):
+    def __init__(self, root_dir, anno_dir, transform=None, model_type='dann', eval_mode=False, artist_mode=False):
+        self.root_dir = root_dir
+        self.transform = transform
+        self.model_type = model_type
+
+        dedup_dir = os.path.join(anno_dir, 'dedup_info')
+        cache_path = os.path.join(dedup_dir, 'joined.cache')
+        if os.path.exists(cache_path):
+            with open(cache_path, 'rb') as tmp:
+                keys, labels, rejects = pickle.load(tmp)
+        else:
+            keys, labels, rejects = create_laion_dedup_cache(dedup_dir)
+
+        keys = np.array(keys)[~rejects]
+        self.pathlist = [os.path.join(root_dir, 'data', key[:5], key + '.jpg') for key in keys]
+        self.labels = labels[~rejects]
+        self.namelist = list(map(lambda x: x.split('/')[-1], self.pathlist))
+
+        if eval_mode:
+            q_dset = LAION(root_dir, anno_dir, split='query')
+            self.query_db = vx.from_arrays(
+                name=[x.split('.')[0] for x in q_dset.namelist],
+                multilabel=q_dset.labels)
+
+            self.name_to_label = q_dset.name_to_label
+            self.labels_to_index = q_dset.labels_to_index
+            self.index_to_labels = q_dset.index_to_labels
+
+            self.val_db = vx.from_arrays(
+                name=keys.tolist(),
+                multilabel=self.labels)
+
+            if artist_mode:
+                # Filtering the db to include images which have hit on an artist
+                artist_inds = []
+                for label, index in self.labels_to_index.items():
+                    if label < 1000000:
+                        artist_inds.append(index)
+                artist_labels = self.labels[:, artist_inds]
+                artist_images = np.argwhere(np.sum(artist_labels, axis=1) > 0)
+                self.val_db = self.val_db.take(artist_images.squeeze()).extract()
+
+    def __len__(self):
+        return len(self.pathlist)
+
+    def __getitem__(self, idx):
+        img_loc = self.pathlist[idx]
+        image = Image.open(img_loc).convert("RGB")
+
+        if self.transform:
+            images = self.transform(image)
+
+        style = self.labels[idx]
+        if self.model_type == 'dann':
+            return images, style, idx
+        else:
+            return images, idx
+
+    def get_query_col(self, col):
+        return np.asarray(self.query_db[col].tolist())
+
+    def get_val_col(self, col):
+        return np.asarray(self.val_db[col].tolist())
+
+
+class SDSynth400:
+    def __init__(self, root_dir, query_split='user_caps', transform=None, eval_mode=False):
+        self.root_dir = root_dir
+        self.transform = transform
+        self.query_split = query_split
+        assert query_split in ['user_caps', 'simple_caps', 'woman_caps', 'house_caps', 'dog_caps']
+        assert os.path.exists(os.path.join(root_dir, f'{query_split}.csv'))
+        annotations = vx.from_csv(f'{self.root_dir}/{query_split}.csv')
+
+        self.pathlist = annotations['filepath'].tolist()
+        self.namelist = list(map(lambda x: x.split('/')[-1], self.pathlist))
+
+        # Dummy variables, not actually needed
+        self.query_images = []
+        self.val_images = []
+
+        if eval_mode:
+            data_dir = '-datasets/improved_aesthetics_6plus'
+            anno_dir = '-projects/diffusion_rep/data/laion_style_subset'
+            val_dset = LAIONDedup(data_dir, anno_dir, transform=transform, eval_mode=True, artist_mode=True)
+            # val_dset = LAION(data_dir, anno_dir, transform=transform)
+            # Needed for search code
+            filenames = [f.split('.')[0] for f in self.namelist]
+            q_names = [[l.lower().strip() for l in eval(label)] for label in annotations['labels'].tolist()]
+            q_labels = [[val_dset.name_to_label[n] for n in names if n in val_dset.name_to_label] for names in q_names]
+            q_index = [[val_dset.labels_to_index[l] for l in labels if l in val_dset.labels_to_index] for labels in
+                       q_labels]
+
+            eye = np.eye(len(val_dset.index_to_labels))
+            q_binlabels = [np.sum(eye[ind], axis=0).astype(bool) for ind in q_index]
+            self.query_db = vx.from_arrays(
+                name=filenames, multilabel=q_binlabels)
+            self.val_db = val_dset.val_db
+
+    def __len__(self):
+        return len(self.namelist)
+
+    def __getitem__(self, idx):
+        img_loc = self.pathlist[idx]
+        image = Image.open(img_loc).convert("RGB")
+        if self.transform:
+            image = self.transform(image)
+
+        return image, idx
+
+    def get_query_col(self, col):
+        return np.asarray(self.query_db[col].tolist())
+
+    def get_val_col(self, col):
+        return np.asarray(self.val_db[col].tolist())
+
+
+if __name__ == "__main__":
+    # dset = WikiArt(
+    #     "-projects/diffusion_rep/data/wikiart", 'database')
+
+    dset = LAION(
+        "-datasets/improved_aesthetics_6plus",
+        "-projects/diffusion_rep/data/laion_style_subset",
+        split='database')
+    print(f"{len(dset)} images in the dataset")
+
+    index_to_labels = []
+    index_to_keys = []
+    index_to_texts = []
+    label_to_name = {v: k for k, v in dset.name_to_label.items()}
+    for label in dset.index_to_labels:
+        index_to_texts.append(label_to_name[label])
+        index_to_labels.append(label)
+        if label < 1000000:
+            index_to_keys.append('artist')
+        elif label < 2000000:
+            index_to_keys.append('medium')
+        else:
+            index_to_keys.append('movement')
+
+    path = "-projects/diffusion_rep/data/laion_style_subset/index_to_labels_keys_texts.pkl"
+    with open(path, 'wb') as tmp:
+        pickle.dump((index_to_labels, index_to_keys, index_to_texts), tmp)
+
+    # dset = LAION(
+    #     "-datasets/improved_aesthetics_6plus",
+    #     "-projects/diffusion_rep/data/laion_style_subset",
+    #     split='query',
+    #     min_images_per_label=10,
+    #     max_images_per_label=100000)
+
+    # print(f"{len(dset)} images in the dataset")
+
+    # dset = LAIONDedup(
+    #     "-datasets/improved_aesthetics_6plus",
+    #     "-projects/diffusion_rep/data/laion_style_subset",
+    #     eval_mode=True)

CSD/data/wikiart.py

@@ -0,0 +1,102 @@
+import pathlib
+import os
+import sys
+import os.path as osp
+from PIL import Image
+from torch.utils.data import Dataset
+import pandas as pd
+import vaex as vx
+import numpy as np
+
+
+sys.path.insert(0, str(pathlib.Path(__file__).parent.resolve()))
+
+
+class WikiArt(object):
+    def __init__(self, root_dir):
+        assert osp.exists(osp.join(root_dir, 'wikiart.csv'))
+        self.root_dir = root_dir
+        annotations = vx.from_csv(f'{self.root_dir}/wikiart.csv')
+        acceptable_artists = list(set(annotations[annotations['split'] == 'database']['artist'].tolist()))
+        temprepo = annotations[annotations['artist'].isin(acceptable_artists)]
+        self.query_images = temprepo[temprepo['split'] == 'query']['name'].tolist()
+        self.val_images = temprepo[temprepo['split'] == 'database']['name'].tolist()
+        self.query_db = annotations[annotations['name'].isin(self.query_images)]
+        self.val_db = annotations[annotations['name'].isin(self.val_images)]
+        self.query_db['name'] = self.query_db['name'].apply(lambda x: '.'.join(x.split('.')[:-1]))
+        self.val_db['name'] = self.val_db['name'].apply(lambda x: '.'.join(x.split('.')[:-1]))
+
+    def get_query_col(self, col):
+        return np.asarray(self.query_db[col].tolist())
+
+    def get_val_col(self, col):
+        return np.asarray(self.val_db[col].tolist())
+
+
+class WikiArtD(Dataset):
+    def __init__(self, root_dir, split, transform=None):
+        self.root_dir = root_dir
+        self.transform = transform
+        self.split = split
+        assert osp.exists(osp.join(root_dir, 'wikiart.csv'))
+        annotations = vx.from_csv(f'{self.root_dir}/wikiart.csv')
+        acceptable_artists = list(set(annotations[annotations['split'] == 'database']['artist'].tolist()))
+        temprepo = annotations[annotations['artist'].isin(acceptable_artists)]
+        self.pathlist = temprepo[temprepo['split'] == split]['path'].tolist()
+
+        self.namelist = list(map(lambda x: x.split('/')[-1], self.pathlist))
+
+    def __len__(self):
+        return len(self.namelist)
+
+    def __getitem__(self, idx):
+        img_loc = self.pathlist[idx]  # os.path.join(self.root_dir, self.split,self.artists[idx] ,self.pathlist[idx])
+        image = Image.open(img_loc).convert("RGB")
+        if self.transform:
+            image = self.transform(image)
+
+        return image, idx
+
+
+class WikiArtTrain(Dataset):
+    def __init__(self, root_dir, split='database', transform=None, maxsize=None):
+        self.root_dir = root_dir
+        self.transform = transform
+        self.split = split
+        assert os.path.exists(os.path.join(root_dir, 'wikiart.csv'))
+        annotations = pd.read_csv(f'{self.root_dir}/wikiart.csv')
+        acceptable_artists = list(
+            set(annotations[annotations['split'] == 'database']['artist'].tolist())
+        )
+        temprepo = annotations[annotations['artist'].isin(acceptable_artists)]
+        self.pathlist = temprepo[temprepo['split'] == split]['path'].tolist()
+        self.labels = temprepo[temprepo['split'] == split]['artist'].tolist()
+
+        self.artist_to_index = {artist: i for i, artist in enumerate(acceptable_artists)}
+        self.index_to_artist = acceptable_artists
+
+        # Convert labels to one-hot
+        self.labels = list(map(lambda x: self.artist_to_index[x], self.labels))
+        self.labels = np.eye(len(acceptable_artists))[self.labels].astype(bool)
+        self.namelist = list(map(lambda x: x.split('/')[-1], self.pathlist))
+
+        # Select maxsize number of images
+        if maxsize is not None:
+            ind = np.random.randint(0, len(self.namelist), maxsize)
+            self.namelist = [self.namelist[i] for i in ind]
+            self.pathlist = [self.pathlist[i] for i in ind]
+            self.labels = self.labels[ind]
+
+    def __len__(self):
+        return len(self.namelist)
+
+    def __getitem__(self, idx):
+
+        img_loc = self.pathlist[idx]
+        image = Image.open(img_loc).convert("RGB")
+
+        if self.transform:
+            images = self.transform(image)
+
+        artist = self.labels[idx]
+        return images, artist, idx

CSD/environment.yaml

@@ -0,0 +1,29 @@
+name: style
+channels:
+  - pytorch
+  - defaults
+  - conda-forge
+dependencies:
+  - pillow
+  - pip
+  - python=3.9
+  - pytorch=*=*cuda11.3*
+  - cudatoolkit>=11.3
+  - scipy
+  - torchvision
+  - jupyterlab
+  - ipywidgets
+  - scikit-image
+  - faiss-gpu
+  - tensorboard
+  - pip:
+    - git+https://github.com/openai/CLIP.git
+    - pandas
+    - ipdb
+    - wandb
+    - timm==0.6.12
+    - matplotlib
+    - einops
+    - vaex
+    - seaborn
+    - scikit-learn

CSD/laion-styles-subset-tags.txt

@@ -0,0 +1,3480 @@
+a j casson
+aaron bohrod
+aaron douglas
+aaron jasinski
+aaron miller
+aaron nagel
+abbott handerson thayer
+abdur rahman chughtai
+abraham bloemaert
+abraham de vries
+abraham hondius
+abraham mignon
+abraham storck
+abraham van den tempel
+abraham willaerts
+abram arkhipov
+adam bruce thomson
+adam elsheimer
+adam paquette
+adam rex
+adolf dietrich
+adolf schrödter
+adolf ulric wertmüller
+adolph gottlieb
+adolph menzel
+adriaen brouwer
+adriaen coorte
+adriaen hanneman
+adriaen isenbrant
+adriaen van de velde
+adriaen van de venne
+adriaen van der werff
+adriaen van ostade
+adrian ghenie
+adrian zingg
+aelbert cuyp
+aert de gelder
+aert van der neer
+afewerk tekle
+affandi
+agnes martin
+agnolo bronzino
+agnolo gaddi
+agostino carracci
+ai weiwei
+ai xuan
+aimé barraud
+akihiko yoshida
+akira toriyama
+al feldstein
+al williamson
+alan bean
+alan davis
+alan lee
+alasdair gray
+albert anker
+albert aublet
+albert bierstadt
+albert dorne
+albert edelfelt
+albert gleizes
+albert guillaume
+albert joseph moore
+albert marquet
+albert namatjira
+alberto giacometti
+alberto morrocco
+alberto seveso
+alberto vargas
+albrecht altdorfer
+albrecht dürer
+alejandro burdisio
+aleksander gierymski
+aleksander kobzdej
+aleksandr gerasimov
+aleksi briclot
+alena aenami
+alessandro allori
+alesso baldovinetti
+alex grey
+alex horley
+alex katz
+alex ross
+alex toth
+alexander archipenko
+alexander brook
+alexander calder
+alexander carse
+alexander deyneka
+alexander ivanov
+alexander jansson
+alexander johnston
+alexander kanoldt
+alexander kucharsky
+alexander litovchenko
+alexander mann
+alexander mcqueen
+alexander nasmyth
+alexander robertson
+alexander rodchenko
+alexander roslin
+alexander scott
+alexander sharpe ross
+alexander stirling calder
+alexandre benois
+alexandre cabanel
+alexandre falguière
+alexei kondratyevich savrasov
+alexej von jawlensky
+alexey venetsianov
+alexis grimou
+alexis simon belle
+alfons walde
+alfonse mucha
+alfred east
+alfred edward chalon
+alfred eisenstaedt
+alfred freddy krupa
+alfred janes
+alfred jensen
+alfred kubin
+alfred leslie
+alfred leyman
+alfred richard gurrey
+alfred sisley
+alfred thompson bricher
+algernon talmage
+alice bailly
+alice mason
+alice neel
+alice prin
+alison watt
+allaert van everdingen
+allan brooks
+allan linder
+allan ramsay
+allen butler talcott
+allen jones
+allen tupper true
+alma thomas
+almada negreiros
+almeida júnior
+aloysius okelly
+alphonse legros
+alphonse mucha
+alvan fisher
+amadeo de souza cardoso
+amalia lindegren
+amanda sage
+amano
+ambrose mccarthy patterson
+ambrosius benson
+ambrosius bosschaert
+ambrosius holbein
+amedeo modigliani
+americo makk
+amir zand
+ammi phillips
+amos sewell
+amy sol
+amy weber
+an gyeon
+anato finnstark
+anders zorn
+andré charles biéler
+andre derain
+andré derain
+andré françois
+andré kertész
+andré lhote
+andré masson
+andrea del sarto
+andrea del verrocchio
+andrea kowch
+andrea mantegna
+andrea orcagna
+andrea pozzo
+andreas achenbach
+andreas gursky
+andreas rocha
+andrei riabovitchev
+andrei rublev
+andrei ryabushkin
+andrew bell
+andrew ferez
+andrew geddes
+andrew henderson
+andrew law
+andrew loomis
+andrew robertson
+andrew robinson
+andrew stevovich
+andrew wyeth
+android jones
+andrzej wróblewski
+andy goldsworthy
+andy warhol
+ángel botello
+angelica kauffman
+aniello falcone
+anish kapoor
+anita kunz
+anka zhuravleva
+anna ancher
+anna boch
+anna dittmann
+anna mary robertson moses
+anne dunn
+anne geddes
+anne redpath
+anne ryan
+anne said
+anne savage
+anne stokes
+anni albers
+annibale carracci
+annie leibovitz
+annie rose laing
+ansel adams
+anselm kiefer
+antanas sutkus
+anthony devas
+anthony palumbo
+anthony van dyck
+antoine blanchard
+antoine ignace melling
+antoine le nain
+antoine wiertz
+anton ažbe
+anton fadeev
+anton graff
+anton mauve
+anton otto fischer
+anton pieck
+anton räderscheidt
+antonello da messina
+antoni brodowski
+antonin artaud
+antonín chittussi
+antonín slavíček
+antonio canova
+antonio cavallucci
+antonio ciseri
+antonio de la gandara
+antonio donghi
+antonio mancini
+antônio parreiras
+antonio rotta
+apelles
+apollinary vasnetsov
+apollonia saintclair
+archibald motley
+archibald robertson
+archibald skirving
+archibald standish hartrick
+arcimboldo
+arie smit
+aristide maillol
+arkhip kuindzhi
+armand guillaumin
+armin hansen
+arnold blanch
+arnold böcklin
+arnold mesches
+arnold newman
+arshile gorky
+art fitzpatrick
+art frahm
+art green
+art spiegelman
+artemisia gentileschi
+artgerm
+arthur adams
+arthur b carles
+arthur boyd
+arthur burdett frost
+arthur dove
+arthur hughes
+arthur lismer
+arthur melville
+arthur pan
+arthur quartley
+arthur rackham
+arthur sarkissian
+arthur streeton
+artur grottger
+arvid nyholm
+asaf hanuka
+asai chū
+asher brown durand
+ashley wood
+atey ghailan
+attila meszlenyi
+aubrey beardsley
+audrey kawasaki
+august friedrich schenck
+august macke
+august sander
+auguste herbin
+augustus earle
+augustus john
+augustus vincent tack
+auseklis ozols
+austin briggs
+austin english
+austin osman spare
+ayo
+ayami kojima
+balthasar van der ast
+balthus
+banksy
+barbara longhi
+barclay shaw
+barent fabritius
+barkley hendricks
+barnett newman
+barron storey
+bartholomeus breenbergh
+bartholomeus strobel
+bartholomeus van bassen
+bartholomeus van der helst
+bartolomé esteban murillo
+bartolomeo vivarini
+bascove
+basil blackshaw
+basuki abdullah
+bayard wu
+beatrice ethel lithiby
+beatrix potter
+beauford delaney
+beeple
+beksinski
+ben enwonwu
+ben nicholson
+ben shahn
+ben stahl
+ben templesmith
+ben thompson
+benito quinquela martín
+benjamin block
+benjamin franklin
+benjamin gerritsz cuyp
+benjamin west
+benjamin williams leader
+benoit b mandelbrot
+bernard buffet
+bernard meninsky
+bernard van orley
+bernardo bellotto
+bernardo cavallino
+bernardo daddi
+bernardo strozzi
+bernie wrightson
+bert hardy
+3d render
+black white photo
+bert stern
+cartoon
+berthe morisot
+character portrait
+charcoal drawing
+bronze sculpture
+bertalan székely
+colorized photo
+cave painting
+color pencil sketch
+cross stitch
+cubist painting
+detailed drawing
+bhupen khakhar
+detailed painting
+diagram
+digital painting
+digital rendering
+drawing
+fine art painting
+bill ward
+gouache
+billy childish
+hyperrealistic painting
+jigsaw puzzle
+bob byerley
+bob eggleton
+bob ross
+flemish baroque
+matte painting
+hologram
+macro photograph
+manga drawing
+mosaic
+low poly render
+pastel
+pencil sketch
+boris kustodiev
+microscopic photo
+bourgeois
+photorealistic painting
+pointillism painting
+photocopy
+brad kunkle
+pop art painting
+brassaï
+polaroid photo
+renaissance painting
+screenprint
+screenshot
+silk screen
+sketch
+statue
+still life
+stipple
+brett whiteley
+brian froud
+surrealist painting
+storybook illustration
+tattoo
+tilt shift photo
+watercolor painting
+surrealist sculpture
+woodcut
+abstract drawing
+brom
+abstract painting
+acrylic painting
+brothers hildebrandt
+brooke shaden
+album cover
+bruce davidson
+bruce gilden
+airbrush painting
+bruce pennington
+etching
+bruno liljefors
+impressionist painting
+ink drawing
+oil canvas painting
+buckminster fuller
+art deco painting
+chalk art
+computer graphics
+concept art
+cyberpunk art
+cagnaccio di san pietro
+egyptian art
+graffiti art
+camille corot
+lineart
+camille pissarro
+poster art
+vector art
+camille bombois
+canaletto
+camilo egas
+camilo mori
+candido portinari
+cao zhibai
+caravaggio
+carel fabritius
+carel weight
+carel willink
+carl barks
+carl eytel
+carl frederik von breda
+carl gustaf pilo
+carl heinrich bloch
+carl hoppe
+carl larsson
+carl rahl
+carl spitzweg
+carl walter liner
+carla wyzgala
+carlo carrà
+carlo crivelli
+carlo galli bibiena
+carlo mense
+carlos schwabe
+carne griffiths
+carol bove
+carol sutton
+caroline lucy scott
+caroline mytinger
+carrie mae weems
+casey baugh
+caspar david friedrich
+caspar netscher
+caspar van wittel
+caspar wolf
+cassandra austen
+cassius marcellus coolidge
+catrin welzstein
+cecil beaton
+cecilia beaux
+cecily brown
+cedric peyravernay
+cerith wyn evans
+cézanne
+chagall
+chaim soutine
+chaïm soutine
+charles alston
+charles angrand
+charles bird king
+charles blackman
+charles codman
+charles conder
+charles cundall
+charles dana gibson
+charles demuth
+charles e burchfield
+charles furneaux
+charles ginner
+charles gleyre
+charles h woodbury
+charles harold davis
+charles haslewood shannon
+charles hopkinson
+charles joshua chaplin
+charles le brun
+charles mahoney
+charles marion russell
+charles martin
+charles mcauley
+charles ragland bunnell
+charles rennie mackintosh
+charles ricketts
+charles roka
+charles schulz
+charles thomson
+charles vess
+charles w bartlett
+charles williams
+charles willson peale
+charlie bowater
+charlotte harding
+charlotte nasmyth
+chase stone
+chen chi
+chen chun
+chen daofu
+chen hong
+chen hongshou
+chen lin
+chen lu
+chen yifei
+cheng shifa
+cheng zhengkui
+chesley bonestell
+chiharu shiota
+childe hassam
+chip zdarsky
+chippy
+choi buk
+chris cold
+chris foss
+chris friel
+chris labrooy
+chris moore
+chris rahn
+chris rallis
+chris ware
+christen dalsgaard
+christen købke
+christian jane fergusson
+christian krohg
+christian rohlfs
+christo
+christoffer wilhelm eckersberg
+christoph amberger
+christoph ludwig agricola
+christophe vacher
+christopher balaskas
+christopher moeller
+christopher perkins
+christopher williams
+christopher wood
+christopher wren
+chuck close
+cicely mary barker
+cimabue
+cindy sherman
+cindy wright
+claire dalby
+claire hummel
+clara miller burd
+clara peeters
+clara weaver parrish
+clarence holbrook carter
+clarice beckett
+clark voorhees
+claude cahun
+claude lorrain
+claude monet
+cleon peterson
+cleve gray
+cliff childs
+clifford ross
+clint cearley
+clyde caldwell
+clyfford still
+coby whitmore
+coles phillips
+colijn de coter
+colin campbell cooper
+colin gill
+colin hayes
+colin mccahon
+colin middleton
+colin moss
+conrad roset
+conroy maddox
+constant
+constant permeke
+constantin hansen
+corneille
+cornelia parker
+cornelis anthonisz
+cornelis bisschop
+cornelis de heem
+cornelis de man
+cornelis dusart
+cornelis saftleven
+cornelis van haarlem
+correggio
+cosmo alexander
+craig davison
+craig mullins
+craig thompson
+craola
+cristofano allori
+csaba markus
+cuno amiet
+cy twombly
+cynthia sheppard
+cyril rolando
+d howard hitchcock
+daarken
+dai jin
+dai xi
+dali
+dalí
+damien hirst
+dan frazier
+dan hillier
+dan luvisi
+dan mumford
+dan scott
+dan smith
+daniel f gerhartz
+daniel garber
+daniel lieske
+daniel ljunggren
+daniel maclise
+daniel merriam
+daniël mijtens
+daniel taylor
+dante gabriel rossetti
+daphne fedarb
+darek zabrocki
+daren bader
+dariusz zawadzki
+dave arredondo
+dave dorman
+dave gibbons
+dave kendall
+dave mckean
+david alfaro siqueiros
+david allan
+david annand
+david bailly
+david bomberg
+david boyd
+david brewster
+david budd
+david burliuk
+david chipperfield
+david diao
+david donaldson
+david eugene henry
+david gilmour blythe
+david hockney
+david inshaw
+david lachapelle
+david ligare
+david martin
+david octavius hill
+david palumbo
+david park
+david roberts
+david simpson
+david small
+david teniers iii
+david wilkie
+david wojnarowicz
+david young cameron
+dean cornwell
+dean ellis
+dean roger
+delaunay
+delphin enjolras
+dennis flanders
+dennis miller bunker
+derek gores
+derek hill
+derek jarman
+derf
+desmond morris
+diane arbus
+diane dillon
+diego giacometti
+diego gisbert llorens
+diego rivera
+diego velázquez
+dieric bouts
+ding guanpeng
+ding yunpeng
+dino valls
+dionisio baixeras verdaguer
+dirck de bray
+dirck hals
+dirck van baburen
+dirck van delen
+disney
+ditlev blunck
+dmitry levitzky
+dod procter
+domenichino
+domenico di pace beccafumi
+domenico ghirlandaio
+domenico induno
+domenico zampieri
+don eddy
+donald judd
+donald roller wilson
+donato giancola
+dong kingman
+dong qichang
+dong yuan
+dora carrington
+dora maar
+dorothea lange
+dorothea tanning
+dorothy burroughes
+dorothy hood
+dorothy johnstone
+dorothy king
+dosso dossi
+douglas shuler
+dr atl
+dr seuss
+drew struzan
+drew tucker
+du jin
+duccio
+dugald sutherland maccoll
+abstract art
+abstract expressionism
+dürer
+academic art
+action painting
+aestheticism
+afrofuturism
+duncan grant
+dwight william tryon
+american impressionism
+american realism
+american romanticism
+american scene painting
+earle bergey
+e charlton fortune
+arabesque
+ed benedict
+ed binkley
+art brut
+art deco
+ed roth
+art nouveau
+art photography
+eddie mendoza
+edgar degas
+arts crafts movement
+ashcan school
+assemblage
+eddie campbell
+edith lawrence
+barbizon school
+baroque
+bauhaus
+edmund blampied
+edmund charles tarbell
+edmund dulac
+brutalism
+classical realism
+edmund leighton
+cobra
+color field
+computer art
+conceptual art
+édouard manet
+constructivism
+concrete art
+crayon art
+eduardo kingman
+cubism
+eduard von grützner
+edvard munch
+dada
+edward armitage
+edward atkinson hornel
+de stijl
+edward arthur walton
+digital art
+deconstructivism
+environmental art
+edward clark
+expressionism
+fantastic realism
+fantasy art
+fauvism
+edward henry potthast
+edward gorey
+edward hopper
+figurative art
+fine art
+edward lamson henry
+folk art
+edward lear
+edward mitchell bannister
+futurism
+furry art
+edward robert hughes
+figurativism
+edward simmons
+graffiti
+gothic art
+edward weston
+happening
+harlem renaissance
+edwin deakin
+holography
+edwin austin abbey
+edward willis redfield
+hyperrealism
+hudson river school
+edwin georgi
+edwin landseer
+impressionism
+eero järnefelt
+egon schiele
+egbert van der poel
+eiq
+interactive art
+land art
+kinetic art
+les nabis
+egbert van heemskerck
+light space
+lowbrow
+ejnar nielsen
+el greco
+magic realism
+magical realism
+mail art
+mannerism
+el lissitzky
+maximalism
+metaphysical painting
+lyrical abstraction
+minimalism
+elaine de kooning
+modernism
+eleanor fortescuebrickdale
+naive art
+naturalism
+mingei
+eleanor vere boyle
+eliot hodgkin
+élisabeth vigée le brun
+eliseu visconti
+neoclassicism
+neogeo
+elizabeth forbes
+elizabeth jane lloyd
+net art
+new objectivity
+elizabeth murray
+elizabeth shippen green
+new sculpture
+elke vogelsang
+op art
+optical illusion
+elliott erwitt
+orphism
+elmer bischoff
+photorealism
+pixel art
+ellsworth kelly
+plein air
+pointillism
+pop art
+pop surrealism
+elsa beskow
+postimpressionism
+elmyr de hory
+precisionism
+emanuel leutze
+emanuel de witte
+process art
+psychedelic art
+emil bisttram
+emil carlsen
+primitivism
+emil fuchs
+emil nolde
+realism
+regionalism
+émile bernard
+renaissance
+retrofuturism
+rococo
+romanesque
+emily carr
+romanticism
+emiliano ponzi
+shin hanga
+emiliano di cavalcanti
+socialist realism
+emily shanks
+space art
+street art
+emory douglas
+emma lampert cooper
+superflat
+suprematism
+surrealism
+symbolism
+enrique simonet
+enrique grau
+enki bilal
+tachisme
+temporary art
+tonalism
+eric auld
+eric deschamps
+ukiyoe
+eric peterson
+eric taylor
+eric zener
+vanitas
+erich heckel
+video art
+erin hanson
+visual art
+ernest biéler
+underground comix
+ernest buckmaster
+ernest hébert
+ernest lawson
+ernest morgan
+ernest procter
+ernest william christmas
+ernie barnes
+ernst
+ernst fuchs
+ernst haeckel
+ernst ludwig kirchner
+ernst thoms
+ernst wilhelm nay
+erwin bowien
+esaias van de velde
+esao
+esao andrews
+esteban vicente
+etienne delessert
+ettore tito
+euan uglow
+eugène boudin
+eugène burnand
+eugène carrière
+eugene delacroix
+eugène delacroix
+eugène grasset
+eugène isabey
+childs drawing
+eugene von guerard
+eugeniusz zak
+eva gonzalès
+évariste vital luminais
+evaristo baschenis
+evelyn abelson
+evelyn de morgan
+everett raymond kinstler
+everett shinn
+computer rendering
+evert collier
+evgeny lushpin
+eyvind earle
+f scott hess
+fabien charuau
+fairfield porter
+fan kuan
+fan qi
+fang congyi
+farel dalrymple
+fede galizia
+federico barocci
+federico uribe
+federico zandomeneghi
+federico zuccari
+fedot sychkov
+detailed matte painting
+felice casorati
+felicity charlton
+fei danxu
+félix vallotton
+félix ziem
+feng zhu
+fenghua zhong
+ferdinand bol
+ferdinand hodler
+ferdinand knab
+ferdynand ruszczyc
+fern coppedge
+fernand léger
+fernand pelez
+fernand toussaint
+fernando amorsolo
+fernando botero
+filip hodas
+filippino lippi
+fiona stephenson
+fitz henry lane
+fitz hugh lane
+fletcher martin
+flora macdonald reid
+floris van dyck
+floris van schooten
+ford madox brown
+fra angelico
+fra bartolomeo
+fra filippo lippi
+frances c fairman
+frances hodgkins
+frances macdonald
+francesco albani
+francesco bartolozzi
+francesco bonsignori
+francesco clemente
+francesco del cossa
+francesco filippini
+francesco guardi
+francesco hayez
+francesco raibolini
+francis bacon
+francis bourgeois
+francis cadell
+francis davis millet
+francis ernest jackson
+francis focer brown
+francis helps
+francis picabia
+marble sculpture
+francisco de holanda
+francisco de zurbarán
+francisco goya
+francisco oller
+francisco zúñiga
+franciszek smuglewicz
+françois barraud
+françois bocion
+françois boucher
+françois clouet
+françois joseph heim
+françois quesnel
+frank auerbach
+minimalist painting
+frank buchser
+frank dumond
+frank frazetta
+frank leonard brooks
+frank mason
+frank mckelvey
+frank miller
+frank montague moore
+frank omeara
+frank schoonover
+frank stella
+frank weston benson
+frank xavier leyendecker
+franklin booth
+franklin carmichael
+frans hals
+frans koppelaar
+frans masereel
+františek kaván
+františek kupka
+franz kline
+franz marc
+franz sedlacek
+franz stuck
+franz vohwinkel
+franz von lenbach
+franz xaver winterhalter
+fred cress
+fred ludekens
+fred mitchell
+fred williams
+frédéric bazille
+frederic church
+frederic edwin church
+frederic leighton
+frederic remington
+frederick carl frieseke
+frederick edwin church
+frederick goodall
+frederick hammersley
+frederick lord leighton
+frederick mccubbin
+frederik de moucheron
+frederik vermehren
+frida kahlo
+friedel dzubas
+friedensreich hundertwasser
+friedrich gauermann
+friedrich von amerling
+frieke janssens
+frits thaulow
+fritz von dardel
+fritz von uhde
+fu baoshi
+fujishima takeji
+fyodor alekseyev
+fyodor rokotov
+fyodor vasilyev
+gabriel ba
+gabriel dawe
+gabriel metsu
+gabriele münter
+gaetano previati
+gai qi
+galen dara
+gao cen
+gao fenghan
+garry winogrand
+gaston anglade
+gaston bussiere
+gaston bussière
+gaudi
+gaugin
+gavin hamilton
+gawen hamilton
+gediminas pranckevicius
+geertgen tot sint jans
+gen paul
+gene davis
+gentile bellini
+geof darrow
+geoffrey dyer
+georg baselitz
+georg friedrich kersting
+georg friedrich schmidt
+georg muche
+georg scholz
+georg schrimpf
+george abe
+george ault
+george bain
+george barbier
+george barker
+george barret sr
+george bell
+george bellows
+george benjamin luks
+george biddle
+george caleb bingham
+george catlin
+george cruikshank
+george fiddes watt
+george frederic watts
+george frederick harris
+george gardner symons
+george grosz
+george hendrik breitner
+george henry
+george hurrell
+george inness
+george jamesone
+george lucas
+george luks
+george morrison
+george paul chalmers
+george pirie
+george reid
+george romney
+george stubbs
+george tooker
+abstract sculpture
+georges braque
+georges de la tour
+georges lacombe
+georges lemmen
+georges rouault
+georges seurat
+georges stein
+georgia okeeffe
+gerald brom
+gerald kelly
+gerard david
+gerard de lairesse
+gerard houckgeest
+gerard seghers
+gerard sekoto
+anime drawing
+gerard ter borch
+gerard soest
+gerda wegener
+gerhard richter
+gerbrand van den eeckhout
+germaine krull
+gerrit adriaenszoon berckheyde
+gerrit dou
+gertrude abercrombie
+art deco sculpture
+gertrude harvey
+géza dósa
+géza udvary
+engraving
+giacomo balla
+gian lorenzo bernini
+giger
+gil elvgren
+gilbert stuart
+gilles beloeil
+gillis rombouts
+gino severini
+giorgio de chirico
+giorgio morandi
+giorgione
+giotto
+giovanni antonio galli
+giovanni battista cipriani
+giovanni battista gaulli
+giovanni battista piazzetta
+giovanni battista piranesi
+giovanni battista tiepolo
+giovanni bellini
+giovanni bernardino azzolini
+giovanni boldini
+giovanni fattori
+giovanni francesco barbieri
+giovanni giacometti
+giovanni lanfranco
+ultrafine detailed painting
+giovanni paolo pannini
+giuseppe abbati
+giuseppe antonio petrini
+giuseppe arcimboldo
+giuseppe bernardino bison
+giuseppe camuncoli
+giuseppe de nittis
+giuseppe grisoni
+giuseppe tominz
+glen angus
+glen keane
+glenn fabry
+glennray tutor
+gloria stoll karn
+godfried schalcken
+gong xian
+gordon parks
+goro fujita
+gottfried helnwein
+govert dircksz camphuysen
+govert flinck
+goyō hashiguchi
+grace cossington smith
+grace english
+graham forsythe
+graham sutherland
+grandma moses
+grant wood
+grayson perry
+greg hildebrandt
+greg rutkowski
+greg spalenka
+greg staples
+gregory crewdson
+gregory gillespie
+gregory manchess
+grete stern
+grigoriy myasoyedov
+grzegorz rutkowski
+gu an
+gu hongzhong
+guan daosheng
+guido borelli da caluso
+guido reni
+guillermo del toro
+guo xi
+gustaf tenggren
+gustav dore
+gustav doré
+gustav klimt
+gustave baumann
+gustave boulanger
+gustave caillebotte
+gustave courbet
+gustave dore
+gustave doré
+gustave moreau
+gustave van de woestijne
+guy denning
+guy rose
+gwen john
+gwenny griffiths
+gwilym prichard
+gyula aggházy
+gyula batthyány
+gyula benczúr
+gyula derkovits
+h r giger
+hp lovecraft
+haddon sundblom
+hajime sorayama
+hal foster
+hamilton sloan
+hamish macdonald
+han gan
+hannabarbera
+hannah frank
+hanns katz
+hans asper
+hans baldung
+hans baluschek
+hans bellmer
+hans bol
+hans burgkmair
+hans erni
+hans fischer
+hans gude
+hans hofmann
+hans makart
+hans memling
+hans mertens
+hans von aachen
+hans von bartels
+harald giersing
+harold gilman
+harold harvey
+harold sandys williamson
+harold von schmidt
+harriet backer
+harrington mann
+harrison fisher
+harry clarke
+harry morley
+harumi hironaka
+harvey dunn
+harvey kurtzman
+harvey pratt
+hasegawa tōhaku
+hasui kawase
+hayao miyazaki
+heather hudson
+hedda sterne
+heinrich hofmann
+heinrich kley
+heinrich lefler
+heinrich maria davringhausen
+heinz anger
+helen edwards
+helen frankenthaler
+helen huang
+helene schjerfbeck
+helmut newton
+hendrick avercamp
+hendrick bloemaert
+hendrick terbrugghen
+hendrick van balen
+hendrick van streeck
+hendrik goltzius
+hendrik martenszoon sorgh
+hendrik van steenwijk i
+hendrik van steenwijk ii
+hendrik willem mesdag
+henri alphonse barnoin
+henri biva
+henri cartierbresson
+henri harpignies
+henri le sidaner
+henri matisse
+henri rousseau
+henriette wyeth
+henrik weber
+henry bright
+henry carr
+henry fuseli
+henry heerup
+henry justice ford
+henry lamb
+henry moore
+henry ossawa tanner
+henry otto wix
+henry raeburn
+henry raleigh
+henry scott tuke
+henry tonks
+henry van de velde
+henry wallis
+henry woods
+henryk siemiradzki
+herb ritts
+herbert bayer
+herbert james gunn
+herman saftleven
+herman van swanevelt
+hermenegildo anglada camarasa
+hieronymous bosch
+hieronymus bosch
+hikari shimoda
+hilma af klint
+hiromu arakawa
+hiroshi nagai
+hiroshi yoshida
+hiroshige
+hishikawa moronobu
+hisui sugiura
+hokusai
+holger roed
+honoré daumier
+horace vernet
+horatio mcculloch
+horatio nelson poole
+hovsep pushman
+howard butterworth
+howard chandler christy
+howard chaykin
+howard finster
+howard lyon
+howard pyle
+hr giger
+hu jieqing
+hua yan
+huang binhong
+huang ding
+huang gongwang
+huang guangjian
+huang ji
+huang shen
+huang tingjian
+hubert robert
+hubert van eyck
+hubert von herkomer
+hugh ferriss
+hugh william williams
+hugo anton fisher
+hugo heyrman
+hugo scheiber
+hugo simberg
+hugo van der goes
+humberto castro
+hundertwasser
+hyacinthe rigaud
+ian mcque
+ian miller
+ian spriggs
+ida rentoul outhwaite
+ignacio zuloaga
+ignacy witkiewicz
+ignat bednarik
+igor grabar
+igor kieryluk
+igor morski
+igor zenin
+ikuo hirayama
+illarion pryanishnikov
+ilya glazunov
+ilya kuvshinov
+ilya ostroukhov
+ilya repin
+ilya yefimovich repin
+ina wong
+ino
+ion andreescu
+irakli nadar
+irma stern
+isaac grünewald
+isaac levitan
+isaac soyer
+isabel codrington
+isabel naftel
+isamu noguchi
+isidor kaufman
+ismail acar
+ismail gulgee
+ismail inceoglu
+israel tsvaygenbaum
+istván csók
+istván orosz
+istván réti
+itō jakuchū
+itō shinsui
+itshak holtz
+ivan aivazovsky
+ivan albright
+ivan bilibin
+ivan generalić
+ivan kramskoi
+ivan mrkvička
+ivan shishkin
+ivan trush
+ivana kobilca
+ivor davies
+ivor williams
+iwasa matabei
+j alden weir
+j c leyendecker
+j frederick smith
+j l lund
+j m w turner
+j ottis adams
+jc leyendecker
+jmw turner
+jacek malczewski
+jacek yerka
+jack boul
+jack butler yeats
+jack davis
+jack kirby
+jack levine
+jack roth
+jack smith
+jackson pollock
+jacob adriaensz backer
+jacob burck
+jacob collins
+jacob de heusch
+jacob gerritsz cuyp
+jacob jordaens
+jacob kainen
+jacob koninck
+jacob lawrence
+jacob maris
+jacob more
+jacob ochtervelt
+jacob philipp hackert
+jacob pynas
+jacob savery
+jacob toorenvliet
+jacob van campen
+jacob van der ulft
+jacob van ruisdael
+jacopo amigoni
+jacopo bassano
+jacopo bellini
+jacopo de barbari
+jacopo pontormo
+jacques blanchard
+jacques callot
+jacques daret
+jacques sablet
+jacques villon
+jacqueslouis david
+jaime colson
+jaime jones
+jakob gauermann
+jakub rozalski
+jakub różalski
+jakub schikaneder
+james abbott mcneill whistler
+james barry
+james bateman
+james baynes
+james bolivar manson
+james c christensen
+james cadenhead
+james campbell noble
+james christensen
+james cowie
+james cromar watt
+james dickson innes
+james ensor
+james giles
+james gilleard
+james gillick
+james gillray
+james gurney
+james guthrie
+james humbert craig
+james jean
+james mcbey
+james mcintosh patrick
+james mcneill whistler
+james montgomery flagg
+james morris
+james morrison
+james paick
+james paterson
+james peale
+james pittendrigh macgillivray
+james rosenquist
+james ryman
+james thomas watts
+james tissot
+james warhola
+james wood
+jamie hewlett
+jamie wyeth
+jan antonisz van ravesteyn
+jan asselijn
+jan baptist weenix
+jan brett
+jan cornelisz vermeyen
+jan cox
+jan davidsz de heem
+jan de baen
+jan de bray
+jan gossaert
+jan griffier
+jan hackaert
+jan kip
+jan lievens
+jan matejko
+jan miel
+jan miense molenaer
+jan steen
+jan toorop
+jan van bijlert
+jan van de cappelle
+jan van der heyden
+jan van eyck
+jan van goyen
+jan van huysum
+jan van mieris
+jan verkolje
+jan victors
+jan wijnants
+jan wyck
+jan zrzavý
+jane carpanini
+jane frank
+jane freeman
+jane freilicher
+jane hawkins
+jane kelly
+jane nasmyth
+jane small
+janet archer
+janet dawson
+janet fish
+jános vaszary
+january suchodolski
+jarosław jaśnikowski
+jason benjamin
+jason chan
+jason edmiston
+jason felix
+jasper francis cropsey
+jasper johns
+jean antoine watteau
+jean arp
+jean auguste dominique ingres
+jean baptiste debret
+jean béraud
+jean clark
+jean colombe
+jean delville
+jean dubuffet
+jean dufy
+jean fouquet
+jean giraud
+jean hélion
+jean hey
+jean jouvenet
+jean metzinger
+jean micheal basquiat
+jean moebius giraud
+jean petitot
+jeanaugustedominique ingres
+jeanlouisernest meissonier
+jeanmarc nattier
+jeanmichel basquiat
+jeanna bauck
+jeanne hébuterne
+jeff easley
+jeff koons
+jeff miracola
+jeffrey catherine jones
+jeffrey smith
+jennifer janesko
+jenny eakin delony
+jenny saville
+jenő barcsay
+jens ferdinand willumsen
+jens juel
+jeong seon
+jeremiah ketner
+jeremy chong
+jeremy geddes
+jerry pinkney
+jerry schatzberg
+jerry weiss
+jerzy kossak
+jesper ejsing
+jesper myrfors
+jesse richards
+jessica rossier
+jessie willcox smith
+jiao bingzhen
+jim burns
+jim davis
+jim dine
+jim lee
+jim murray
+jim nelson
+jin nong
+jiro yoshihara
+joachim patinir
+joan brown
+joan miro
+joan miró
+joan snyder
+joanna carrington
+joaquín clausell
+joaquín sorolla
+jodorowsky
+joe bowler
+joe de mers
+joe fenton
+joe jusko
+joe machine
+joe mangrum
+joe shuster
+johan christian dahl
+johan jongkind
+johann berthelsen
+johann bodin
+johann christian brand
+johann friedrich overbeck
+johann gottfried steffan
+johann heinrich bleuler
+johann heinrich meyer
+johann jakob biedermann
+johann ludwig bleuler
+johann zoffany
+johannes cornelisz verspronck
+johannes helgeson
+johannes itten
+johannes lingelbach
+johannes mytens
+johannes vermeer
+johannes voss
+johfra bosschart
+john alexander
+john anster fitzgerald
+john armstrong
+john atherton
+john atkinson grimshaw
+john avon
+john bauer
+john bellany
+john berkey
+john blair
+john blanche
+john brack
+john brown
+john brown abercromby
+john button
+john byrne
+john cale
+john carpenter
+john clayton
+john clayton adams
+john collier
+john constable
+john duncan fergusson
+john e berninger
+john elwood bundy
+john everett millais
+john eyre
+john f francis
+john f peto
+john fabian carlson
+john frederick herring jr
+john frederick herring sr
+john frederick kensett
+john french sloan
+john fulton folinsbee
+john george sowerby
+john gibson
+john harris
+john henderson
+john henry lorimer
+john henry twachtman
+john howe
+john hutton
+john j park
+john james audubon
+john kay
+john keane
+john la gatta
+john lavery
+john linnell
+john lowrie morrison
+john luke
+john macdonald aiken
+john marin
+john martin
+john maxwell
+john mclaughlin
+john michael wright
+john murdoch
+john noble barlow
+john opie
+john parker
+john pettie
+john philip falter
+john platt
+john plumb
+john quinton pringle
+john robertson reid
+john romita jr
+john salminen
+john singer sargent
+john singleton copley
+john skinner prout
+john sloan
+john souch
+john steell
+john steuart curry
+john stuart ingle
+john trumbull
+john watson gordon
+john william godward
+john william waterhouse
+john wilson
+john wollaston
+john wonnacott
+jon foster
+jon whitcomb
+jonas de ro
+jonathan solter
+joos de momper
+jordan grimmer
+jorge jacinto
+jørgen roed
+josan gonzalez
+josé clemente orozco
+josé malhoa
+josef abel
+josef albers
+josef mánes
+josep rovira soler
+joseph badger
+joseph beuys
+joseph bowler
+joseph christian leyendecker
+joseph cornell
+joseph decamp
+joseph delaney
+joseph ducreux
+joseph dwight strong
+joseph henderson
+joseph kleitsch
+joseph noel paton
+joseph raphael
+joseph severn
+joseph stella
+joseph von führich
+joseph werner
+joseph wright of derby
+analytical art
+antipodeans
+josephine wall
+josetsu
+joshua reynolds
+josse lieferinxe
+jozef israëls
+józef mehoffer
+józef pankiewicz
+jozef simmler
+art  language
+józsef borsos
+ju chao
+ju lian
+juan de flandes
+juan giménez
+juan gris
+juan luna
+juan ogorman
+judith brown
+judith leyster
+judy cassab
+judy takács
+jules bastienlepage
+jules breton
+jules chéret
+jules joseph lefebvre
+jules pascin
+arte povera
+jules tavernier
+julia margaret cameron
+julian fałat
+julian onderdonk
+julian schnabel
+julie bell
+ascii art
+julio gonzález
+julio larraz
+julius exner
+julius leblanc stewart
+juliusz kossak
+jung park
+junji ito
+justin currie
+justin gerard
+justin sweet
+justus van gent
+kaburagi kiyokata
+kadir nelson
+kahlo
+kaigetsudō ando
+kaii higashiyama
+kalervo palsa
+kamisaka sekka
+kandinsky
+kanō eitoku
+kanō hōgai
+bengal school art
+kanō motonobu
+berlin secession
+kanō sansetsu
+kanō sanraku
+kanō tanyū
+black arts movement
+kanzan shimomura
+karel dujardin
+bertalan karlovszky
+karel van mander
+karl bodmer
+karl bryullov
+karl hagedorn
+cloisonnism
+karl hofer
+karl kopinski
+bertram brooker
+karol bak
+karolis strautniekas
+károly brocky
+károly ferenczy
+károly kernstok
+károly kisfaludy
+károly lotz
+károly patkó
+kate beaton
+kate greenaway
+käthe kollwitz
+kathleen scott
+kati horna
+katia chausheva
+katsukawa shunei
+context art
+katsukawa shunsen
+katsukawa shunshō
+katsushika hokusai
+betye saar
+katsuya terada
+kawai gyokudō
+kawanabe kyōsai
+kawase hasui
+kay nielsen
+kay sage
+crystal cubism
+kazimierz alchimowicz
+kazimir malevich
+kees bol
+kees maks
+kees scherer
+kees van dongen
+keisai eisen
+keith haring
+keith henderson
+keith mallett
+keith parkinson
+cynical realism
+kelly mckernan
+kelly freas
+ken danby
+bikash bhattacharjee
+ken howard
+ken kelly
+bill lewis
+kelly sueda
+kenneth noland
+kentaro miura
+bill sienkiewicz
+keos masons
+danube school
+kerembeyit
+kev walker
+khalil gibran
+kieran yanner
+kilian eng
+kim keever
+kim tschang yeul
+billie waters
+kinuko craft
+ecological art
+kishi ganku
+kitagawa utamaro
+kitao shigemasa
+klimt
+kobayashi kiyochika
+excessivism
+kōno bairei
+blanche hoschedé monet
+konrad grob
+konrad klapheck
+konrad witz
+konstantin korovin
+konstantin makovsky
+konstantin savitsky
+konstantin somov
+konstantin vasilyev
+konstantin westchilov
+konstantin yuon
+konstantinas ciurlionis
+koson ohara
+krenz cushart
+kristian zahrtmann
+kristin nelson
+feminist art
+bob singer
+kun can
+kuroda seiki
+bob thompson
+kurt schwitters
+kurt wenner
+kusama
+kyffin williams
+kyle lambert
+bogi fabian
+ladrönn
+lajos bruck
+lajos gulácsy
+bohumil kubista
+lajos tihanyi
+fluxus
+lam qua
+lambert doomer
+lambert jacobsz
+lan ying
+lari pittman
+larry elmore
+larry fink
+larry rivers
+funk art
+bonnard pierre
+lasar segall
+boris vallejo
+lászló mednyánszky
+lászló paál
+laura ford
+laura knight
+laura muntz lyall
+generative art
+laura wheeler waring
+laurel burch
+laurie lipton
+laurits tuxen
+lawren harris
+boris vladimirski
+geometric abstract art
+lawrence harris
+leandro erlich
+german romanticism
+leconte stewart
+lee jeffries
+lee madgwick
+leland bell
+leng mei
+lennie lee
+brad holland
+leo leuppi
+léon bakst
+leon kapliński
+leon kossoff
+leon kroll
+leon wyczółkowski
+leona wood
+leonaert bramer
+leonard appelbee
+heidelberg school
+leonard long
+leonard ochtman
+leonardo da vinci
+leonid afremov
+leonid pasternak
+leonor fini
+leonora carrington
+leopold gottlieb
+leroy neiman
+les edwards
+lesser ury
+lev lvovich kamenev
+lewis henry meakin
+li cheng
+li chevalier
+li di
+li kan
+li keran
+li shan
+li shixing
+li song
+li tang
+li tiefu
+li zai
+liam wong
+liang kai
+brian bolland
+lichtenstein
+incoherents
+lilia alvarado
+lilla cabot perry
+lillian bassman
+brian despain
+limbourg brothers
+lin liang
+brian dunlop
+linda sutton
+lionel lindsay
+lionel walden
+lisa frank
+lisa milroy
+international gothic
+lisa yuskavage
+lise deharme
+liu haisu
+liu jun
+liza donnelly
+lizzy ansingh
+lodewijk bruckman
+lois dodd
+lois mailou jones
+lois van baarle
+loish
+brian thomas
+lorenzo lotto
+lorraine fox
+lotte reiniger
+louis anquetin
+louis buvelot
+louis comfort tiffany
+louis de caullery
+louis eilshemius
+louis faurer
+bridget bate tichenor
+louis grell
+louis hersent
+louis janmot
+louis le brocquy
+louis le nain
+bridget riley
+louis marcoussis
+louis stettner
+louis valtat
+louis wain
+louisa matthíasdóttir
+louisa puller
+louise abbéma
+louise bourgeois
+louise catherine breslau
+louise nevelson
+lovecraft
+lovis corinth
+lu guang
+lu zhi
+lubin baugin
+luc tuymans
+luca della robbia
+lucas cranach the elder
+lucas graciano
+lucas van leyden
+lucas vorsterman
+lucian freud
+lucien pissarro
+lucio fontana
+bruce mclean
+lucy madox brown
+ludolf bakhuizen
+ludolf leendertsz de jongh
+ludovico carracci
+bruce munro
+ludwig bemelmans
+ludwig knaus
+luděk marold
+bruce nauman
+luigi kasimir
+luis enrique camej
+luis royo
+luo mu
+luo ping
+lydia field emmet
+lyle tuttle
+bruce timm
+lyonel feininger
+lyubov popova
+m c escher
+ma lin
+ma quan
+ma shi
+ma wan
+ma yuan
+bryan organ
+mab graves
+mabel rollins harris
+mac conner
+maciej kuciara
+mads berg
+maeda masao
+bunny yeager
+magali villeneuve
+byeon sangbyeok
+makoto aida
+makoto shinkai
+byron galvez
+maksymilian gierymski
+malcolm drummond
+malcolm morley
+malczewski
+malevich
+malvin gray johnson
+man ray
+caesar van everdingen
+mandy jurgens
+marc bell
+marc chagall
+marc simonetti
+marcel duchamp
+marcello bacciarelli
+marcin zaleski
+marco mazzoni
+neoromanticism
+marek okon
+margaret boden
+margaret graeme niven
+margaret keane
+margaret macdonald mackintosh
+marguerite zorach
+marià fortuny
+maria sibylla merian
+marianne north
+marianne von werefkin
+marie angel
+marie bashkirtseff
+marie bracquemond
+marie krøyer
+marie laurencin
+marilyn bendell
+marina abramović
+mario sironi
+marion wachtel
+mariotto albertinelli
+marius borgeaud
+mark arian
+mark boyle
+mark brooks
+mark english
+mark gertler
+mark keathley
+mark poole
+mark rothko
+mark ryden
+mark tedin
+mark zug
+marsden hartley
+marshall arisman
+martin deschambault
+martin johnson heade
+martin kober
+martin schoeller
+martin schongauer
+martine johanna
+martinus rørbye
+martiros saryan
+paris school
+maruyama ōkyo
+mary adshead
+mary agnes yerkes
+mary beale
+mary black
+mary blair
+mary callery
+mary cameron
+mary cassatt
+plasticien
+mary davis
+mary dignam
+mary elizabeth price
+mary grant
+mary hallock foote
+mary mccrossan
+mary moser
+masamune shirow
+masolino
+mathias kollros
+mathieu le nain
+mati klarwein
+matsumura goshun
+matt cavotta
+preraphaelitism
+matt stewart
+matt groening
+matthew smith
+matthias jung
+matthias stom
+matthijs maris
+mattias adolfsson
+private press
+maurice boitel
+maurice braun
+maurice de vlaminck
+maurice denis
+maude kaufman eggemeyer
+maurice prendergast
+maurice sendak
+maurice utrillo
+maurycy gottlieb
+max beckmann
+max buri
+max dupain
+max ernst
+max gubler
+max klinger
+max liebermann
+max pechstein
+max slevogt
+max švabinský
+qajar art
+max weber
+maxfield parrish
+maxim verehin
+maximilien luce
+maxwell bates
+maxwell gordon lightfoot
+may louise greville cooksey
+mc escher
+mckadesinsanity
+rayonism
+mead schaeffer
+mei qing
+meindert hobbema
+melchior dhondecoeter
+melchior lorck
+melissa benson
+melozzo da forlì
+menez
+meredith dillman
+mi fu
+miao fu
+michael ancher
+michael andrews
+michael cheval
+michael dahl
+michael flohr
+michael ford
+michael garmash
+michael goldberg
+michael james smith
+michael komarck
+michael leunig
+michael malm
+michael sittow
+michael whelan
+michaelangelo
+michal karcz
+michał karcz
+michalis oikonomou
+michel delacroix
+michelangelo
+michelangelo buonarotti
+michelangelo buonarroti
+michelangelo merisi da caravaggio
+michiel jansz van mierevelt
+michiel van musscher
+mihály munkácsy
+mihály zichy
+miho hirano
+mikalojus konstantinas ciurlionis
+serial art
+mike bierek
+mike deodato
+mike mignola
+mike winkelmann
+mikhail evstafiev
+mikhail larionov
+shock art
+mikhail nesterov
+mikhail vrubel
+mikhail yuryevich lermontov
+miklós barabás
+mikhail lebedev
+mildred anne butler
+miles johnston
+millard sheets
+milton avery
+milton caniff
+milton glaser
+mirabello cavalori
+mitchell johnson
+miyamoto
+miyazaki
+moebius
+mœbius
+moïse kisling
+mondrian
+monet
+morgan russell
+mori sosen
+mort künstler
+moses soyer
+mstislav dobuzhinsky
+mucha
+muirhead bone
+synchromism
+munch
+muqi
+murakami
+muriel brandt
+murray tinkelman
+synthetism
+mykola burachek
+myles birket foster
+n c wyeth
+nc wyeth
+nadim karam
+nadir afonso
+nagasawa rosetsu
+nan goldin
+nancy graves
+naoko takeuchi
+naomi okubo
+natalia goncharova
+nathan oliveira
+nathan wyburn
+nathaniel hone
+national geographic
+naza
+neal adams
+neil blevins
+neil boyle
+neil welliver
+neil williams
+nell dorr
+nelson alexander ross
+nene thomas
+nevercrew
+neysa mcmein
+ni zan
+niccolò dell abbate
+nicholas hilliard
+nicholas roerich
+nick gentry
+nicola samori
+nicolaes maes
+nicolaes pieterszoon berchem
+nicolas de staël
+nicolas lancret
+nicolas poussin
+nicolas toussaint charlet
+nicoletta ceccoli
+nikita veprikov
+niklaus manuel
+niko henrichon
+nikolai astrup
+nikolai ge
+nikolai yaroshenko
+nikolaj abraham abildgaard
+nikolay makovsky
+nikolay nikanorovich dubovskoy
+nil gleyen
+nils von dardel
+nina hamnett
+nishikawa sukenobu
+noah bradley
+noel counihan
+noémi ferenczy
+norah neilson gray
+noriyoshi ohrai
+norma bull
+norman garstin
+norman hepple
+norman lewis
+norman rockwell
+norman saunders
+nuno gonçalves
+okeeffe
+odd nerdrum
+odilon redon
+ogata gekkō
+ogata kōrin
+ohara koson
+okumura masanobu
+oleg lipchenko
+oleg oprisco
+olga boznańska
+olha darchuk
+oliver sin
+olivia de berardinis
+olivia peguero
+orazio gentileschi
+osamu tezuka
+oskar kokoschka
+oskar schlemmer
+osman hamdi bey
+ossip zadkine
+oswald achenbach
+oswald birley
+oswaldo guayasamín
+otakar kubín
+ottó baditz
+otto dix
+otto eckmann
+otto piene
+otto pilny
+otto stark
+pablo carpio
+pablo munoz gomez
+pablo picasso
+pacita abad
+pál szinyei merse
+pamphilus
+pan yuliang
+paolo uccello
+paolo veronese
+parmigianino
+pascale campion
+pat adams
+patrick adam
+patrick brown
+patrick ching
+patrick dougherty
+patrick hall
+patrick henry bruce
+patrick heron
+patrick nagel
+patrick nasmyth
+patrick pietropoli
+patrick woodroffe
+paul bird
+paul bril
+paul cadmus
+paul cezanne
+paul cézanne
+paul cornoyer
+paul davis
+paul delvaux
+paul émile chabas
+paul gauguin
+paul georges
+paul guigou
+paul gustav fischer
+paul gustave fischer
+paul harvey
+paul henry
+paul jacob naftel
+paul kane
+paul kelpe
+paul klee
+paul lehr
+paul lohse
+paul nash
+paul ranson
+paul signac
+paula rego
+paulus moreelse
+paulus potter
+pavel fedotov
+pavel filonov
+pearl frush
+peder severin krøyer
+pedro álvarez castelló
+pedro figari
+peggy angus
+peggy bacon
+penleigh boyd
+penry williams
+per kirkeby
+perle fine
+peter alexander hay
+peter basch
+peter birmann
+peter blume
+peter brook
+peter churcher
+peter de seve
+peter doig
+peter elson
+peter fiore
+peter gric
+peter helck
+peter lanyon
+peter lely
+peter lindbergh
+peter madsen
+peter max
+peter michael
+peter mohrbacher
+peter paul rubens
+peter prendergast
+peter scott
+peter snow
+peter wells
+peter wtewael
+peter zumthor
+petrus christus
+petrus van der velden
+phil koch
+philip de lászló
+philip evergood
+philip guston
+philip wilson steer
+philipp veit
+philippe druillet
+philips wouwerman
+phillip otto runge
+picasso
+piero della francesca
+piero di cosimo
+pierre adolphe valette
+pierre auguste cot
+pierre bonnard
+pierre brissaud
+pierre mion
+pierre pellegrini
+pierre puvis de chavannes
+pierre roy
+pierre soulages
+pierreauguste renoir
+piet mondrian
+pieter aertsen
+pieter bruegel
+pieter brueghel the younger
+pieter claesz
+pieter codde
+pieter cornelisz van slingelandt
+pieter de grebber
+pieter de hooch
+pieter de ring
+pieter huys
+pieter janssens elinga
+pieter jansz saenredam
+pieter lastman
+pieter mulier ii
+pieter van anraedt
+pieter van der werff
+pieter van laer
+pietro da cortona
+pietro longhi
+pietro lorenzetti
+pietro perugino
+pinchus kremegne
+pinturicchio
+piranesi
+pisanello
+pixar
+pollock
+pompeo batoni
+prince hoare
+prudence heward
+pruett carter
+pu hua
+puru
+qi baishi
+qian du
+qian gu
+qian xuan
+qiu ying
+quentin blake
+quentin matsys
+quint buchholz
+r b kitaj
+r r mcian
+rachel reckitt
+rachel ruysch
+rachel whiteread
+rackstraw downes
+radi nedelchev
+rafail levitsky
+rafal olbinski
+raja ravi varma
+ralph albert blakelock
+ralph burke tyree
+ralph earl
+ralph horsley
+ralph mcquarrie
+randolph caldecott
+randolph schwabe
+randy gallegos
+randy post
+randy vargas
+raoul dufy
+raphael
+raphaël collin
+raphael kirchner
+raphael lacoste
+raphael soyer
+raphaelle peale
+ravi zupa
+ray caesar
+ray crooke
+ray parker
+raymond briggs
+raymond coxon
+raymond han
+raymond leech
+raymond saunders
+raymond swanland
+raymond teague cowern
+rebecca guay
+relja penezic
+rembrandt
+rembrandt peale
+rembrandt van rijn
+remedios varo
+ren hang
+ren xiong
+ren xun
+rené auberjonois
+rené burri
+rene magritte
+rené magritte
+renoir
+reynolds beal
+rhads
+ric nagualero
+ricardo bofill
+richard anuszkiewicz
+richard avedon
+richard benning
+richard carline
+richard corben
+richard dadd
+richard demarco
+richard diebenkorn
+richard doyle
+richard estes
+richard gerstl
+richard hamilton
+richard hess
+richard mayhew
+richard parkes bonington
+richard pionk
+richard schmid
+richard wilson
+richard wright
+richmond barthé
+richter
+rick amor
+rick griffin
+ridley scott
+ridolfo ghirlandaio
+rihard jakopič
+rinaldo cuneo
+rita angus
+riusuke fukahori
+riza abbasi
+rob alexander
+rob gonsalves
+rob liefeld
+robert adamson
+robert antoine pinchon
+robert ballagh
+robert bateman
+robert bechtle
+róbert berény
+robert bevan
+robert brackman
+robert brough
+robert bryden
+robert campin
+robert colquhoun
+robert crumb
+robert delaunay
+robert dickerson
+robert falk
+robert fawcett
+robert freebairn
+robert gavin
+robert griffier
+robert henderson blyth
+robert henri
+robert jacobsen
+robert koehler
+robert lenkiewicz
+robert macbryde
+robert maguire
+robert mapplethorpe
+robert mccall
+robert mcginnis
+robert motherwell
+robert noble
+robert peak
+robert rauschenberg
+robert reid
+robert scott lauder
+robert sivell
+robert thomas
+robert walker macbeth
+robert weaver
+robert weir allan
+robert william vonnoh
+robert zünd
+roberto ferri
+roberto parada
+rockwell kent
+rodel gonzalez
+rodney matthews
+rodolfo amoedo
+rodolfo escalera
+rodolfo morales
+rodolphe wytsman
+roelant savery
+roger ballen
+roger deakins
+roger dean
+roger wilson dennis
+rogier van der weyden
+rolf armstrong
+romaine brooks
+romare bearden
+ron english
+ron spears
+ron spencer
+ron walotsky
+ronald davis
+rory mcewen
+rosa bonheur
+rosalie emslie
+rose maynard barton
+rosemary allan
+ross tran
+rossdraws
+rowena meeks abdy
+roy de maistre
+roy decarava
+roy lichtenstein
+roy petley
+roz chast
+ruan jia
+rube goldberg
+rubens peale
+rudolf ernst
+rudolf hausner
+rudolf koller
+rudolf schlichter
+rudolf von alt
+rudolph belarski
+rudy siswanto
+rufino tamayo
+rupert bunny
+russell chatham
+russell dongjun lu
+russell drysdale
+russell patterson
+ruth hollingsworth
+ruth orkin
+ruth sanderson
+ruth simpson
+ryan barger
+ryan pancoast
+ryan yee
+ryohei hase
+ryōhei koiso
+ryoji ikeda
+sadao watanabe
+sailor moon
+saitō kiyoshi
+sakai hōitsu
+salomon de bray
+salomon koninck
+salomon van ruysdael
+salvador dali
+salvador dalí
+sam black
+sam bosma
+sam charles
+sam spratt
+samuel colman
+samuel dirksz van hoogstraten
+samuel f b morse
+samuel peploe
+samuel prout
+samuel scott
+samuel shelley
+samuel silva
+sándor bortnyik
+sandra chevrier
+sandro botticelli
+sanford robinson gifford
+santiago caruso
+santiago rusiñol
+sarah lucas
+sarah morris
+satoshi kon
+saul steinberg
+saul tepper
+scarlett hooft graafland
+scott gustafson
+scott listfield
+scott naismith
+sean scully
+seb mckinnon
+sebastian vrancx
+sebastiano ricci
+sengai
+senior artist
+senior environment artist
+serge sudeikin
+sergio larraín
+serhii vasylkivsky
+sesshū tōyō
+seuss dr
+shaddy safadi
+shang xi
+shao mi
+shen quan
+shen zhou
+sheng mao
+sheng maoye
+shibata zeshin
+shigeru aoki
+shin saimdang
+shin yunbok
+shinji aramaki
+shitao
+shukei sesson
+sidney nolan
+sidney richard percy
+siegfried haas
+sigurd swane
+silvestro lega
+silvia dimitrova
+silvia pelissero
+simon bisley
+simon marmion
+simon stalenhag
+simon stålenhag
+simon vouet
+simone martini
+sin wi
+sir alfred munnings
+sir jacob epstein
+sir john tenniel
+sir william orpen
+sir william russell flint
+slawomir maniak
+sofonisba anguissola
+sohrab sepehri
+soma orlai petrich
+song xu
+sonia delaunay
+sophie anderson
+sophie gengembre anderson
+sophie pemberton
+sōtarō yasui
+sparth
+spencer gore
+stan galli
+stan stokes
+stanhope forbes
+stanislas lépine
+stanislav zhukovsky
+stanisław ignacy witkiewicz
+stanisław masłowski
+stanisław wyspiański
+stanley artgerm
+stanley spencer
+stefan lochner
+stephan martiniere
+stephan martinière
+stephen bone
+stephen greene
+stephen little
+stephen pace
+stevan dohanos
+steve argyle
+steve dillon
+steve hanks
+steve mccurry
+steven belledin
+stokely webster
+storm thorgerson
+stuart davis
+studio ghibli
+sudip roy
+sugimura jihei
+sun long
+sung choi
+sunil das
+susan crile
+suzanne valadon
+suzuki harunobu
+svetlin velinov
+svetoslav roerich
+syd barrett
+syd mead
+sydney carline
+sydney prior hall
+sylvain sarrailh
+sylvester shchedrin
+sylvia molloy
+sylvia sleigh
+szymon czechowicz
+t c steele
+tadao ando
+taddeo gaddi
+tadeusz makowski
+taiyō matsumoto
+takahashi yuichi
+takashi murakami
+takato yamamoto
+takehisa yumeji
+takeshi obata
+takeuchi seihō
+tamara de lempicka
+tamara lempicka
+tang di
+tang yifen
+tang yin
+tani bunchō
+taro okamoto
+taro yamamoto
+tatiana hordiienko
+tatsuyuki tanaka
+tawaraya sōtatsu
+ted degrazia
+ted nasmith
+telemaco signorini
+terese nielsen
+terry morris
+terry oakes
+terry redlin
+the brothers hildebrandt
+thechamba
+theo van doesburg
+theodor philipsen
+théodore chassériau
+theodore earl butler
+théodore géricault
+theodore major
+theodore robinson
+théodore rousseau
+théodule ribot
+thierry bisch
+thomas baines
+thomas barker
+thomas blackshear
+thomas bock
+thomas campbell
+thomas cantrell dugdale
+thomas carr
+thomas cole
+thomas couture
+thomas crane
+thomas dalziel
+thomas de keyser
+thomas dewing
+thomas doughty
+thomas eakins
+thomas fogarty
+thomas gainsborough
+thomas hart benton
+thomas hill
+thomas kinkade
+thomas kluge
+thomas lawrence
+thomas mann baynes
+thomas millie dow
+thomas moran
+thomas nast
+thomas rowlandson
+thomas scholes
+thomas stothard
+thomas struth
+thomas wijck
+thornton oakley
+tim biskup
+tim doyle
+tim hildebrandt
+tim okamura
+tim white
+tina blondell
+tina modotti
+tintoretto
+titian
+titus lunter
+todd lockwood
+tom bagshaw
+tom bonson
+tom chambers
+tom lovell
+tom phillips
+tom roberts
+tom scott rsa
+tom thomson
+tom wesselmann
+tom whalen
+tomasz alen kopera
+tomasz jedruszek
+tomek setowski
+tomer hanuka
+tomi ungerer
+tomioka tessai
+tommaso masaccio
+tomokazu matsuyama
+tony diterlizzi
+tony sart
+tooth wu
+torii kiyomasu
+torii kiyomitsu
+torii kiyonaga
+torii kiyonobu i
+tosa mitsunobu
+tosa mitsuoki
+tōshi yoshida
+toshiko okanoue
+tove jansson
+toyen
+toyohara chikanobu
+toyohara kunichika
+tracey emin
+tracy harris
+tran nguyen
+trevor brown
+tsuchida bakusen
+tsuchiya koitsu
+tsuguharu foujita
+tsukioka yoshitoshi
+tuomas korpi
+tyler edlin
+tyler jacobson
+uemura shōen
+ulrika pasch
+umberto boccioni
+unichi hiratsuka
+urakusai nagahide
+utagawa hirokage
+utagawa hiroshige ii
+utagawa kunimasa
+utagawa kunisada
+utagawa kunisada ii
+utagawa kuniyoshi
+utagawa toyoharu
+utagawa toyokuni
+utagawa yoshiiku
+utagawa yoshitaki
+utagawa yoshitora
+utagawa yoshitsuya
+václav brožík
+valentin aleksandrovich serov
+valentine hugo
+valerie petts
+van gogh
+vanessa beecroft
+vanessa bell
+vasily andreevich tropinin
+vasily perov
+vasily polenov
+vasily surikov
+vasily vereshchagin
+vassily maximov
+vermeer
+vicente juan masip
+victo ngai
+victor adame minguez
+victor brauner
+victor enrich
+victor meirelles
+victor mosquera
+victor nizovtsev
+victor vasarely
+victor wang
+victoria francés
+viktor madarász
+viktor oliva
+viktor vasnetsov
+vilhelm kyhn
+vincent di fate
+vincent evans
+vincent lefevre
+vincent proce
+vincent van gogh
+vincenzo cabianca
+vincenzo irolli
+viola paterson
+violet oakley
+virgil finlay
+virginia lee burton
+vito dancona
+vittore carpaccio
+vivian maier
+vladimir borovikovsky
+vladimir kush
+vladimir makovsky
+vladimir tatlin
+vladimir tretchikoff
+vlaho bukovac
+volkan baga
+wadim kashin
+waldo peirce
+walenty wańkowicz
+wally wood
+walt disney
+walt reed
+walter bayes
+walter beach humphrey
+walter crane
+walter emerson baum
+walter haskell hinton
+walter humphrey
+walter leighton clark
+walter osborne
+walter sickert
+walter stuempfig
+wang duo
+wang e
+wang fu
+wang hui
+wang jian
+wang lü
+wang meng
+wang mian
+wang shimin
+wang shishen
+wang wei
+wang wu
+wang ximeng
+wang yi
+wang yuan
+wang yuanqi
+wang zhenpeng
+warhol
+warren mahy
+warwick goble
+washington allston
+wassily kandinsky
+wayne barlowe
+wayne england
+wayne reynolds
+wayne thiebaud
+weiwei
+wen boren
+wen jia
+wen tong
+wen zhengming
+wendell minor
+wendy froud
+wes anderson
+wes wilson
+wesley burt
+wifredo lam
+wilhelm bendz
+wilhelm leibl
+wilhelm marstrand
+wilhelm schnarrenberger
+wilhelm trübner
+will barnet
+will eisner
+will ellis
+willard metcalf
+willem claeszoon heda
+willem cornelisz duyster
+willem de kooning
+willem drost
+willem kalf
+willem maris
+willem van aelst
+willem van der vliet
+willem van haecht
+willem van mieris
+william berra
+william blake
+william blake richmond
+william bliss baker
+william bonnar
+william brodie
+william coldstream
+william conor
+william crosbie
+william crozier
+william dargie
+william dobell
+william dobson
+william dring
+william edouard scott
+william edward west
+william etty
+william fettes douglas
+william forsyth
+william gear
+william george gillies
+william glackens
+william gropper
+william harnett
+william hoare
+william hogarth
+william holman hunt
+william holmes sullivan
+william home lizars
+william jacob baer
+william jennys
+william john thomson
+william kentridge
+william langson lathrop
+william mactaggart
+william mcgregor paxton
+william mctaggart
+william merritt chase
+william michael harnett
+william miller
+william morris
+william nicholson
+william powhida
+william quiller orchardson
+william simpson
+william steig
+william stott
+william stout
+william trost richards
+william turner
+william woodward
+william york macgregor
+william zorach
+williamadolphe bouguereau
+willian murai
+willie ito
+willy finch
+wilson irvine
+winona nelson
+winslow homer
+winsor mccay
+winston churchill
+władysław czachórski
+władysław podkowiński
+wlop
+wojciech gerson
+wojciech korneli stattler
+wojciech kossak
+wojciech weiss
+wolf huber
+wolf kahn
+wolfgang letti
+wolfgang lettl
+wouter pietersz crabeth
+wu bin
+wu changshuo
+wu guanzhong
+wu hong
+wu li
+wu shixian
+wu wei
+wu zhen
+wu zuoren
+wylie beckert
+wyndham lewis
+xanthus russell smith
+xi gang
+xia chang
+xia gui
+xia yong
+xiang shengmo
+xiao yuncong
+xie he
+xie huan
+xie sun
+xu beihong
+xu wei
+xu xi
+xuande emperor
+xul solar
+yan hui
+yan liben
+yanagawa shigenobu
+yang j
+yang jin
+yanjun cheng
+yasar vurdem
+yasuo kuniyoshi
+yasutomo oka
+yayoi kusama
+yayou kusama
+yerkaland
+yi jaegwan
+yoann lossel
+yoji shinkawa
+yokoyama taikan
+yosa buson
+yoshihiko wada
+yoshio markino
+yoshitaka amano
+yoshitoshi mori
+yousuf karsh
+yu zhiding
+yuan jiang
+yuan yao
+yue minjun
+yuko shimizu
+yun duseo
+yun shouping
+yuri ivanovich pimenov
+yuumei
+yves klein
+yves tanguy
+yvonne jacquette
+zack snyder
+zack stella
+zaha hadid
+zdzislaw beksinski
+zdzisław beksiński
+zeen chin
+zeng jing
+zhang han
+zhang kechun
+zhang lu
+zhang shuqi
+zhang wo
+zhang xiaogang
+zhang xuan
+zhang yan
+zhang yin
+zhang zeduan
+zhang zongcang
+zhao mengfu
+zhao yong
+zhao zuo
+zheng xie
+zhichao cai
+zhou chen
+zhou fang
+zhou jichang
+zhou wenjing
+zhu da
+zhu derun
+zinaida serebriakova
+zoë mozert
+zou yigui
+zou zhe
+zsolt bodoni
+zygmunt waliszewski
+dustin nguyen
+e simms campbell
+e william gollings
+ed emshwiller
+ed paschke
+edi rama
+edmund f ward
+édouard detaille
+édouard vuillard
+eduardo paolozzi
+edward bailey
+edward burnejones
+edward george handel lucas
+edward hicks
+edward okuń
+edward ruscha
+edward wadsworth
+edwin dickinson
+edwin g lucas
+eglon van der neer
+eiichiro oda
+einar hakonarson
+elbridge ayer burbank
+ellen gallagher
+elsa bleda
+emil orlik
+emilio grau sala
+emily mason
+emma geary
+ken elias
+brice marden

CSD/main_sim.py

@@ -0,0 +1,356 @@
+#!/usr/bin/env python
+
+# Copyright (c) Facebook, Inc. and its affiliates.
+# All rights reserved.
+
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+import argparse
+import builtins
+import os
+import pathlib
+import random
+import sys
+import warnings
+
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.parallel
+import torch.backends.cudnn as cudnn
+import torch.distributed as dist
+import torch.optim
+import torch.multiprocessing as mp
+import torch.utils.data
+import torch.utils.data.distributed
+from torchvision import transforms
+import torchvision.models as torchvision_models
+from torchvision.models import VGG16_Weights
+
+sys.path.insert(0, str(pathlib.Path(__file__).parent.resolve()))
+
+import utils
+from utils import extract_features_pca
+from models import dino_vits, moco_vits
+from data.wikiart import WikiArtD
+
+
+parser = argparse.ArgumentParser('dynamicDistances-Embedding Generation Module')
+parser.add_argument('--dataset', type=str, required=True, help="Name of the dataset",
+                    choices=['wikiart'])
+
+parser.add_argument('--qsplit', default='query', choices=['query', 'database'], type=str, help="The inferences")
+parser.add_argument('--data-dir', type=str, default=None,
+                        help='The directory of concerned dataset')
+parser.add_argument('--pt_style', default='csd', type=str)
+parser.add_argument('-a', '--arch', metavar='ARCH', default='resnet50')
+
+parser.add_argument('-j', '--workers', default=4, type=int, metavar='N',
+                    help='number of data loading workers (default: 32)')
+parser.add_argument('-b', '--batch-size', default=64, type=int,
+                    metavar='N',
+                    help='mini-batch size (default: 128), this is the total '
+                         'batch size of all GPUs on all nodes when '
+                         'using Data Parallel or Distributed Data Parallel')
+parser.add_argument('--world-size', default=-1, type=int,
+                    help='number of nodes for distributed training')
+parser.add_argument('--rank', default=-1, type=int,
+                    help='node rank for distributed training')
+parser.add_argument('--dist-url', default='tcp://224.66.41.62:23456', type=str,
+                    help='url used to set up distributed training')
+parser.add_argument('--dist-backend', default='nccl', type=str,
+                    help='distributed backend')
+parser.add_argument('--seed', default=None, type=int,
+                    help='seed for initializing training. ')
+parser.add_argument('--gpu', default=None, type=int,
+                    help='GPU id to use.')
+parser.add_argument('--multiprocessing-distributed', action='store_true',
+                    help='Use multi-processing distributed training to launch '
+                         'N processes per node, which has N GPUs. This is the '
+                         'fastest way to use PyTorch for either single node or '
+                         'multi node data parallel training')
+
+parser.add_argument('--multiscale', default=False, type=utils.bool_flag)
+
+# additional configs:
+parser.add_argument('--pretrained', default='', type=str,
+                    help='path to moco pretrained checkpoint')
+parser.add_argument('--num_loss_chunks', default=1, type=int)
+parser.add_argument('--isvit', action='store_true')
+parser.add_argument('--layer', default=1, type=int, help="layer from end to create descriptors from.")
+parser.add_argument('--feattype', default='normal', type=str, choices=['otprojected', 'weighted', 'concated', 'gram', 'normal'])
+parser.add_argument('--projdim', default=256, type=int)
+
+parser.add_argument('-mp', '--model_path', type=str, default=None)
+parser.add_argument('--gram_dims', default=1024, type=int)
+parser.add_argument('--query_count', default=-1, type=int, help='Number of queries to consider for final evaluation. Works only for domainnet')
+
+parser.add_argument('--embed_dir', default='./embeddings', type=str, help='Directory to save embeddings')
+
+## Additional config for CSD
+parser.add_argument('--eval_embed', default='head', choices=['head', 'backbone'], help="Which embed to use for eval")
+parser.add_argument('--skip_val', action='store_true')
+
+
+best_acc1 = 0
+
+
+def main():
+    args = parser.parse_args()
+
+    if args.seed is not None:
+        random.seed(args.seed)
+        torch.manual_seed(args.seed)
+        cudnn.deterministic = True
+        warnings.warn('You have chosen to seed training. '
+                      'This will turn on the CUDNN deterministic setting, '
+                      'which can slow down your training considerably! '
+                      'You may see unexpected behavior when restarting '
+                      'from checkpoints.')
+    # utils.init_distributed_mode(args)
+    if args.gpu is not None:
+        warnings.warn('You have chosen a specific GPU. This will completely '
+                      'disable data parallelism.')
+
+    if args.dist_url == "env://" and args.world_size == -1:
+        args.world_size = int(os.environ["WORLD_SIZE"])
+
+    args.distributed = args.world_size > 1 or args.multiprocessing_distributed
+
+    ngpus_per_node = torch.cuda.device_count()
+    if args.multiprocessing_distributed:
+        # Since we have ngpus_per_node processes per node, the total world_size
+        # needs to be adjusted accordingly
+        args.world_size = ngpus_per_node * args.world_size
+        # Use torch.multiprocessing.spawn to launch distributed processes: the
+        # main_worker process function
+        mp.spawn(main_worker, nprocs=ngpus_per_node, args=(ngpus_per_node, args))
+    else:
+        # Simply call main_worker function
+        main_worker(args.gpu, ngpus_per_node, args)
+
+
+def main_worker(gpu, ngpus_per_node, args):
+    global best_acc1
+    args.gpu = gpu
+
+    # suppress printing if not master
+    if args.multiprocessing_distributed and args.gpu != 0:
+        def print_pass(*args):
+            pass
+
+        builtins.print = print_pass
+
+    if args.gpu is not None:
+        print("Use GPU: {} for training".format(args.gpu))
+
+    if args.distributed:
+        if args.dist_url == "env://" and args.rank == -1:
+            args.rank = int(os.environ["RANK"])
+        if args.multiprocessing_distributed:
+            # For multiprocessing distributed training, rank needs to be the
+            # global rank among all the processes
+            args.rank = args.rank * ngpus_per_node + gpu
+        dist.init_process_group(backend=args.dist_backend, init_method=args.dist_url,
+                                world_size=args.world_size, rank=args.rank)
+        torch.distributed.barrier()
+
+    # create model
+    if args.pt_style == 'dino':
+        dinomapping = {
+            'vit_base': 'dino_vitb16',
+            'vit_base8': 'dino_vitb8',  # TODO: this mapping is incorrect. Change it later
+        }
+        if args.arch not in dinomapping:
+            raise NotImplementedError('This model type does not exist/supported for DINO')
+        model = dino_vits.__dict__[dinomapping[args.arch]](
+                pretrained=True
+            )
+    elif args.pt_style == 'moco':
+        if args.arch == 'vit_base':
+            model = moco_vits.__dict__[args.arch]()
+            pretrained = torch.load('./pretrainedmodels/vit-b-300ep.pth.tar', map_location='cpu')
+            state_dict = pretrained['state_dict']
+            for k in list(state_dict.keys()):
+                # retain only base_encoder up to before the embedding layer
+                if k.startswith('module.base_encoder'):
+                    # remove prefix
+                    state_dict[k[len("module.base_encoder."):]] = state_dict[k]
+                # delete renamed or unused k
+                del state_dict[k]
+            model.load_state_dict(state_dict, strict=False)
+        else:
+            raise NotImplementedError('This model type does not exist/supported for MoCo')
+    elif args.pt_style == 'clip':
+        from models import clip
+        clipmapping = {
+            'vit_large': 'ViT-L/14',
+            'vit_base': 'ViT-B/16',
+        }
+        if args.arch not in clipmapping:
+            raise NotImplementedError('This model type does not exist/supported for CLIP')
+        model, preprocess = clip.load(clipmapping[args.arch])
+    elif args.pt_style == 'vgg':
+        model = torchvision_models.vgg16(weights=VGG16_Weights.IMAGENET1K_V1)
+    elif args.pt_style == 'sscd':
+        if args.arch == 'resnet50':
+            model = torch.jit.load("./pretrainedmodels/sscd_disc_mixup.torchscript.pt")
+        elif args.arch == 'resnet50_disc':
+            model = torch.jit.load("./pretrainedmodels/sscd_disc_large.torchscript.pt")
+        else:
+            NotImplementedError('This model type does not exist/supported for SSCD')
+    elif args.pt_style.startswith('csd'):
+        assert args.model_path is not None, "Model path missing for CSD model"
+        from CSD.model import CSD_CLIP
+        from CSD.utils import has_batchnorms, convert_state_dict
+        from CSD.loss_utils import transforms_branch0
+
+        args.content_proj_head = "default"
+        model = CSD_CLIP(args.arch, args.content_proj_head)
+        if has_batchnorms(model):
+            model = nn.SyncBatchNorm.convert_sync_batchnorm(model)
+
+        checkpoint = torch.load(args.model_path, map_location="cpu")
+        state_dict = convert_state_dict(checkpoint['model_state_dict'])
+        msg = model.load_state_dict(state_dict, strict=False)
+        print(f"=> loaded checkpoint with msg {msg}")
+        preprocess = transforms_branch0
+
+    if not torch.cuda.is_available():
+        print('using CPU, this will be slow')
+    elif args.distributed:
+        # For multiprocessing distributed, DistributedDataParallel constructor
+        # should always set the single device scope, otherwise,
+        # DistributedDataParallel will use all available devices.
+        if args.gpu is not None:
+            torch.cuda.set_device(args.gpu)
+            model.cuda(args.gpu)
+            # When using a single GPU per process and per
+            # DistributedDataParallel, we need to divide the batch size
+            # ourselves based on the total number of GPUs we have
+            args.batch_size = int(args.batch_size / args.world_size)
+            args.workers = int((args.workers + ngpus_per_node - 1) / ngpus_per_node)
+            model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.gpu])
+        else:
+            model.cuda()
+            # DistributedDataParallel will divide and allocate batch_size to all
+            # available GPUs if device_ids are not set
+            model = torch.nn.parallel.DistributedDataParallel(model)
+    elif args.gpu is not None:
+        torch.cuda.set_device(args.gpu)
+        model = model.cuda(args.gpu)
+    else:
+        # DataParallel will divide and allocate batch_size to all available GPUs
+        if args.arch.startswith('alexnet') or args.arch.startswith('vgg'):
+            model.features = torch.nn.DataParallel(model.features)
+            model.cuda()
+        model = torch.nn.DataParallel(model).cuda()
+
+    cudnn.benchmark = True
+
+    # Data loading code
+    if args.pt_style == 'clip':  # and args.arch == 'resnet50':
+        ret_transform = preprocess
+    elif args.pt_style.startswith('csd'):
+        ret_transform = preprocess
+    elif args.pt_style in ['dino', 'moco', 'vgg']:
+        ret_transform = transforms.Compose([
+            transforms.Resize(256),
+            transforms.CenterCrop(224),
+            transforms.ToTensor(),
+            transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
+        ])
+    else:
+        ret_transform = transforms.Compose([
+            transforms.Resize(256),
+            transforms.CenterCrop(224),
+            transforms.ToTensor(),
+            transforms.Normalize([0.5], [0.5]),
+        ])
+
+    if args.dataset == 'wikiart':
+        dataset_query = WikiArtD(args.data_dir, args.qsplit, ret_transform)
+        dataset_values = WikiArtD(args.data_dir, 'database', ret_transform)
+    else:
+        raise NotImplementedError
+
+    ## creating dataloader
+    if args.distributed:
+        sampler = torch.utils.data.distributed.DistributedSampler(dataset_values, shuffle=False)
+        qsampler = torch.utils.data.distributed.DistributedSampler(dataset_query, shuffle=False)
+    else:
+        sampler = None
+        qsampler = None
+    data_loader_values = torch.utils.data.DataLoader(
+        dataset_values,
+        sampler=sampler,
+        batch_size=args.batch_size,
+        num_workers=args.workers,
+        pin_memory=True,
+        drop_last=False,
+    )
+    data_loader_query = torch.utils.data.DataLoader(
+        dataset_query,
+        sampler=qsampler,
+        batch_size=args.batch_size if args.feattype != 'gram' else 32,
+        num_workers=args.workers,
+        pin_memory=True,
+        drop_last=False,
+    )
+    print(f"train: {len(dataset_values)} imgs / query: {len(dataset_query)} imgs")
+    model.eval()
+
+    ############################################################################
+    if not args.multiprocessing_distributed:
+        utils.init_distributed_mode(args)
+    if args.rank == 0:  # only rank 0 will work from now on
+
+        # Step 1: extract features
+        os.makedirs(args.embed_dir, exist_ok=True)
+        embsavepath = os.path.join(
+            args.embed_dir,
+            f'{args.pt_style}_{args.arch}_{args.dataset}_{args.feattype}',
+            f'{str(args.layer)}')
+        if args.feattype == 'gram':
+            path1, path2 = embsavepath.split('_gram')
+            embsavepath = '_'.join([path1, 'gram', str(args.gram_dims), args.qsplit, path2])
+
+        if os.path.isfile(os.path.join(embsavepath, 'database/embeddings_0.pkl')) or args.skip_val:
+            valexist = True
+        else:
+            valexist = False
+        if args.feattype == 'gram':
+            pca_dirs, meanvals = None, None
+            query_features, pca_dirs = extract_features_pca(args, model, pca_dirs, args.gram_dims, data_loader_query,
+                                                                    False, multiscale=args.multiscale)
+            if not valexist:
+                values_features, _ = extract_features_pca(args, model, pca_dirs, args.gram_dims, data_loader_values,
+                                                          False, multiscale=args.multiscale)
+
+        elif args.pt_style.startswith('csd'):
+            from CSD.utils import extract_features
+            query_features = extract_features(model, data_loader_query, use_cuda=False, use_fp16=True, eval_embed=args.eval_embed)
+
+            if not valexist:
+                values_features = extract_features(model, data_loader_values, use_cuda=False, use_fp16=True, eval_embed=args.eval_embed)
+        else:
+            from utils import extract_features
+            query_features = extract_features(args, model, data_loader_query, False, multiscale=args.multiscale)
+            if not valexist:
+                values_features = extract_features(args, model, data_loader_values, False,
+                                                   multiscale=args.multiscale)
+
+        from search.embeddings import save_chunk
+        l_query_features = list(np.asarray(query_features.cpu().detach(), dtype=np.float16))
+
+        save_chunk(l_query_features, dataset_query.namelist, 0, f'{embsavepath}/{args.qsplit}')
+        if not valexist:
+            l_values_features = list(np.asarray(values_features.cpu().detach(), dtype=np.float16))
+            save_chunk(l_values_features, dataset_values.namelist, 0, f'{embsavepath}/database')
+
+        print(f'Embeddings saved to: {embsavepath}')
+
+
+if __name__ == '__main__':
+    main()

CSD/metrics/metrics.py

@@ -0,0 +1,74 @@
+import pathlib
+import sys
+
+import numpy as np
+
+sys.path.insert(0, str(pathlib.Path(__file__).parent.resolve()))
+
+
+class Metrics(object):
+    def __init__(self):
+        self.data = None
+
+    @staticmethod
+    def get_recall(preds, gts, topk=5):
+        preds = preds[:, :topk]
+        preds -= gts[:, None]
+        found = np.where(np.amin(np.absolute(preds), axis=1) == 0)[0]
+        return found.shape[0] / gts.shape[0]
+
+    @staticmethod
+    def get_mrr(preds, gts, topk=5):
+        preds = preds[:, :topk]
+        preds -= gts[:, None]
+        rows, cols = np.where(preds == 0)
+        _, unique_rows = np.unique(rows, return_index=True)
+        valid_cols = cols[unique_rows]
+        valid_cols += 1
+        return np.mean(1/valid_cols)
+
+    @staticmethod
+    def get_map(preds, gts, topk=5):
+        preds = preds[:, :topk]
+        preds -= gts[:, None]
+        rows, cols = np.where(preds == 0)
+        _, unique_rows = np.unique(rows, return_index=True)
+        row_cols = np.split(cols, unique_rows)[1:]
+        row_cols = [np.hstack([x[0], np.diff(x), topk - x[-1]]) for x in row_cols]
+        row_cols = [np.pad(x, (0, topk + 1 - x.shape[0]), 'constant', constant_values=(0, 0)) for x in row_cols]
+        precision = np.asarray([np.repeat(np.arange(topk + 1), x) / np.arange(1, topk + 1) for x in row_cols])
+        return np.sum(np.mean(precision, axis=1)) / preds.shape[0]
+        # numpy increasing array according to bins
+
+    @staticmethod
+    def get_recall_bin(preds, topk=5):
+        # preds is a binary matrix of size Q x K
+        preds = preds[:, :topk]
+        found = np.where(np.amax(preds, axis=1) == True)[0]
+        return found.shape[0] / preds.shape[0]
+
+    @staticmethod
+    def get_mrr_bin(preds, topk=5):
+        # preds is a binary matrix of size Q x K
+        preds = preds[:, :topk]
+        rows, cols = np.where(preds)
+        _, unique_rows = np.unique(rows, return_index=True)
+        valid_cols = cols[unique_rows]
+        valid_cols += 1
+        return np.mean(1/valid_cols)
+
+    @staticmethod
+    def get_map_bin(preds, topk=5):
+        # preds is a binary matrix of size Q x K
+        preds = preds[:, :topk]
+        rows, cols = np.where(preds)
+        _, unique_rows = np.unique(rows, return_index=True)
+        row_cols = np.split(cols, unique_rows)[1:]
+        row_cols = [np.hstack([x[0], np.diff(x), topk - x[-1]]) for x in row_cols]
+        row_cols = [np.pad(x, (0, topk + 1 - x.shape[0]), 'constant', constant_values=(0, 0)) for x in row_cols]
+        precision = np.asarray([np.repeat(np.arange(topk + 1), x) / np.arange(1, topk + 1) for x in row_cols])
+        return np.sum(np.mean(precision, axis=1)) / preds.shape[0]
+
+    @staticmethod
+    def get_per_query_precision_bin(preds):
+        return np.sum(preds, axis=1)/preds.shape[1]

CSD/models/clip/__init__.py

@@ -0,0 +1 @@
+from .clip import *

CSD/models/clip/bpe_simple_vocab_16e6.txt.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:924691ac288e54409236115652ad4aa250f48203de50a9e4722a6ecd48d6804a
+size 1356917

CSD/models/clip/clip.py

@@ -0,0 +1,237 @@
+import hashlib
+import os
+import urllib
+import warnings
+from typing import Any, Union, List
+from pkg_resources import packaging
+
+import torch
+from PIL import Image
+from torchvision.transforms import Compose, Resize, CenterCrop, ToTensor, Normalize
+from tqdm import tqdm
+
+from .model import build_model
+from .simple_tokenizer import SimpleTokenizer as _Tokenizer
+
+try:
+    from torchvision.transforms import InterpolationMode
+    BICUBIC = InterpolationMode.BICUBIC
+except ImportError:
+    BICUBIC = Image.BICUBIC
+
+
+if packaging.version.parse(torch.__version__) < packaging.version.parse("1.7.1"):
+    warnings.warn("PyTorch version 1.7.1 or higher is recommended")
+
+
+__all__ = ["available_models", "load", "tokenize"]
+_tokenizer = _Tokenizer()
+
+_MODELS = {
+    "RN50": "https://openaipublic.azureedge.net/clip/models/afeb0e10f9e5a86da6080e35cf09123aca3b358a0c3e3b6c78a7b63bc04b6762/RN50.pt",
+    "RN101": "https://openaipublic.azureedge.net/clip/models/8fa8567bab74a42d41c5915025a8e4538c3bdbe8804a470a72f30b0d94fab599/RN101.pt",
+    "RN50x4": "https://openaipublic.azureedge.net/clip/models/7e526bd135e493cef0776de27d5f42653e6b4c8bf9e0f653bb11773263205fdd/RN50x4.pt",
+    "RN50x16": "https://openaipublic.azureedge.net/clip/models/52378b407f34354e150460fe41077663dd5b39c54cd0bfd2b27167a4a06ec9aa/RN50x16.pt",
+    "RN50x64": "https://openaipublic.azureedge.net/clip/models/be1cfb55d75a9666199fb2206c106743da0f6468c9d327f3e0d0a543a9919d9c/RN50x64.pt",
+    "ViT-B/32": "https://openaipublic.azureedge.net/clip/models/40d365715913c9da98579312b702a82c18be219cc2a73407c4526f58eba950af/ViT-B-32.pt",
+    "ViT-B/16": "https://openaipublic.azureedge.net/clip/models/5806e77cd80f8b59890b7e101eabd078d9fb84e6937f9e85e4ecb61988df416f/ViT-B-16.pt",
+    "ViT-L/14": "https://openaipublic.azureedge.net/clip/models/b8cca3fd41ae0c99ba7e8951adf17d267cdb84cd88be6f7c2e0eca1737a03836/ViT-L-14.pt",
+    "ViT-L/14@336px": "https://openaipublic.azureedge.net/clip/models/3035c92b350959924f9f00213499208652fc7ea050643e8b385c2dac08641f02/ViT-L-14-336px.pt",
+}
+
+
+def _download(url: str, root: str):
+    os.makedirs(root, exist_ok=True)
+    filename = os.path.basename(url)
+
+    expected_sha256 = url.split("/")[-2]
+    download_target = os.path.join(root, filename)
+
+    if os.path.exists(download_target) and not os.path.isfile(download_target):
+        raise RuntimeError(f"{download_target} exists and is not a regular file")
+
+    if os.path.isfile(download_target):
+        if hashlib.sha256(open(download_target, "rb").read()).hexdigest() == expected_sha256:
+            return download_target
+        else:
+            warnings.warn(f"{download_target} exists, but the SHA256 checksum does not match; re-downloading the file")
+
+    with urllib.request.urlopen(url) as source, open(download_target, "wb") as output:
+        with tqdm(total=int(source.info().get("Content-Length")), ncols=80, unit='iB', unit_scale=True, unit_divisor=1024) as loop:
+            while True:
+                buffer = source.read(8192)
+                if not buffer:
+                    break
+
+                output.write(buffer)
+                loop.update(len(buffer))
+
+    if hashlib.sha256(open(download_target, "rb").read()).hexdigest() != expected_sha256:
+        raise RuntimeError("Model has been downloaded but the SHA256 checksum does not not match")
+
+    return download_target
+
+
+def _convert_image_to_rgb(image):
+    return image.convert("RGB")
+
+
+def _transform(n_px):
+    return Compose([
+        Resize(n_px, interpolation=BICUBIC),
+        CenterCrop(n_px),
+        _convert_image_to_rgb,
+        ToTensor(),
+        Normalize((0.48145466, 0.4578275, 0.40821073), (0.26862954, 0.26130258, 0.27577711)),
+    ])
+
+
+def available_models() -> List[str]:
+    """Returns the names of available CLIP models"""
+    return list(_MODELS.keys())
+
+
+def load(name: str, device: Union[str, torch.device] = "cuda" if torch.cuda.is_available() else "cpu", jit: bool = False, download_root: str = None):
+    """Load a CLIP model
+
+    Parameters
+    ----------
+    name : str
+        A model name listed by `clip.available_models()`, or the path to a model checkpoint containing the state_dict
+
+    device : Union[str, torch.device]
+        The device to put the loaded model
+
+    jit : bool
+        Whether to load the optimized JIT model or more hackable non-JIT model (default).
+
+    download_root: str
+        path to download the model files; by default, it uses "~/.cache/clip"
+
+    Returns
+    -------
+    model : torch.nn.Module
+        The CLIP model
+
+    preprocess : Callable[[PIL.Image], torch.Tensor]
+        A torchvision transform that converts a PIL image into a tensor that the returned model can take as its input
+    """
+    if name in _MODELS:
+        model_path = _download(_MODELS[name], download_root or os.path.expanduser("~/.cache/clip"))
+    elif os.path.isfile(name):
+        model_path = name
+    else:
+        raise RuntimeError(f"Model {name} not found; available models = {available_models()}")
+
+    with open(model_path, 'rb') as opened_file:
+        try:
+            # loading JIT archive
+            model = torch.jit.load(opened_file, map_location=device if jit else "cpu").eval()
+            state_dict = None
+        except RuntimeError:
+            # loading saved state dict
+            if jit:
+                warnings.warn(f"File {model_path} is not a JIT archive. Loading as a state dict instead")
+                jit = False
+            state_dict = torch.load(opened_file, map_location="cpu")
+
+    if not jit:
+        model = build_model(state_dict or model.state_dict()).to(device)
+        if str(device) == "cpu":
+            model.float()
+        return model, _transform(model.visual.input_resolution)
+
+    # patch the device names
+    device_holder = torch.jit.trace(lambda: torch.ones([]).to(torch.device(device)), example_inputs=[])
+    device_node = [n for n in device_holder.graph.findAllNodes("prim::Constant") if "Device" in repr(n)][-1]
+
+    def patch_device(module):
+        try:
+            graphs = [module.graph] if hasattr(module, "graph") else []
+        except RuntimeError:
+            graphs = []
+
+        if hasattr(module, "forward1"):
+            graphs.append(module.forward1.graph)
+
+        for graph in graphs:
+            for node in graph.findAllNodes("prim::Constant"):
+                if "value" in node.attributeNames() and str(node["value"]).startswith("cuda"):
+                    node.copyAttributes(device_node)
+
+    model.apply(patch_device)
+    patch_device(model.encode_image)
+    patch_device(model.encode_text)
+
+    # patch dtype to float32 on CPU
+    if str(device) == "cpu":
+        float_holder = torch.jit.trace(lambda: torch.ones([]).float(), example_inputs=[])
+        float_input = list(float_holder.graph.findNode("aten::to").inputs())[1]
+        float_node = float_input.node()
+
+        def patch_float(module):
+            try:
+                graphs = [module.graph] if hasattr(module, "graph") else []
+            except RuntimeError:
+                graphs = []
+
+            if hasattr(module, "forward1"):
+                graphs.append(module.forward1.graph)
+
+            for graph in graphs:
+                for node in graph.findAllNodes("aten::to"):
+                    inputs = list(node.inputs())
+                    for i in [1, 2]:  # dtype can be the second or third argument to aten::to()
+                        if inputs[i].node()["value"] == 5:
+                            inputs[i].node().copyAttributes(float_node)
+
+        model.apply(patch_float)
+        patch_float(model.encode_image)
+        patch_float(model.encode_text)
+
+        model.float()
+
+    return model, _transform(model.input_resolution.item())
+
+
+def tokenize(texts: Union[str, List[str]], context_length: int = 77, truncate: bool = False) -> Union[torch.IntTensor, torch.LongTensor]:
+    """
+    Returns the tokenized representation of given input string(s)
+
+    Parameters
+    ----------
+    texts : Union[str, List[str]]
+        An input string or a list of input strings to tokenize
+
+    context_length : int
+        The context length to use; all CLIP models use 77 as the context length
+
+    truncate: bool
+        Whether to truncate the text in case its encoding is longer than the context length
+
+    Returns
+    -------
+    A two-dimensional tensor containing the resulting tokens, shape = [number of input strings, context_length].
+    We return LongTensor when torch version is <1.8.0, since older index_select requires indices to be long.
+    """
+    if isinstance(texts, str):
+        texts = [texts]
+
+    sot_token = _tokenizer.encoder["<|startoftext|>"]
+    eot_token = _tokenizer.encoder["<|endoftext|>"]
+    all_tokens = [[sot_token] + _tokenizer.encode(text) + [eot_token] for text in texts]
+    if packaging.version.parse(torch.__version__) < packaging.version.parse("1.8.0"):
+        result = torch.zeros(len(all_tokens), context_length, dtype=torch.long)
+    else:
+        result = torch.zeros(len(all_tokens), context_length, dtype=torch.int)
+
+    for i, tokens in enumerate(all_tokens):
+        if len(tokens) > context_length:
+            if truncate:
+                tokens = tokens[:context_length]
+                tokens[-1] = eot_token
+            else:
+                raise RuntimeError(f"Input {texts[i]} is too long for context length {context_length}")
+        result[i, :len(tokens)] = torch.tensor(tokens)
+
+    return result

CSD/models/clip/model.py

@@ -0,0 +1,486 @@
+from collections import OrderedDict
+from typing import Tuple, Union
+
+import numpy as np
+import torch
+import torch.nn.functional as F
+from torch import nn
+
+
+class Bottleneck(nn.Module):
+    expansion = 4
+
+    def __init__(self, inplanes, planes, stride=1):
+        super().__init__()
+
+        # all conv layers have stride 1. an avgpool is performed after the second convolution when stride > 1
+        self.conv1 = nn.Conv2d(inplanes, planes, 1, bias=False)
+        self.bn1 = nn.BatchNorm2d(planes)
+        self.relu1 = nn.ReLU(inplace=True)
+
+        self.conv2 = nn.Conv2d(planes, planes, 3, padding=1, bias=False)
+        self.bn2 = nn.BatchNorm2d(planes)
+        self.relu2 = nn.ReLU(inplace=True)
+
+        self.avgpool = nn.AvgPool2d(stride) if stride > 1 else nn.Identity()
+
+        self.conv3 = nn.Conv2d(planes, planes * self.expansion, 1, bias=False)
+        self.bn3 = nn.BatchNorm2d(planes * self.expansion)
+        self.relu3 = nn.ReLU(inplace=True)
+
+        self.downsample = None
+        self.stride = stride
+
+        if stride > 1 or inplanes != planes * Bottleneck.expansion:
+            # downsampling layer is prepended with an avgpool, and the subsequent convolution has stride 1
+            self.downsample = nn.Sequential(OrderedDict([
+                ("-1", nn.AvgPool2d(stride)),
+                ("0", nn.Conv2d(inplanes, planes * self.expansion, 1, stride=1, bias=False)),
+                ("1", nn.BatchNorm2d(planes * self.expansion))
+            ]))
+
+    def forward(self, x: torch.Tensor):
+        identity = x
+
+        out = self.relu1(self.bn1(self.conv1(x)))
+        out = self.relu2(self.bn2(self.conv2(out)))
+        out = self.avgpool(out)
+        out = self.bn3(self.conv3(out))
+
+        if self.downsample is not None:
+            identity = self.downsample(x)
+
+        out += identity
+        out = self.relu3(out)
+        return out
+
+
+class AttentionPool2d(nn.Module):
+    def __init__(self, spacial_dim: int, embed_dim: int, num_heads: int, output_dim: int = None):
+        super().__init__()
+        self.positional_embedding = nn.Parameter(torch.randn(spacial_dim ** 2 + 1, embed_dim) / embed_dim ** 0.5)
+        self.k_proj = nn.Linear(embed_dim, embed_dim)
+        self.q_proj = nn.Linear(embed_dim, embed_dim)
+        self.v_proj = nn.Linear(embed_dim, embed_dim)
+        self.c_proj = nn.Linear(embed_dim, output_dim or embed_dim)
+        self.num_heads = num_heads
+
+    def forward(self, x):
+        x = x.flatten(start_dim=2).permute(2, 0, 1)  # NCHW -> (HW)NC
+        x = torch.cat([x.mean(dim=0, keepdim=True), x], dim=0)  # (HW+1)NC
+        x = x + self.positional_embedding[:, None, :].to(x.dtype)  # (HW+1)NC
+        x, _ = F.multi_head_attention_forward(
+            query=x[:1], key=x, value=x,
+            embed_dim_to_check=x.shape[-1],
+            num_heads=self.num_heads,
+            q_proj_weight=self.q_proj.weight,
+            k_proj_weight=self.k_proj.weight,
+            v_proj_weight=self.v_proj.weight,
+            in_proj_weight=None,
+            in_proj_bias=torch.cat([self.q_proj.bias, self.k_proj.bias, self.v_proj.bias]),
+            bias_k=None,
+            bias_v=None,
+            add_zero_attn=False,
+            dropout_p=0,
+            out_proj_weight=self.c_proj.weight,
+            out_proj_bias=self.c_proj.bias,
+            use_separate_proj_weight=True,
+            training=self.training,
+            need_weights=False
+        )
+        return x.squeeze(0)
+
+
+class ModifiedResNet(nn.Module):
+    """
+    A ResNet class that is similar to torchvision's but contains the following changes:
+    - There are now 3 "stem" convolutions as opposed to 1, with an average pool instead of a max pool.
+    - Performs anti-aliasing strided convolutions, where an avgpool is prepended to convolutions with stride > 1
+    - The final pooling layer is a QKV attention instead of an average pool
+    """
+
+    def __init__(self, layers, output_dim, heads, input_resolution=224, width=64):
+        super().__init__()
+        self.output_dim = output_dim
+        self.input_resolution = input_resolution
+
+        # the 3-layer stem
+        self.conv1 = nn.Conv2d(3, width // 2, kernel_size=3, stride=2, padding=1, bias=False)
+        self.bn1 = nn.BatchNorm2d(width // 2)
+        self.relu1 = nn.ReLU(inplace=True)
+        self.conv2 = nn.Conv2d(width // 2, width // 2, kernel_size=3, padding=1, bias=False)
+        self.bn2 = nn.BatchNorm2d(width // 2)
+        self.relu2 = nn.ReLU(inplace=True)
+        self.conv3 = nn.Conv2d(width // 2, width, kernel_size=3, padding=1, bias=False)
+        self.bn3 = nn.BatchNorm2d(width)
+        self.relu3 = nn.ReLU(inplace=True)
+        self.avgpool = nn.AvgPool2d(2)
+
+        # residual layers
+        self._inplanes = width  # this is a *mutable* variable used during construction
+        self.layer1 = self._make_layer(width, layers[0])
+        self.layer2 = self._make_layer(width * 2, layers[1], stride=2)
+        self.layer3 = self._make_layer(width * 4, layers[2], stride=2)
+        self.layer4 = self._make_layer(width * 8, layers[3], stride=2)
+
+        embed_dim = width * 32  # the ResNet feature dimension
+        self.attnpool = AttentionPool2d(input_resolution // 32, embed_dim, heads, output_dim)
+
+    def _make_layer(self, planes, blocks, stride=1):
+        layers = [Bottleneck(self._inplanes, planes, stride)]
+
+        self._inplanes = planes * Bottleneck.expansion
+        for _ in range(1, blocks):
+            layers.append(Bottleneck(self._inplanes, planes))
+
+        return nn.Sequential(*layers)
+
+    def forward(self, x):
+        def stem(x):
+            x = self.relu1(self.bn1(self.conv1(x)))
+            x = self.relu2(self.bn2(self.conv2(x)))
+            x = self.relu3(self.bn3(self.conv3(x)))
+            x = self.avgpool(x)
+            return x
+
+        x = x.type(self.conv1.weight.dtype)
+        x = stem(x)
+        x = self.layer1(x)
+        x = self.layer2(x)
+        x = self.layer3(x)
+        x = self.layer4(x)
+        x = self.attnpool(x)
+
+        return x
+    def get_intermediate_layers(self, x): 
+        def stem(x):
+            x = self.relu1(self.bn1(self.conv1(x)))
+            x = self.relu2(self.bn2(self.conv2(x)))
+            x = self.relu3(self.bn3(self.conv3(x)))
+            x = self.avgpool(x)
+            return x
+
+        x = x.type(self.conv1.weight.dtype)
+        output = []
+        x = stem(x)
+        output.append(x)
+        x = self.layer1(x)
+        output.append(x)
+        x = self.layer2(x)
+        output.append(x)
+        x = self.layer3(x)
+        output.append(x)
+        x = self.layer4(x)
+        output.append(x)
+        x = self.attnpool(x)
+        output.append(x)
+        return output
+
+class LayerNorm(nn.LayerNorm):
+    """Subclass torch's LayerNorm to handle fp16."""
+
+    def forward(self, x: torch.Tensor):
+        orig_type = x.dtype
+        ret = super().forward(x.type(torch.float32))
+        return ret.type(orig_type)
+
+
+class QuickGELU(nn.Module):
+    def forward(self, x: torch.Tensor):
+        return x * torch.sigmoid(1.702 * x)
+
+
+class ResidualAttentionBlock(nn.Module):
+    def __init__(self, d_model: int, n_head: int, attn_mask: torch.Tensor = None):
+        super().__init__()
+
+        self.attn = nn.MultiheadAttention(d_model, n_head)
+        self.ln_1 = LayerNorm(d_model)
+        self.mlp = nn.Sequential(OrderedDict([
+            ("c_fc", nn.Linear(d_model, d_model * 4)),
+            ("gelu", QuickGELU()),
+            ("c_proj", nn.Linear(d_model * 4, d_model))
+        ]))
+        self.ln_2 = LayerNorm(d_model)
+        self.attn_mask = attn_mask
+
+    def attention(self, x: torch.Tensor):
+        self.attn_mask = self.attn_mask.to(dtype=x.dtype, device=x.device) if self.attn_mask is not None else None
+        return self.attn(x, x, x, need_weights=False, attn_mask=self.attn_mask)[0]
+
+    def forward(self, x: torch.Tensor):
+        x = x + self.attention(self.ln_1(x))
+        x = x + self.mlp(self.ln_2(x))
+        return x
+
+
+class Transformer(nn.Module):
+    def __init__(self, width: int, layers: int, heads: int, attn_mask: torch.Tensor = None):
+        super().__init__()
+        self.width = width
+        self.layers = layers
+        self.resblocks = nn.Sequential(*[ResidualAttentionBlock(width, heads, attn_mask) for _ in range(layers)])
+
+    def forward(self, x: torch.Tensor):
+        return self.resblocks(x)
+    
+    def get_activations(self, x: torch.Tensor):
+        output = []
+        
+        for i in range(self.layers):
+            # import ipdb; ipdb.set_trace()
+            x = self.resblocks[i](x)
+            output.append(x.permute(1, 0, 2))
+        return output
+
+
+
+class VisionTransformer(nn.Module):
+    def __init__(self, input_resolution: int, patch_size: int, width: int, layers: int, heads: int, output_dim: int):
+        super().__init__()
+        self.input_resolution = input_resolution
+        self.output_dim = output_dim
+        self.conv1 = nn.Conv2d(in_channels=3, out_channels=width, kernel_size=patch_size, stride=patch_size, bias=False)
+
+        scale = width ** -0.5
+        self.class_embedding = nn.Parameter(scale * torch.randn(width))
+        self.positional_embedding = nn.Parameter(scale * torch.randn((input_resolution // patch_size) ** 2 + 1, width))
+        self.ln_pre = LayerNorm(width)
+
+        self.transformer = Transformer(width, layers, heads)
+
+        self.ln_post = LayerNorm(width)
+        self.proj = nn.Parameter(scale * torch.randn(width, output_dim))
+
+    def forward(self, x: torch.Tensor):
+        x = self.conv1(x)  # shape = [*, width, grid, grid]
+        x = x.reshape(x.shape[0], x.shape[1], -1)  # shape = [*, width, grid ** 2]
+        x = x.permute(0, 2, 1)  # shape = [*, grid ** 2, width]
+        x = torch.cat([self.class_embedding.to(x.dtype) + torch.zeros(x.shape[0], 1, x.shape[-1], dtype=x.dtype, device=x.device), x], dim=1)  # shape = [*, grid ** 2 + 1, width]
+        x = x + self.positional_embedding.to(x.dtype)
+        x = self.ln_pre(x)
+
+        x = x.permute(1, 0, 2)  # NLD -> LND
+        x = self.transformer(x)
+        x = x.permute(1, 0, 2)  # LND -> NLD
+
+        x = self.ln_post(x[:, 0, :])
+
+        if self.proj is not None:
+            x = x @ self.proj
+
+        return x
+
+    def get_intermediate_layers(self, x: torch.Tensor):
+        x = self.conv1(x)  # shape = [*, width, grid, grid]
+        x = x.reshape(x.shape[0], x.shape[1], -1)  # shape = [*, width, grid ** 2]
+        x = x.permute(0, 2, 1)  # shape = [*, grid ** 2, width]
+        x = torch.cat([self.class_embedding.to(x.dtype) + torch.zeros(x.shape[0], 1, x.shape[-1], dtype=x.dtype, device=x.device), x], dim=1)  # shape = [*, grid ** 2 + 1, width]
+        x = x + self.positional_embedding.to(x.dtype)
+        x = self.ln_pre(x)
+
+        x = x.permute(1, 0, 2)  # NLD -> LND
+        # x = self.transformer(x)
+        op = self.transformer.get_activations(x)
+        # x = x.permute(1, 0, 2)  # LND -> NLD
+
+        # x = self.ln_post(x[:, 0, :])
+
+        # if self.proj is not None:
+        #     x = x @ self.proj
+        return op
+    
+class CLIP(nn.Module):
+    def __init__(self,
+                 embed_dim: int,
+                 # vision
+                 image_resolution: int,
+                 vision_layers: Union[Tuple[int, int, int, int], int],
+                 vision_width: int,
+                 vision_patch_size: int,
+                 # text
+                 context_length: int,
+                 vocab_size: int,
+                 transformer_width: int,
+                 transformer_heads: int,
+                 transformer_layers: int
+                 ):
+        super().__init__()
+
+        self.context_length = context_length
+
+        if isinstance(vision_layers, (tuple, list)):
+            vision_heads = vision_width * 32 // 64
+            self.visual = ModifiedResNet(
+                layers=vision_layers,
+                output_dim=embed_dim,
+                heads=vision_heads,
+                input_resolution=image_resolution,
+                width=vision_width
+            )
+        else:
+            vision_heads = vision_width // 64
+            self.visual = VisionTransformer(
+                input_resolution=image_resolution,
+                patch_size=vision_patch_size,
+                width=vision_width,
+                layers=vision_layers,
+                heads=vision_heads,
+                output_dim=embed_dim
+            )
+
+        self.transformer = Transformer(
+            width=transformer_width,
+            layers=transformer_layers,
+            heads=transformer_heads,
+            attn_mask=self.build_attention_mask()
+        )
+
+        self.vocab_size = vocab_size
+        self.token_embedding = nn.Embedding(vocab_size, transformer_width)
+        self.positional_embedding = nn.Parameter(torch.empty(self.context_length, transformer_width))
+        self.ln_final = LayerNorm(transformer_width)
+
+        self.text_projection = nn.Parameter(torch.empty(transformer_width, embed_dim))
+        self.logit_scale = nn.Parameter(torch.ones([]) * np.log(1 / 0.07))
+
+        self.initialize_parameters()
+
+    def initialize_parameters(self):
+        nn.init.normal_(self.token_embedding.weight, std=0.02)
+        nn.init.normal_(self.positional_embedding, std=0.01)
+
+        if isinstance(self.visual, ModifiedResNet):
+            if self.visual.attnpool is not None:
+                std = self.visual.attnpool.c_proj.in_features ** -0.5
+                nn.init.normal_(self.visual.attnpool.q_proj.weight, std=std)
+                nn.init.normal_(self.visual.attnpool.k_proj.weight, std=std)
+                nn.init.normal_(self.visual.attnpool.v_proj.weight, std=std)
+                nn.init.normal_(self.visual.attnpool.c_proj.weight, std=std)
+
+            for resnet_block in [self.visual.layer1, self.visual.layer2, self.visual.layer3, self.visual.layer4]:
+                for name, param in resnet_block.named_parameters():
+                    if name.endswith("bn3.weight"):
+                        nn.init.zeros_(param)
+
+        proj_std = (self.transformer.width ** -0.5) * ((2 * self.transformer.layers) ** -0.5)
+        attn_std = self.transformer.width ** -0.5
+        fc_std = (2 * self.transformer.width) ** -0.5
+        for block in self.transformer.resblocks:
+            nn.init.normal_(block.attn.in_proj_weight, std=attn_std)
+            nn.init.normal_(block.attn.out_proj.weight, std=proj_std)
+            nn.init.normal_(block.mlp.c_fc.weight, std=fc_std)
+            nn.init.normal_(block.mlp.c_proj.weight, std=proj_std)
+
+        if self.text_projection is not None:
+            nn.init.normal_(self.text_projection, std=self.transformer.width ** -0.5)
+
+    def build_attention_mask(self):
+        # lazily create causal attention mask, with full attention between the vision tokens
+        # pytorch uses additive attention mask; fill with -inf
+        mask = torch.empty(self.context_length, self.context_length)
+        mask.fill_(float("-inf"))
+        mask.triu_(1)  # zero out the lower diagonal
+        return mask
+
+    @property
+    def dtype(self):
+        return self.visual.conv1.weight.dtype
+
+    def encode_image(self, image):
+        return self.visual(image.type(self.dtype))
+
+    def encode_text(self, text):
+        x = self.token_embedding(text).type(self.dtype)  # [batch_size, n_ctx, d_model]
+
+        x = x + self.positional_embedding.type(self.dtype)
+        x = x.permute(1, 0, 2)  # NLD -> LND
+        x = self.transformer(x)
+        x = x.permute(1, 0, 2)  # LND -> NLD
+        x = self.ln_final(x).type(self.dtype)
+
+        # x.shape = [batch_size, n_ctx, transformer.width]
+        # take features from the eot embedding (eot_token is the highest number in each sequence)
+        x = x[torch.arange(x.shape[0]), text.argmax(dim=-1)] @ self.text_projection
+
+        return x
+
+    def forward(self, image, text):
+        image_features = self.encode_image(image)
+        text_features = self.encode_text(text)
+
+        # normalized features
+        image_features = image_features / image_features.norm(dim=1, keepdim=True)
+        text_features = text_features / text_features.norm(dim=1, keepdim=True)
+
+        # cosine similarity as logits
+        logit_scale = self.logit_scale.exp()
+        logits_per_image = logit_scale * image_features @ text_features.t()
+        logits_per_text = logits_per_image.t()
+
+        # shape = [global_batch_size, global_batch_size]
+        return logits_per_image, logits_per_text
+
+
+def convert_weights(model: nn.Module):
+    """Convert applicable model parameters to fp16"""
+
+    def _convert_weights_to_fp16(l):
+        if isinstance(l, (nn.Conv1d, nn.Conv2d, nn.Linear)):
+            l.weight.data = l.weight.data.half()
+            if l.bias is not None:
+                l.bias.data = l.bias.data.half()
+
+        if isinstance(l, nn.MultiheadAttention):
+            for attr in [*[f"{s}_proj_weight" for s in ["in", "q", "k", "v"]], "in_proj_bias", "bias_k", "bias_v"]:
+                tensor = getattr(l, attr)
+                if tensor is not None:
+                    tensor.data = tensor.data.half()
+
+        for name in ["text_projection", "proj"]:
+            if hasattr(l, name):
+                attr = getattr(l, name)
+                if attr is not None:
+                    attr.data = attr.data.half()
+
+    model.apply(_convert_weights_to_fp16)
+
+
+def build_model(state_dict: dict):
+    vit = "visual.proj" in state_dict
+
+    if vit:
+        vision_width = state_dict["visual.conv1.weight"].shape[0]
+        vision_layers = len([k for k in state_dict.keys() if k.startswith("visual.") and k.endswith(".attn.in_proj_weight")])
+        vision_patch_size = state_dict["visual.conv1.weight"].shape[-1]
+        grid_size = round((state_dict["visual.positional_embedding"].shape[0] - 1) ** 0.5)
+        image_resolution = vision_patch_size * grid_size
+    else:
+        counts: list = [len(set(k.split(".")[2] for k in state_dict if k.startswith(f"visual.layer{b}"))) for b in [1, 2, 3, 4]]
+        vision_layers = tuple(counts)
+        vision_width = state_dict["visual.layer1.0.conv1.weight"].shape[0]
+        output_width = round((state_dict["visual.attnpool.positional_embedding"].shape[0] - 1) ** 0.5)
+        vision_patch_size = None
+        assert output_width ** 2 + 1 == state_dict["visual.attnpool.positional_embedding"].shape[0]
+        image_resolution = output_width * 32
+
+    embed_dim = state_dict["text_projection"].shape[1]
+    context_length = state_dict["positional_embedding"].shape[0]
+    vocab_size = state_dict["token_embedding.weight"].shape[0]
+    transformer_width = state_dict["ln_final.weight"].shape[0]
+    transformer_heads = transformer_width // 64
+    transformer_layers = len(set(k.split(".")[2] for k in state_dict if k.startswith("transformer.resblocks")))
+
+    model = CLIP(
+        embed_dim,
+        image_resolution, vision_layers, vision_width, vision_patch_size,
+        context_length, vocab_size, transformer_width, transformer_heads, transformer_layers
+    )
+
+    for key in ["input_resolution", "context_length", "vocab_size"]:
+        if key in state_dict:
+            del state_dict[key]
+
+    convert_weights(model)
+    model.load_state_dict(state_dict)
+    return model.eval()

CSD/models/clip/simple_tokenizer.py

@@ -0,0 +1,132 @@
+import gzip
+import html
+import os
+from functools import lru_cache
+
+import ftfy
+import regex as re
+
+
+@lru_cache()
+def default_bpe():
+    return os.path.join(os.path.dirname(os.path.abspath(__file__)), "bpe_simple_vocab_16e6.txt.gz")
+
+
+@lru_cache()
+def bytes_to_unicode():
+    """
+    Returns list of utf-8 byte and a corresponding list of unicode strings.
+    The reversible bpe codes work on unicode strings.
+    This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.
+    When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.
+    This is a signficant percentage of your normal, say, 32K bpe vocab.
+    To avoid that, we want lookup tables between utf-8 bytes and unicode strings.
+    And avoids mapping to whitespace/control characters the bpe code barfs on.
+    """
+    bs = list(range(ord("!"), ord("~")+1))+list(range(ord("¡"), ord("¬")+1))+list(range(ord("®"), ord("ÿ")+1))
+    cs = bs[:]
+    n = 0
+    for b in range(2**8):
+        if b not in bs:
+            bs.append(b)
+            cs.append(2**8+n)
+            n += 1
+    cs = [chr(n) for n in cs]
+    return dict(zip(bs, cs))
+
+
+def get_pairs(word):
+    """Return set of symbol pairs in a word.
+    Word is represented as tuple of symbols (symbols being variable-length strings).
+    """
+    pairs = set()
+    prev_char = word[0]
+    for char in word[1:]:
+        pairs.add((prev_char, char))
+        prev_char = char
+    return pairs
+
+
+def basic_clean(text):
+    text = ftfy.fix_text(text)
+    text = html.unescape(html.unescape(text))
+    return text.strip()
+
+
+def whitespace_clean(text):
+    text = re.sub(r'\s+', ' ', text)
+    text = text.strip()
+    return text
+
+
+class SimpleTokenizer(object):
+    def __init__(self, bpe_path: str = default_bpe()):
+        self.byte_encoder = bytes_to_unicode()
+        self.byte_decoder = {v: k for k, v in self.byte_encoder.items()}
+        merges = gzip.open(bpe_path).read().decode("utf-8").split('\n')
+        merges = merges[1:49152-256-2+1]
+        merges = [tuple(merge.split()) for merge in merges]
+        vocab = list(bytes_to_unicode().values())
+        vocab = vocab + [v+'</w>' for v in vocab]
+        for merge in merges:
+            vocab.append(''.join(merge))
+        vocab.extend(['<|startoftext|>', '<|endoftext|>'])
+        self.encoder = dict(zip(vocab, range(len(vocab))))
+        self.decoder = {v: k for k, v in self.encoder.items()}
+        self.bpe_ranks = dict(zip(merges, range(len(merges))))
+        self.cache = {'<|startoftext|>': '<|startoftext|>', '<|endoftext|>': '<|endoftext|>'}
+        self.pat = re.compile(r"""<\|startoftext\|>|<\|endoftext\|>|'s|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+""", re.IGNORECASE)
+
+    def bpe(self, token):
+        if token in self.cache:
+            return self.cache[token]
+        word = tuple(token[:-1]) + ( token[-1] + '</w>',)
+        pairs = get_pairs(word)
+
+        if not pairs:
+            return token+'</w>'
+
+        while True:
+            bigram = min(pairs, key = lambda pair: self.bpe_ranks.get(pair, float('inf')))
+            if bigram not in self.bpe_ranks:
+                break
+            first, second = bigram
+            new_word = []
+            i = 0
+            while i < len(word):
+                try:
+                    j = word.index(first, i)
+                    new_word.extend(word[i:j])
+                    i = j
+                except:
+                    new_word.extend(word[i:])
+                    break
+
+                if word[i] == first and i < len(word)-1 and word[i+1] == second:
+                    new_word.append(first+second)
+                    i += 2
+                else:
+                    new_word.append(word[i])
+                    i += 1
+            new_word = tuple(new_word)
+            word = new_word
+            if len(word) == 1:
+                break
+            else:
+                pairs = get_pairs(word)
+        word = ' '.join(word)
+        self.cache[token] = word
+        return word
+
+    def encode(self, text):
+        bpe_tokens = []
+        text = whitespace_clean(basic_clean(text)).lower()
+        for token in re.findall(self.pat, text):
+            token = ''.join(self.byte_encoder[b] for b in token.encode('utf-8'))
+            bpe_tokens.extend(self.encoder[bpe_token] for bpe_token in self.bpe(token).split(' '))
+        return bpe_tokens
+
+    def decode(self, tokens):
+        text = ''.join([self.decoder[token] for token in tokens])
+        text = bytearray([self.byte_decoder[c] for c in text]).decode('utf-8', errors="replace").replace('</w>', ' ')
+        return text

CSD/models/dino_vits.py

@@ -0,0 +1,485 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Copied from dino transformers and added the global pool layer
+"""
+import math
+from functools import partial
+import warnings
+
+import torch
+import torch.nn as nn
+
+
+def _no_grad_trunc_normal_(tensor, mean, std, a, b):
+    # Cut & paste from PyTorch official master until it's in a few official releases - RW
+    # Method based on https://people.sc.fsu.edu/~jburkardt/presentations/truncated_normal.pdf
+    def norm_cdf(x):
+        # Computes standard normal cumulative distribution function
+        return (1. + math.erf(x / math.sqrt(2.))) / 2.
+
+    if (mean < a - 2 * std) or (mean > b + 2 * std):
+        warnings.warn("mean is more than 2 std from [a, b] in nn.init.trunc_normal_. "
+                      "The distribution of values may be incorrect.",
+                      stacklevel=2)
+
+    with torch.no_grad():
+        # Values are generated by using a truncated uniform distribution and
+        # then using the inverse CDF for the normal distribution.
+        # Get upper and lower cdf values
+        l = norm_cdf((a - mean) / std)
+        u = norm_cdf((b - mean) / std)
+
+        # Uniformly fill tensor with values from [l, u], then translate to
+        # [2l-1, 2u-1].
+        tensor.uniform_(2 * l - 1, 2 * u - 1)
+
+        # Use inverse cdf transform for normal distribution to get truncated
+        # standard normal
+        tensor.erfinv_()
+
+        # Transform to proper mean, std
+        tensor.mul_(std * math.sqrt(2.))
+        tensor.add_(mean)
+
+        # Clamp to ensure it's in the proper range
+        tensor.clamp_(min=a, max=b)
+        return tensor
+
+
+def trunc_normal_(tensor, mean=0., std=1., a=-2., b=2.):
+    # type: (Tensor, float, float, float, float) -> Tensor
+    return _no_grad_trunc_normal_(tensor, mean, std, a, b)
+
+
+def drop_path(x, drop_prob: float = 0., training: bool = False):
+    if drop_prob == 0. or not training:
+        return x
+    keep_prob = 1 - drop_prob
+    shape = (x.shape[0],) + (1,) * (x.ndim - 1)  # work with diff dim tensors, not just 2D ConvNets
+    random_tensor = keep_prob + torch.rand(shape, dtype=x.dtype, device=x.device)
+    random_tensor.floor_()  # binarize
+    output = x.div(keep_prob) * random_tensor
+    return output
+
+
+class DropPath(nn.Module):
+    """Drop paths (Stochastic Depth) per sample  (when applied in main path of residual blocks).
+    """
+    def __init__(self, drop_prob=None):
+        super(DropPath, self).__init__()
+        self.drop_prob = drop_prob
+
+    def forward(self, x):
+        return drop_path(x, self.drop_prob, self.training)
+
+
+class Mlp(nn.Module):
+    def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU, drop=0.):
+        super().__init__()
+        out_features = out_features or in_features
+        hidden_features = hidden_features or in_features
+        self.fc1 = nn.Linear(in_features, hidden_features)
+        self.act = act_layer()
+        self.fc2 = nn.Linear(hidden_features, out_features)
+        self.drop = nn.Dropout(drop)
+
+    def forward(self, x):
+        x = self.fc1(x)
+        x = self.act(x)
+        x = self.drop(x)
+        x = self.fc2(x)
+        x = self.drop(x)
+        return x
+
+
+class Attention(nn.Module):
+    def __init__(self, dim, num_heads=8, qkv_bias=False, qk_scale=None, attn_drop=0., proj_drop=0.):
+        super().__init__()
+        self.num_heads = num_heads
+        head_dim = dim // num_heads
+        self.scale = qk_scale or head_dim ** -0.5
+
+        self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
+        self.attn_drop = nn.Dropout(attn_drop)
+        self.proj = nn.Linear(dim, dim)
+        self.proj_drop = nn.Dropout(proj_drop)
+
+    def forward(self, x):
+        B, N, C = x.shape
+        qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4)
+        q, k, v = qkv[0], qkv[1], qkv[2]
+
+        attn = (q @ k.transpose(-2, -1)) * self.scale
+        attn = attn.softmax(dim=-1)
+        attn = self.attn_drop(attn)
+
+        x = (attn @ v).transpose(1, 2).reshape(B, N, C)
+        x = self.proj(x)
+        x = self.proj_drop(x)
+        return x, attn
+
+
+class Block(nn.Module):
+    def __init__(self, dim, num_heads, mlp_ratio=4., qkv_bias=False, qk_scale=None, drop=0., attn_drop=0.,
+                 drop_path=0., act_layer=nn.GELU, norm_layer=nn.LayerNorm):
+        super().__init__()
+        self.norm1 = norm_layer(dim)
+        self.attn = Attention(
+            dim, num_heads=num_heads, qkv_bias=qkv_bias, qk_scale=qk_scale, attn_drop=attn_drop, proj_drop=drop)
+        self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity()
+        self.norm2 = norm_layer(dim)
+        mlp_hidden_dim = int(dim * mlp_ratio)
+        self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop)
+
+    def forward(self, x, return_attention=False):
+        y, attn = self.attn(self.norm1(x))
+        if return_attention:
+            return attn
+        x = x + self.drop_path(y)
+        x = x + self.drop_path(self.mlp(self.norm2(x)))
+        return x
+
+
+class PatchEmbed(nn.Module):
+    """ Image to Patch Embedding
+    """
+    def __init__(self, img_size=224, patch_size=16, in_chans=3, embed_dim=768):
+        super().__init__()
+        num_patches = (img_size // patch_size) * (img_size // patch_size)
+        self.img_size = img_size
+        self.patch_size = patch_size
+        self.num_patches = num_patches
+
+        self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=patch_size)
+
+    def forward(self, x):
+        B, C, H, W = x.shape
+        x = self.proj(x).flatten(2).transpose(1, 2)
+        return x
+
+
+class VisionTransformer(nn.Module):
+    """ Vision Transformer """
+    def __init__(self, img_size=[224], patch_size=16, in_chans=3, num_classes=0, embed_dim=768, depth=12,
+                 num_heads=12, mlp_ratio=4., qkv_bias=False, qk_scale=None, drop_rate=0., attn_drop_rate=0.,
+                 drop_path_rate=0., norm_layer=nn.LayerNorm, global_pool='token',**kwargs):
+        super().__init__()
+        self.num_features = self.embed_dim = embed_dim
+
+        self.patch_embed = PatchEmbed(
+            img_size=img_size[0], patch_size=patch_size, in_chans=in_chans, embed_dim=embed_dim)
+        num_patches = self.patch_embed.num_patches
+
+        self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim))
+        self.pos_embed = nn.Parameter(torch.zeros(1, num_patches + 1, embed_dim))
+        self.pos_drop = nn.Dropout(p=drop_rate)
+
+        dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)]  # stochastic depth decay rule
+        self.blocks = nn.ModuleList([
+            Block(
+                dim=embed_dim, num_heads=num_heads, mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, qk_scale=qk_scale,
+                drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[i], norm_layer=norm_layer)
+            for i in range(depth)])
+        self.norm = norm_layer(embed_dim)
+
+        # Classifier head
+        self.head = nn.Linear(embed_dim, num_classes) if num_classes > 0 else nn.Identity()
+
+        trunc_normal_(self.pos_embed, std=.02)
+        trunc_normal_(self.cls_token, std=.02)
+        self.apply(self._init_weights)
+
+        self.global_pool = global_pool
+
+    def _init_weights(self, m):
+        if isinstance(m, nn.Linear):
+            trunc_normal_(m.weight, std=.02)
+            if isinstance(m, nn.Linear) and m.bias is not None:
+                nn.init.constant_(m.bias, 0)
+        elif isinstance(m, nn.LayerNorm):
+            nn.init.constant_(m.bias, 0)
+            nn.init.constant_(m.weight, 1.0)
+
+    def interpolate_pos_encoding(self, x, w, h):
+        npatch = x.shape[1] - 1
+        N = self.pos_embed.shape[1] - 1
+        if npatch == N and w == h:
+            return self.pos_embed
+        class_pos_embed = self.pos_embed[:, 0]
+        patch_pos_embed = self.pos_embed[:, 1:]
+        dim = x.shape[-1]
+        w0 = w // self.patch_embed.patch_size
+        h0 = h // self.patch_embed.patch_size
+        # we add a small number to avoid floating point error in the interpolation
+        # see discussion at https://github.com/facebookresearch/dino/issues/8
+        w0, h0 = w0 + 0.1, h0 + 0.1
+        patch_pos_embed = nn.functional.interpolate(
+            patch_pos_embed.reshape(1, int(math.sqrt(N)), int(math.sqrt(N)), dim).permute(0, 3, 1, 2),
+            scale_factor=(w0 / math.sqrt(N), h0 / math.sqrt(N)),
+            mode='bicubic',
+        )
+        assert int(w0) == patch_pos_embed.shape[-2] and int(h0) == patch_pos_embed.shape[-1]
+        patch_pos_embed = patch_pos_embed.permute(0, 2, 3, 1).view(1, -1, dim)
+        return torch.cat((class_pos_embed.unsqueeze(0), patch_pos_embed), dim=1)
+
+    def prepare_tokens(self, x):
+        B, nc, w, h = x.shape
+        x = self.patch_embed(x)  # patch linear embedding
+
+        # add the [CLS] token to the embed patch tokens
+        cls_tokens = self.cls_token.expand(B, -1, -1)
+        x = torch.cat((cls_tokens, x), dim=1)
+
+        # add positional encoding to each token
+        x = x + self.interpolate_pos_encoding(x, w, h)
+
+        return self.pos_drop(x)
+
+    def forward(self, x):
+        x = self.prepare_tokens(x)
+        for blk in self.blocks:
+            x = blk(x)
+        x = self.norm(x)
+        if self.global_pool == 'token':
+            return x[:, 0]
+        elif self.global_pool == '':
+            return x
+
+    def get_last_selfattention(self, x):
+        x = self.prepare_tokens(x)
+        for i, blk in enumerate(self.blocks):
+            if i < len(self.blocks) - 1:
+                x = blk(x)
+            else:
+                # return attention of the last block
+                return blk(x, return_attention=True)
+
+    def get_intermediate_layers(self, x, n=1):
+        x = self.prepare_tokens(x)
+        # we return the output tokens from the `n` last blocks
+        output = []
+        for i, blk in enumerate(self.blocks):
+            x = blk(x)
+            if len(self.blocks) - i <= n:
+                output.append(self.norm(x))
+        return output
+
+
+def vit_tiny(patch_size=16, **kwargs):
+    model = VisionTransformer(
+        patch_size=patch_size, embed_dim=192, depth=12, num_heads=3, mlp_ratio=4,
+        qkv_bias=True, norm_layer=partial(nn.LayerNorm, eps=1e-6), **kwargs)
+    return model
+
+
+def vit_small(patch_size=16, **kwargs):
+    model = VisionTransformer(
+        patch_size=patch_size, embed_dim=384, depth=12, num_heads=6, mlp_ratio=4,
+        qkv_bias=True, norm_layer=partial(nn.LayerNorm, eps=1e-6), **kwargs)
+    return model
+
+
+def vit_base(patch_size=16, **kwargs):
+    model = VisionTransformer(
+        patch_size=patch_size, embed_dim=768, depth=12, num_heads=12, mlp_ratio=4,
+        qkv_bias=True, norm_layer=partial(nn.LayerNorm, eps=1e-6), **kwargs)
+    return model
+
+
+class DINOHead(nn.Module):
+    def __init__(self, in_dim, out_dim, use_bn=False, norm_last_layer=True, nlayers=3, hidden_dim=2048, bottleneck_dim=256):
+        super().__init__()
+        nlayers = max(nlayers, 1)
+        if nlayers == 1:
+            self.mlp = nn.Linear(in_dim, bottleneck_dim)
+        else:
+            layers = [nn.Linear(in_dim, hidden_dim)]
+            if use_bn:
+                layers.append(nn.BatchNorm1d(hidden_dim))
+            layers.append(nn.GELU())
+            for _ in range(nlayers - 2):
+                layers.append(nn.Linear(hidden_dim, hidden_dim))
+                if use_bn:
+                    layers.append(nn.BatchNorm1d(hidden_dim))
+                layers.append(nn.GELU())
+            layers.append(nn.Linear(hidden_dim, bottleneck_dim))
+            self.mlp = nn.Sequential(*layers)
+        self.apply(self._init_weights)
+        self.last_layer = nn.utils.weight_norm(nn.Linear(bottleneck_dim, out_dim, bias=False))
+        self.last_layer.weight_g.data.fill_(1)
+        if norm_last_layer:
+            self.last_layer.weight_g.requires_grad = False
+
+    def _init_weights(self, m):
+        if isinstance(m, nn.Linear):
+            trunc_normal_(m.weight, std=.02)
+            if isinstance(m, nn.Linear) and m.bias is not None:
+                nn.init.constant_(m.bias, 0)
+
+    def forward(self, x):
+        x = self.mlp(x)
+        x = nn.functional.normalize(x, dim=-1, p=2)
+        x = self.last_layer(x)
+        return x
+
+
+def dino_vits16(pretrained=True, **kwargs):
+    """
+    ViT-Small/16x16 pre-trained with DINO.
+    Achieves 74.5% top-1 accuracy on ImageNet with k-NN classification.
+    """
+    model = vit_small(patch_size=16, num_classes=0, **kwargs)
+    if pretrained:
+        state_dict = torch.hub.load_state_dict_from_url(
+            url="https://dl.fbaipublicfiles.com/dino/dino_deitsmall16_pretrain/dino_deitsmall16_pretrain.pth",
+            map_location="cpu",
+        )
+        model.load_state_dict(state_dict, strict=True)
+    return model
+
+
+def dino_vits8(pretrained=True, **kwargs):
+    """
+    ViT-Small/8x8 pre-trained with DINO.
+    Achieves 78.3% top-1 accuracy on ImageNet with k-NN classification.
+    """
+    model = vit_small(patch_size=8, num_classes=0, **kwargs)
+    if pretrained:
+        state_dict = torch.hub.load_state_dict_from_url(
+            url="https://dl.fbaipublicfiles.com/dino/dino_deitsmall8_pretrain/dino_deitsmall8_pretrain.pth",
+            map_location="cpu",
+        )
+        model.load_state_dict(state_dict, strict=True)
+    return model
+
+
+def dino_vitb16(pretrained=True, **kwargs):
+    """
+    ViT-Base/16x16 pre-trained with DINO.
+    Achieves 76.1% top-1 accuracy on ImageNet with k-NN classification.
+    """
+    model = vit_base(patch_size=16, num_classes=0, **kwargs)
+    if pretrained:
+        state_dict = torch.hub.load_state_dict_from_url(
+            url="https://dl.fbaipublicfiles.com/dino/dino_vitbase16_pretrain/dino_vitbase16_pretrain.pth",
+            map_location="cpu",
+        )
+        model.load_state_dict(state_dict, strict=True)
+    return model
+
+
+def dino_vitb8(pretrained=True, **kwargs):
+    """
+    ViT-Base/8x8 pre-trained with DINO.
+    Achieves 77.4% top-1 accuracy on ImageNet with k-NN classification.
+    """
+    model = vit_base(patch_size=8, num_classes=0, **kwargs)
+    if pretrained:
+        state_dict = torch.hub.load_state_dict_from_url(
+            url="https://dl.fbaipublicfiles.com/dino/dino_vitbase8_pretrain/dino_vitbase8_pretrain.pth",
+            map_location="cpu",
+        )
+        model.load_state_dict(state_dict, strict=True)
+    return model
+
+def dino_vitb_cifar10(pretrained=True, **kwargs):
+    """
+    ViT-Base/16x16 pre-trained with DINO.
+    Achieves 76.1% top-1 accuracy on ImageNet with k-NN classification.
+    """
+    model = vit_base(patch_size=16, num_classes=0, **kwargs)
+    if pretrained:
+        state_dict = torch.hub.load_state_dict_from_url(
+            url="https://dl.fbaipublicfiles.com/dino/cifar100_ViT_B_dino.pth",
+            map_location="cpu",
+        )
+        model.load_state_dict(state_dict, strict=False)
+    return model
+
+
+
+
+def dino_resnet50(pretrained=True, **kwargs):
+    """
+    ResNet-50 pre-trained with DINO.
+    Achieves 75.3% top-1 accuracy on ImageNet linear evaluation benchmark (requires to train `fc`).
+    """
+    from torchvision.models.resnet import resnet50
+    
+    model = resnet50(pretrained=False, **kwargs)
+    model.fc = torch.nn.Identity()
+    if pretrained:
+        state_dict = torch.hub.load_state_dict_from_url(
+            url="https://dl.fbaipublicfiles.com/dino/dino_resnet50_pretrain/dino_resnet50_pretrain.pth",
+            map_location="cpu",
+        )
+        model.load_state_dict(state_dict, strict=False)
+    return model
+
+
+def dino_xcit_small_12_p16(pretrained=True, **kwargs):
+    """
+    XCiT-Small-12/16 pre-trained with DINO.
+    """
+    model = torch.hub.load('facebookresearch/xcit:main', "xcit_small_12_p16", num_classes=0, **kwargs)
+    if pretrained:
+        state_dict = torch.hub.load_state_dict_from_url(
+            url="https://dl.fbaipublicfiles.com/dino/dino_xcit_small_12_p16_pretrain/dino_xcit_small_12_p16_pretrain.pth",
+            map_location="cpu",
+        )
+        model.load_state_dict(state_dict, strict=True)
+    return model
+
+
+def dino_xcit_small_12_p8(pretrained=True, **kwargs):
+    """
+    XCiT-Small-12/8 pre-trained with DINO.
+    """
+    model = torch.hub.load('facebookresearch/xcit:main', "xcit_small_12_p8", num_classes=0, **kwargs)
+    if pretrained:
+        state_dict = torch.hub.load_state_dict_from_url(
+            url="https://dl.fbaipublicfiles.com/dino/dino_xcit_small_12_p8_pretrain/dino_xcit_small_12_p8_pretrain.pth",
+            map_location="cpu",
+        )
+        model.load_state_dict(state_dict, strict=True)
+    return model
+
+
+def dino_xcit_medium_24_p16(pretrained=True, **kwargs):
+    """
+    XCiT-Medium-24/16 pre-trained with DINO.
+    """
+    model = torch.hub.load('facebookresearch/xcit:main', "xcit_medium_24_p16", num_classes=0, **kwargs)
+    if pretrained:
+        state_dict = torch.hub.load_state_dict_from_url(
+            url="https://dl.fbaipublicfiles.com/dino/dino_xcit_medium_24_p16_pretrain/dino_xcit_medium_24_p16_pretrain.pth",
+            map_location="cpu",
+        )
+        model.load_state_dict(state_dict, strict=True)
+    return model
+
+
+def dino_xcit_medium_24_p8(pretrained=True, **kwargs):
+    """
+    XCiT-Medium-24/8 pre-trained with DINO.
+    """
+    model = torch.hub.load('facebookresearch/xcit:main', "xcit_medium_24_p8", num_classes=0, **kwargs)
+    if pretrained:
+        state_dict = torch.hub.load_state_dict_from_url(
+            url="https://dl.fbaipublicfiles.com/dino/dino_xcit_medium_24_p8_pretrain/dino_xcit_medium_24_p8_pretrain.pth",
+            map_location="cpu",
+        )
+        model.load_state_dict(state_dict, strict=True)
+    return model

CSD/models/moco_vits.py

@@ -0,0 +1,143 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+# All rights reserved.
+
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+import math
+import torch
+import torch.nn as nn
+from functools import partial, reduce
+from operator import mul
+
+from timm.models.vision_transformer import VisionTransformer, _cfg
+from timm.models.layers.helpers import to_2tuple
+from timm.models.layers import PatchEmbed
+
+__all__ = [
+    'vit_small',
+    'vit_base',
+    'vit_conv_small',
+    'vit_conv_base',
+]
+
+
+class VisionTransformerMoCo(VisionTransformer):
+    def __init__(self, stop_grad_conv1=False, **kwargs):
+        super().__init__(**kwargs)
+        # Use fixed 2D sin-cos position embedding
+        self.build_2d_sincos_position_embedding()
+
+        # weight initialization
+        for name, m in self.named_modules():
+            if isinstance(m, nn.Linear):
+                if 'qkv' in name:
+                    # treat the weights of Q, K, V separately
+                    val = math.sqrt(6. / float(m.weight.shape[0] // 3 + m.weight.shape[1]))
+                    nn.init.uniform_(m.weight, -val, val)
+                else:
+                    nn.init.xavier_uniform_(m.weight)
+                nn.init.zeros_(m.bias)
+        nn.init.normal_(self.cls_token, std=1e-6)
+
+        if isinstance(self.patch_embed, PatchEmbed):
+            # xavier_uniform initialization
+            val = math.sqrt(6. / float(3 * reduce(mul, self.patch_embed.patch_size, 1) + self.embed_dim))
+            nn.init.uniform_(self.patch_embed.proj.weight, -val, val)
+            nn.init.zeros_(self.patch_embed.proj.bias)
+
+            if stop_grad_conv1:
+                self.patch_embed.proj.weight.requires_grad = False
+                self.patch_embed.proj.bias.requires_grad = False
+
+    def build_2d_sincos_position_embedding(self, temperature=10000.):
+        h, w = self.patch_embed.grid_size
+        grid_w = torch.arange(w, dtype=torch.float32)
+        grid_h = torch.arange(h, dtype=torch.float32)
+        grid_w, grid_h = torch.meshgrid(grid_w, grid_h)
+        assert self.embed_dim % 4 == 0, 'Embed dimension must be divisible by 4 for 2D sin-cos position embedding'
+        pos_dim = self.embed_dim // 4
+        omega = torch.arange(pos_dim, dtype=torch.float32) / pos_dim
+        omega = 1. / (temperature**omega)
+        out_w = torch.einsum('m,d->md', [grid_w.flatten(), omega])
+        out_h = torch.einsum('m,d->md', [grid_h.flatten(), omega])
+        pos_emb = torch.cat([torch.sin(out_w), torch.cos(out_w), torch.sin(out_h), torch.cos(out_h)], dim=1)[None, :, :]
+
+        # assert self.num_tokens == 1, 'Assuming one and only one token, [cls]'
+        pe_token = torch.zeros([1, 1, self.embed_dim], dtype=torch.float32)
+        self.pos_embed = nn.Parameter(torch.cat([pe_token, pos_emb], dim=1))
+        self.pos_embed.requires_grad = False
+
+
+class ConvStem(nn.Module):
+    """
+    ConvStem, from Early Convolutions Help Transformers See Better, Tete et al. https://arxiv.org/abs/2106.14881
+    """
+    def __init__(self, img_size=224, patch_size=16, in_chans=3, embed_dim=768, norm_layer=None, flatten=True):
+        super().__init__()
+
+        assert patch_size == 16, 'ConvStem only supports patch size of 16'
+        assert embed_dim % 8 == 0, 'Embed dimension must be divisible by 8 for ConvStem'
+
+        img_size = to_2tuple(img_size)
+        patch_size = to_2tuple(patch_size)
+        self.img_size = img_size
+        self.patch_size = patch_size
+        self.grid_size = (img_size[0] // patch_size[0], img_size[1] // patch_size[1])
+        self.num_patches = self.grid_size[0] * self.grid_size[1]
+        self.flatten = flatten
+
+        # build stem, similar to the design in https://arxiv.org/abs/2106.14881
+        stem = []
+        input_dim, output_dim = 3, embed_dim // 8
+        for l in range(4):
+            stem.append(nn.Conv2d(input_dim, output_dim, kernel_size=3, stride=2, padding=1, bias=False))
+            stem.append(nn.BatchNorm2d(output_dim))
+            stem.append(nn.ReLU(inplace=True))
+            input_dim = output_dim
+            output_dim *= 2
+        stem.append(nn.Conv2d(input_dim, embed_dim, kernel_size=1))
+        self.proj = nn.Sequential(*stem)
+
+        self.norm = norm_layer(embed_dim) if norm_layer else nn.Identity()
+
+    def forward(self, x):
+        B, C, H, W = x.shape
+        assert H == self.img_size[0] and W == self.img_size[1], \
+            f"Input image size ({H}*{W}) doesn't match model ({self.img_size[0]}*{self.img_size[1]})."
+        x = self.proj(x)
+        if self.flatten:
+            x = x.flatten(2).transpose(1, 2)  # BCHW -> BNC
+        x = self.norm(x)
+        return x
+
+
+def vit_small(**kwargs):
+    model = VisionTransformerMoCo(
+        patch_size=16, embed_dim=384, depth=12, num_heads=12, mlp_ratio=4, qkv_bias=True,
+        norm_layer=partial(nn.LayerNorm, eps=1e-6), **kwargs)
+    model.default_cfg = _cfg()
+    return model
+
+def vit_base(**kwargs):
+    model = VisionTransformerMoCo(
+        patch_size=16, embed_dim=768, depth=12, num_heads=12, mlp_ratio=4, qkv_bias=True,
+        norm_layer=partial(nn.LayerNorm, eps=1e-6), **kwargs)
+    model.default_cfg = _cfg()
+    return model
+
+def vit_conv_small(**kwargs):
+    # minus one ViT block
+    model = VisionTransformerMoCo(
+        patch_size=16, embed_dim=384, depth=11, num_heads=12, mlp_ratio=4, qkv_bias=True,
+        norm_layer=partial(nn.LayerNorm, eps=1e-6), embed_layer=ConvStem, **kwargs)
+    model.default_cfg = _cfg()
+    return model
+
+def vit_conv_base(**kwargs):
+    # minus one ViT block
+    model = VisionTransformerMoCo(
+        patch_size=16, embed_dim=768, depth=11, num_heads=12, mlp_ratio=4, qkv_bias=True,
+        norm_layer=partial(nn.LayerNorm, eps=1e-6), embed_layer=ConvStem, **kwargs)
+    model.default_cfg = _cfg()
+    return model

CSD/search.py

@@ -0,0 +1,165 @@
+import argparse
+import copy
+import logging
+import logging.handlers as handlers
+import pathlib
+import sys
+
+import faiss
+import numpy as np
+import vaex as vx
+import wandb
+
+sys.path.insert(0, str(pathlib.Path(__file__).parent.resolve()))
+
+from search.embeddings import Embeddings
+from search.faiss_search import FaissIndex
+from metrics import metrics
+from data.wikiart import WikiArt
+
+logger = logging.getLogger()
+
+
+def get_parser():
+    parser = argparse.ArgumentParser('dynamicDistances-NN Search Module')
+    parser.add_argument('--dataset', default='wikiart', type=str, required=True)
+    parser.add_argument('--topk', nargs='+', type=int, default=[5],
+                        help='Number of NN to consider while calculating recall')
+    parser.add_argument('--mode', type=str, required=True, choices=['artist', 'label'],
+                        help='The type of matching to do')
+    parser.add_argument('--method', type=str, default='IP', choices=['IP', 'L2'], help='The method to do NN search')
+    parser.add_argument('--emb-dir', type=str, default=None,
+                        help='The directory where per image embeddings are stored (NOT USED when chunked)')
+    parser.add_argument('--query_count', default=-1, type=int,
+                        help='Number of queries to consider. Works only for domainnet')
+    parser.add_argument('--chunked', action='store_true', help='If I should read from chunked directory instead')
+    parser.add_argument('--query-chunk-dir', type=str, required=True,
+                        help='The directory where chunked query embeddings should be saved/are already saved')
+    parser.add_argument('--database-chunk-dir', type=str, required=True,
+                        help='The directory where chunked val embeddings should be saved/are already saved')
+    parser.add_argument('--data-dir', type=str, default=None,
+                        help='The directory of concerned dataset. (HARD CODED LATER)')
+    parser.add_argument('--multilabel', action='store_true', help='If the dataset is multilabel')
+
+    return parser
+
+
+def get_log_handlers(args):
+    # Create handlers
+    c_handler = logging.StreamHandler()
+    f_handler = handlers.RotatingFileHandler(f'search.log', maxBytes=int(1e6), backupCount=1000)
+    c_handler.setLevel(logging.DEBUG)
+    f_handler.setLevel(logging.DEBUG)
+
+    # Create formatters and add it to handlers
+    c_format = logging.Formatter('%(name)s - %(levelname)s - %(message)s')
+    f_format = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+    c_handler.setFormatter(c_format)
+    f_handler.setFormatter(f_format)
+    return c_handler, f_handler
+
+
+def main():
+    parser = get_parser()
+    args = parser.parse_args()
+
+    handlers = get_log_handlers(args)
+    logger.addHandler(handlers[0])
+    logger.addHandler(handlers[1])
+    logger.setLevel(logging.DEBUG)
+
+    if args.dataset == 'wikiart':
+        dataset = WikiArt(args.data_dir)
+    else:
+        raise NotImplementedError
+
+    query_embeddings = Embeddings(args.emb_dir, args.query_chunk_dir,
+                                  files=list(map(lambda x: f'{x.split(".")[0]}.npy', dataset.query_images)),
+                                  chunked=args.chunked,
+                                  file_ext='.npy')
+    val_embeddings = Embeddings(args.emb_dir, args.database_chunk_dir,
+                                files=list(map(lambda x: f'{x.split(".")[0]}.npy', dataset.val_images)),
+                                chunked=args.chunked,
+                                file_ext='.npy')
+
+    query_embeddings.filenames = list(query_embeddings.filenames)
+    val_embeddings.filenames = list(val_embeddings.filenames)
+
+    # Filtering the dataset based on the files which actually exist.
+    dataset.query_db = dataset.query_db[
+        dataset.query_db['name'].isin(query_embeddings.filenames)]
+    dataset.val_db = dataset.val_db[
+        dataset.val_db['name'].isin(val_embeddings.filenames)]
+
+    # Using only the embeddings corresponding to images in the datasets
+    temp = vx.from_arrays(filename=query_embeddings.filenames, index=np.arange(len(query_embeddings.filenames)))
+    dataset.query_db = dataset.query_db.join(temp, left_on='name', right_on='filename', how='left')
+    query_embeddings.embeddings = query_embeddings.embeddings[dataset.get_query_col('index')]
+    try:
+        b, h, w = query_embeddings.embeddings.shape
+        query_embeddings.embeddings = query_embeddings.embeddings.reshape(b, 1, h * w)
+    except ValueError:
+        b, d = query_embeddings.embeddings.shape
+        query_embeddings.embeddings = query_embeddings.embeddings.reshape(b, 1, d)
+    query_embeddings.filenames = np.asarray(query_embeddings.filenames)[dataset.get_query_col('index')]
+
+    temp = vx.from_arrays(filename=val_embeddings.filenames, index=np.arange(len(val_embeddings.filenames)))
+    dataset.val_db = dataset.val_db.join(temp, left_on='name', right_on='filename', how='left')
+    val_embeddings.embeddings = val_embeddings.embeddings[dataset.get_val_col('index')]
+    try:
+        b, h, w = val_embeddings.embeddings.shape
+        val_embeddings.embeddings = val_embeddings.embeddings.reshape(b, 1, h * w)
+    except ValueError:
+        b, d = val_embeddings.embeddings.shape
+        val_embeddings.embeddings = val_embeddings.embeddings.reshape(b, 1, d)
+    val_embeddings.filenames = np.asarray(val_embeddings.filenames)[dataset.get_val_col('index')]
+
+    # Building the faiss index
+    embedding_size = query_embeddings.embeddings[0].shape[1]
+    if args.method == 'IP':
+        method = faiss.IndexFlatIP
+    else:
+        method = faiss.IndexFlatL2
+    search_module = FaissIndex(embedding_size=embedding_size, index_func=method)
+    queries = np.asarray(query_embeddings.embeddings).reshape(len(query_embeddings.embeddings), embedding_size)
+    database = np.asarray(val_embeddings.embeddings).reshape(len(val_embeddings.embeddings), embedding_size)
+    search_module.build_index(database)
+
+    _, nns_all = search_module.search_nns(queries, max(args.topk))
+    if args.multilabel:
+        q_labels = dataset.query_db['multilabel'].values
+        db_labels = dataset.val_db['multilabel'].values
+        nns_all_pred = [q_labels[i] @ db_labels[nns_all[i]].T for i in range(len(nns_all))]
+        nns_all_pred = np.array(nns_all_pred)
+    else:
+        nns_all_pred = nns_all
+        classes = np.unique(dataset.get_val_col(args.mode))
+        mode_to_index = {classname: i for i, classname in enumerate(classes)}
+        try:
+            gts = np.asarray(list(map(lambda x: mode_to_index[x], dataset.get_query_col(args.mode).tolist())))
+        except KeyError:
+            logger.error('Class not found in database. This query list cannot be evaluated')
+            return
+
+    evals = metrics.Metrics()
+
+    for topk in args.topk:
+        logger.info(f'Calculating recall@{topk}')
+        nns_all_pred_topk = nns_all_pred[:, :topk]
+        if args.multilabel:
+            mode_recall = evals.get_recall_bin(copy.deepcopy(nns_all_pred_topk), topk)
+            mode_mrr = evals.get_mrr_bin(copy.deepcopy(nns_all_pred_topk), topk)
+            mode_map = evals.get_map_bin(copy.deepcopy(nns_all_pred_topk), topk)
+        else:
+            preds = dataset.get_val_col(args.mode)[nns_all_pred_topk.flatten()].reshape(len(queries), topk)
+            preds = np.vectorize(mode_to_index.get)(preds)
+            mode_recall = evals.get_recall(copy.deepcopy(preds), gts, topk)
+            mode_mrr = evals.get_mrr(copy.deepcopy(preds), gts, topk)
+            mode_map = evals.get_map(copy.deepcopy(preds), gts, topk)
+        logger.info(f'Recall@{topk}: {mode_recall}')
+        logger.info(f'MRR@{topk}: {mode_mrr}')
+        logger.info(f'mAP@{topk}: {mode_map}')
+
+
+if __name__ == '__main__':
+    main()

CSD/search/embeddings.py

@@ -0,0 +1,164 @@
+import concurrent.futures as concfut
+import glob
+import os
+
+import pickle
+import logging
+import queue
+import os.path as osp
+import threading
+from multiprocessing import Process
+import math
+import numpy as np
+
+module_logger = logging.getLogger(__name__)
+
+
+class Embeddings(object):
+    """Class to read embeddings from the disk and store them in memory"""
+    def __init__(self, data_dir, chunk_dir, file_ext='.pt', files=None, chunked=False, chunk_size=5000):
+        if files is not None:
+            self.embedding_files = list(map(lambda x: osp.join(data_dir, x), files))
+        else:
+            self.embedding_files = glob.glob(f'{data_dir}/*{file_ext}')
+        self.embedding_queue = queue.Queue()
+        self.embeddings = []
+        self.filenames = []
+        self.chunk_dir = chunk_dir
+        self.chunk_size = chunk_size
+        self.chunked = chunked
+        if not self.chunked:
+            threading.Thread(target=self.__result_consumer, daemon=True).start()
+            self.__read_embeddings()
+            self.embeddings, self.filenames = self.__remove_missing(self.embeddings, self.filenames)
+        else:
+            self.__read_embeddings_chunked()
+        self.__sort_embeddings()
+
+    def __result_consumer(self):
+        """Consumes the results from the embedding queue and saves them to the disk"""
+        processed = 0
+        fnf = 0  # FileNotFound
+        embedding_chunk = []
+        filename_chunk = []
+        chunk_cnt = 0
+        while True:
+            data = self.embedding_queue.get()
+            if not isinstance(data, str):
+                self.filenames.append(data['filename'])
+                if data['embedding'] is not None:
+                    self.embeddings.append(data['embedding'])
+                    processed += 1
+                    if processed % 1000 == 0:
+                        module_logger.info(f'Read {processed}/{len(self.embedding_files)} embeddings')
+                else:
+                    fnf += 1
+                    self.embeddings.append(None)
+                if len(embedding_chunk) < self.chunk_size:
+                    embedding_chunk.append(data['embedding'])
+                    filename_chunk.append(data['filename'])
+                else:
+                    chunk_cnt += 1
+                    embedding_chunk, filename_chunk = self.__remove_missing(embedding_chunk, filename_chunk)
+                    Process(target=save_chunk, args=(embedding_chunk, filename_chunk, chunk_cnt, self.chunk_dir),
+                            daemon=True).start()
+                    embedding_chunk = []
+                    filename_chunk = []
+                self.embedding_queue.task_done()
+            elif data == 'DONE':
+                chunk_cnt += 1
+                embedding_chunk, filename_chunk = self.__remove_missing(embedding_chunk, filename_chunk)
+                save_chunk(embedding_chunk, filename_chunk, chunk_cnt, self.chunk_dir)
+                module_logger.info(
+                    f'Completed reading embeddings. There were {fnf} images for which embeddings were not found')
+                self.embedding_queue.task_done()
+                break
+
+    def __sort_embeddings(self):
+        """Sort embeddings and filenames by filename"""
+        self.filenames = np.asarray(self.filenames)
+        sort_order = np.argsort(self.filenames)
+        self.embeddings = np.asarray(self.embeddings)[sort_order]
+        self.filenames = self.filenames[sort_order]
+
+    def __load_embedding(self, filename):
+        """Loads an embedding from the disk and puts it in the embedding queue"""
+        if osp.exists(filename):
+            embedding = np.load(filename)
+            data = {
+                'embedding': embedding,
+                'filename': filename.split('/')[-1],
+            }
+        else:
+            data = {
+                'filename': filename.split('/')[-1],
+                'embedding': None
+            }
+        self.embedding_queue.put(data)
+
+    def __read_embeddings(self):
+        """Reads embeddings from the disk"""
+        with concfut.ThreadPoolExecutor(max_workers=32) as executor:
+            worker = self.__load_embedding
+            executor.map(worker, self.embedding_files)
+            executor.shutdown(wait=True, cancel_futures=False)
+            self.embedding_queue.put('DONE')
+            self.embedding_queue.join()
+
+    def __read_embeddings_chunked(self):
+        """Reads embeddings from the disk in chunks"""
+        files = os.listdir(self.chunk_dir)
+        cnt = 0
+        with concfut.ProcessPoolExecutor(max_workers=32) as executor:
+            futures = [executor.submit(load_chunk, osp.join(self.chunk_dir, filename)) for filename in files]
+            for future in concfut.as_completed(futures):
+                result = future.result()
+                module_logger.info(f'Consuming {cnt}/{len(files)} chunks')
+                self.embeddings.extend(list(map(lambda x: x.squeeze(), result['embeddings'])))
+                self.filenames.extend(list(map(lambda x: '.'.join(x.split('/')[-1].split('.')[:-1]), result['filenames'])))
+                cnt += 1
+            module_logger.info('Finished reading chunks')
+
+    @staticmethod
+    def get_missing(x):
+        """Returns the indices of missing embeddings"""
+        indices = filter(lambda i_x: i_x[1] is None, enumerate(x))
+        res = np.asarray([i for i, x in indices])
+        return res
+
+    def __remove_missing(self, embeddings, filenames):
+        """Removes embeddings and filenames for which embeddings were not found"""
+        missing_ids = self.get_missing(embeddings)
+        embeddings = [ele for idx, ele in enumerate(embeddings) if idx not in missing_ids]
+        filenames = [ele for idx, ele in enumerate(filenames) if idx not in missing_ids]
+        return embeddings, filenames
+
+
+def load_chunk(filename):
+    """Loads a chunk file containing embeddings and filenames"""
+    data = pickle.load(open(filename, 'rb'))
+    return data
+
+
+def save_chunk(embeddings, filenames, count, chunk_dir, chunk_size=50000):
+    """Saves a chunk file containing embeddings and filenames. If the number of embeddings is less than chunk_size, it
+    saves all embeddings and filenames in one file. Otherwise, it splits the embeddings and filenames into chunks of
+    size chunk_size and saves each chunk in a separate file."""
+    assert len(embeddings) == len(filenames)
+    os.makedirs(chunk_dir, exist_ok=True)
+
+    if len(embeddings) < chunk_size:
+        data = {
+            'embeddings': embeddings,
+            'filenames': filenames,
+        }
+        pickle.dump(data, open(osp.join(chunk_dir, f'embeddings_{count}.pkl'), 'wb'))
+    else:
+        # Split into len(embeddings) / 50000 chunks
+        for i in range(0, math.ceil(len(embeddings)/chunk_size)):
+            data = {
+                'embeddings': embeddings[i*chunk_size: min((i+1)*chunk_size, len(embeddings))],
+                'filenames': filenames[i*chunk_size: min((i+1)*chunk_size, len(embeddings))],
+            }
+            with open(osp.join(chunk_dir, f'embeddings_{i}.pkl'), 'wb') as f:
+                pickle.dump(data, f)

CSD/search/faiss_search.py

@@ -0,0 +1,21 @@
+import logging
+import faiss
+
+module_logger = logging.getLogger(__name__)
+
+
+class FaissIndex(object):
+    def __init__(self, index_func=faiss.IndexFlatIP, embedding_size=512*512):
+        self.index = index_func(embedding_size)
+        # Enable GPU support
+        # self.index_gpu = faiss.index_cpu_to_all_gpus(self.index)
+
+    def build_index(self, nodes):
+        self.index.add(nodes)
+        # Enable GPU support
+        # self.index_gpu.add(nodes)
+
+    def search_nns(self, embeddings, n):
+        # Enable GPU support
+        # return self.index_gpu.search(embeddings, n)
+        return self.index.search(embeddings, n)

CSD/utils.py

@@ -0,0 +1,465 @@
+'''
+Code elements borrowed from 
+https://github.com/clovaai/CutMix-PyTorch/blob/master/train.py
+'''
+import argparse
+import os
+import sys
+from collections import defaultdict, deque
+import time, datetime
+
+import faiss
+import numpy as np
+import torch
+import torch.distributed as dist
+import torch.nn as nn
+
+from einops import rearrange, reduce
+
+
+def is_dist_avail_and_initialized():
+    if not dist.is_available():
+        return False
+    if not dist.is_initialized():
+        return False
+    return True
+
+
+def bool_flag(s):
+    """
+    Parse boolean arguments from the command line.
+    """
+    FALSY_STRINGS = {"off", "false", "0"}
+    TRUTHY_STRINGS = {"on", "true", "1"}
+    if s.lower() in FALSY_STRINGS:
+        return False
+    elif s.lower() in TRUTHY_STRINGS:
+        return True
+    else:
+        raise argparse.ArgumentTypeError("invalid value for a boolean flag")
+
+
+def setup_for_distributed(is_master):
+    """
+    This function disables printing when not in master process
+    """
+    import builtins as __builtin__
+    builtin_print = __builtin__.print
+
+    def print(*args, **kwargs):
+        force = kwargs.pop('force', False)
+        if is_master or force:
+            builtin_print(*args, **kwargs)
+
+    __builtin__.print = print
+
+
+def init_distributed_mode(args):
+    args.distributed = True
+    # launched with torch.distributed.launch
+    if 'RANK' in os.environ and 'WORLD_SIZE' in os.environ:
+        args.rank = int(os.environ["RANK"])
+        args.world_size = int(os.environ['WORLD_SIZE'])
+        args.gpu = int(os.environ['LOCAL_RANK'])
+    # launched with submitit on a slurm cluster
+    elif 'SLURM_PROCID' in os.environ:
+        args.rank = int(os.environ['SLURM_PROCID'])
+        args.gpu = args.rank % torch.cuda.device_count()
+    # launched naively with `python main_dino.py`
+    # we manually add MASTER_ADDR and MASTER_PORT to env variables
+    elif torch.cuda.is_available():
+        print('Will run the code on one GPU.')
+        args.rank, args.gpu, args.world_size = 0, 0, 1
+        os.environ['MASTER_ADDR'] = '127.0.0.1'
+        os.environ['MASTER_PORT'] = '29500'
+    else:
+        print('Does not support training without GPU.')
+        sys.exit(1)
+
+    dist.init_process_group(
+        backend="nccl",
+        init_method=args.dist_url,
+        world_size=args.world_size,
+        rank=args.rank,
+    )
+
+    torch.cuda.set_device(args.gpu)
+    print('| distributed init (rank {}): {}'.format(
+        args.rank, args.dist_url), flush=True)
+    dist.barrier()
+    setup_for_distributed(args.rank == 0)
+
+
+class SmoothedValue(object):
+    """Track a series of values and provide access to smoothed values over a
+    window or the global series average.
+    """
+
+    def __init__(self, window_size=20, fmt=None):
+        if fmt is None:
+            fmt = "{median:.6f} ({global_avg:.6f})"
+        self.deque = deque(maxlen=window_size)
+        self.total = 0.0
+        self.count = 0
+        self.fmt = fmt
+
+    def update(self, value, n=1):
+        self.deque.append(value)
+        self.count += n
+        self.total += value * n
+
+    def synchronize_between_processes(self):
+        """
+        Warning: does not synchronize the deque!
+        """
+        if not is_dist_avail_and_initialized():
+            return
+        t = torch.tensor([self.count, self.total], dtype=torch.float64, device='cuda')
+        dist.barrier()
+        dist.all_reduce(t)
+        t = t.tolist()
+        self.count = int(t[0])
+        self.total = t[1]
+
+    @property
+    def median(self):
+        d = torch.tensor(list(self.deque))
+        return d.median().item()
+
+    @property
+    def avg(self):
+        d = torch.tensor(list(self.deque), dtype=torch.float32)
+        return d.mean().item()
+
+    @property
+    def global_avg(self):
+        return self.total / self.count
+
+    @property
+    def max(self):
+        return max(self.deque)
+
+    @property
+    def value(self):
+        return self.deque[-1]
+
+    def __str__(self):
+        return self.fmt.format(
+            median=self.median,
+            avg=self.avg,
+            global_avg=self.global_avg,
+            max=self.max,
+            value=self.value)
+
+
+class MetricLogger(object):
+    def __init__(self, delimiter="\t"):
+        self.meters = defaultdict(SmoothedValue)
+        self.delimiter = delimiter
+
+    def update(self, **kwargs):
+        for k, v in kwargs.items():
+            if isinstance(v, torch.Tensor):
+                v = v.item()
+            assert isinstance(v, (float, int))
+            self.meters[k].update(v)
+
+    def __getattr__(self, attr):
+        if attr in self.meters:
+            return self.meters[attr]
+        if attr in self.__dict__:
+            return self.__dict__[attr]
+        raise AttributeError("'{}' object has no attribute '{}'".format(
+            type(self).__name__, attr))
+
+    def __str__(self):
+        loss_str = []
+        for name, meter in self.meters.items():
+            loss_str.append(
+                "{}: {}".format(name, str(meter))
+            )
+        return self.delimiter.join(loss_str)
+
+    def synchronize_between_processes(self):
+        for meter in self.meters.values():
+            meter.synchronize_between_processes()
+
+    def add_meter(self, name, meter):
+        self.meters[name] = meter
+
+    def log_every(self, iterable, print_freq, header=None):
+        i = 0
+        if not header:
+            header = ''
+        start_time = time.time()
+        end = time.time()
+        iter_time = SmoothedValue(fmt='{avg:.6f}')
+        data_time = SmoothedValue(fmt='{avg:.6f}')
+        space_fmt = ':' + str(len(str(len(iterable)))) + 'd'
+        if torch.cuda.is_available():
+            log_msg = self.delimiter.join([
+                header,
+                '[{0' + space_fmt + '}/{1}]',
+                'eta: {eta}',
+                '{meters}',
+                'time: {time}',
+                'data: {data}',
+                'max mem: {memory:.0f}'
+            ])
+        else:
+            log_msg = self.delimiter.join([
+                header,
+                '[{0' + space_fmt + '}/{1}]',
+                'eta: {eta}',
+                '{meters}',
+                'time: {time}',
+                'data: {data}'
+            ])
+        MB = 1024.0 * 1024.0
+        for obj in iterable:
+            data_time.update(time.time() - end)
+            yield obj
+            iter_time.update(time.time() - end)
+            if i % print_freq == 0 or i == len(iterable) - 1:
+                eta_seconds = iter_time.global_avg * (len(iterable) - i)
+                eta_string = str(datetime.timedelta(seconds=int(eta_seconds)))
+                if torch.cuda.is_available():
+                    print(log_msg.format(
+                        i, len(iterable), eta=eta_string,
+                        meters=str(self),
+                        time=str(iter_time), data=str(data_time),
+                        memory=torch.cuda.max_memory_allocated() / MB))
+                else:
+                    print(log_msg.format(
+                        i, len(iterable), eta=eta_string,
+                        meters=str(self),
+                        time=str(iter_time), data=str(data_time)))
+            i += 1
+            end = time.time()
+        total_time = time.time() - start_time
+        total_time_str = str(datetime.timedelta(seconds=int(total_time)))
+        print('{} Total time: {} ({:.6f} s / it)'.format(
+            header, total_time_str, total_time / len(iterable)))
+
+
+def multi_scale(samples, model, args):
+    v = None
+    for s in [1, 1 / 2 ** (1 / 2), 1 / 2]:  # we use 3 different scales
+        if s == 1:
+            inp = samples.clone()
+        else:
+            inp = torch.nn.functional.interpolate(samples, scale_factor=s, mode='bilinear', align_corners=False)
+
+        if args.pt_style == 'vicregl':
+            feats = model(inp)[-1].clone()
+        elif args.pt_style == 'clip':
+            feats = model.module.encode_image(samples).to(torch.float32).clone()
+        else:
+            feats = model(inp).clone()
+        feats = torch.squeeze(feats)
+        feats = torch.unsqueeze(feats, 0)
+        if v is None:
+            v = feats
+        else:
+            v += feats
+    v /= 3
+    v /= v.norm()
+    return v
+
+
+def patchify(x, size):
+    patches = rearrange(x, 'b c (h1 h2) (w1 w2) -> (b h1 w1) c h2 w2', h2=size, w2=size)
+    return patches
+
+
+@torch.no_grad()
+def extract_features(args, model, data_loader, use_cuda=True, multiscale=False):
+    metric_logger = MetricLogger(delimiter="  ")
+    features = None
+    # count = 0
+    for samples, index in metric_logger.log_every(data_loader, 100):
+        print(f'At the index {index[0]}')
+        samples = samples.cuda(non_blocking=True)
+        index = index.cuda(non_blocking=True)
+        if multiscale:
+            feats = multi_scale(samples, model, args)
+        else:
+
+            if args.pt_style == 'dino':
+                if args.layer > 1:
+                    feats = model.module.get_intermediate_layers(samples, args.layer)[0][:, 0, :].clone()
+                elif args.layer == -1:
+
+                    allfeats = model.module.get_intermediate_layers(samples, len(model.module.blocks))
+                    feats = [allfeats[i - 1][:, 0, :] for i in args.multilayer]
+                    bdim, _ = feats[0].shape
+                    feats = torch.stack(feats, dim=1).reshape((bdim, -1)).clone()
+                else:
+                    feats = model(samples).clone()
+
+            elif args.pt_style == 'moco':
+                feats = model.module.forward_features(samples)
+                feats = feats[:, 0, :].clone()
+            elif args.pt_style == 'vgg':
+                feats = model.module.features(samples).clone()
+            elif args.pt_style in ['clip', 'clip_wikiart']:
+                #
+                allfeats = model.module.visual.get_intermediate_layers(samples.type(model.module.dtype))
+                # else:
+                # allfeats = model.get_activations(samples) #[::-1]
+                allfeats.reverse()
+
+                if args.arch == 'resnet50':
+                    # import ipdb; ipdb.set_trace()
+                    if args.layer == -1:
+                        raise Exception('Layer=-1 not allowed with clip resnet')
+                    elif args.layer == 1:
+                        feats = allfeats[0].clone()
+                    else:
+                        assert len(allfeats) >= args.layer, "Asking for features of layer that doesnt exist"
+                        feats = reduce(allfeats[args.layer - 1], 'b c h w -> b c', 'mean').clone()
+
+                else:
+                    if args.layer == -1:
+                        feats = [allfeats[i - 1][:, 0, :] for i in args.multilayer]
+                        bdim, _ = feats[0].shape
+                        feats = torch.stack(feats, dim=1).reshape((bdim, -1)).clone()
+                    else:
+                        assert len(allfeats) >= args.layer
+                        feats = allfeats[args.layer - 1][:, 0, :].clone()
+            else:
+                feats = model(samples).clone()
+        # init storage feature matrix
+        feats = nn.functional.normalize(feats, dim=1, p=2).to(torch.float16)
+        if dist.get_rank() == 0 and features is None:
+            features = torch.zeros(len(data_loader.dataset), feats.shape[-1], dtype=feats.dtype)
+            if use_cuda:
+                features = features.cuda(non_blocking=True)
+            print(f"Storing features into tensor of shape {features.shape}")
+        # get indexes from all processes
+        y_all = torch.empty(dist.get_world_size(), index.size(0), dtype=index.dtype, device=index.device)
+        y_l = list(y_all.unbind(0))
+        y_all_reduce = torch.distributed.all_gather(y_l, index, async_op=True)
+        y_all_reduce.wait()
+        index_all = torch.cat(y_l)
+
+        # share features between processes
+        feats_all = torch.empty(
+            dist.get_world_size(),
+            feats.size(0),
+            feats.size(1),
+            dtype=feats.dtype,
+            device=feats.device,
+        )
+        output_l = list(feats_all.unbind(0))
+        output_all_reduce = torch.distributed.all_gather(output_l, feats, async_op=True)
+        output_all_reduce.wait()
+
+        # update storage feature matrix
+        if dist.get_rank() == 0:
+            if use_cuda:
+                features.index_copy_(0, index_all, torch.cat(output_l).cuda())
+            else:
+                features.index_copy_(0, index_all.cpu(), torch.cat(output_l).cpu())
+
+    return features
+
+
+def extract_features_pca(args, model, pca_model, k, data_loader, use_cuda=True, multiscale=False):
+    metric_logger = MetricLogger(delimiter="  ")
+    features = None
+    print('In pca function')
+    for samples, index in metric_logger.log_every(data_loader, 100):
+        print(f'At the index {index[0]}')
+        samples = samples.cuda(non_blocking=True)
+        index = index.cuda(non_blocking=True)
+
+        if multiscale:
+            feats = multi_scale(samples, model, args)
+        else:
+
+            if args.pt_style in ['clip', 'clip_wikiart']:
+                allfeats = model.module.visual.get_intermediate_layers(samples.type(model.module.dtype))
+                allfeats.reverse()
+                if args.arch == 'resnet50':
+                    raise Exception('code not written for this case')
+                else:
+                    temp = allfeats[args.layer - 1]
+                    temp = torch.nn.functional.normalize(temp, dim=2)
+                    # Doing gram matrix
+                    feats = torch.einsum('bij,bik->bjk', temp, temp)
+                    feats = feats.div(temp.shape[1])
+                    feats = rearrange(feats, 'b c d -> b (c d)')
+                    if pca_model is not None:
+                        feats = feats.cpu().detach().numpy()
+                        feats = pca_model.apply_py(feats)
+                        feats = torch.from_numpy(feats).cuda().clone()
+                    else:
+                        feats = feats.detach().clone()
+                    del temp
+                del allfeats
+            elif args.pt_style == 'vgg':
+                temp = model.module.features(samples)
+                temp = temp.view(temp.size(0), temp.size(1), -1)
+                feats = torch.einsum('bji,bki->bjk', temp, temp)
+                feats = feats.div(temp.shape[1])
+                feats = rearrange(feats, 'b c d -> b (c d)')
+                if pca_model is not None:
+                    feats = feats.cpu().detach().numpy()
+                    feats = pca_model.apply_py(feats)
+                    feats = torch.from_numpy(feats).cuda().clone()
+                else:
+                    feats = feats.detach().clone()
+                del temp
+            else:
+                raise Exception('Code not written for these ptstyles. Come back later.')
+
+        feats = nn.functional.normalize(feats, dim=1, p=2).to(torch.float16)
+        # init storage feature matrix
+        if dist.get_rank() == 0 and features is None:
+            features = torch.zeros(len(data_loader.dataset), feats.shape[-1], dtype=feats.dtype)
+            if use_cuda:
+                features = features.cuda(non_blocking=True)
+            print(f"Storing features into tensor of shape {features.shape}")
+
+        # get indexes from all processes
+        y_all = torch.empty(dist.get_world_size(), index.size(0), dtype=index.dtype, device=index.device)
+        y_l = list(y_all.unbind(0))
+        y_all_reduce = torch.distributed.all_gather(y_l, index, async_op=True)
+        y_all_reduce.wait()
+        index_all = torch.cat(y_l)
+
+        # share features between processes
+        feats_all = torch.empty(
+            dist.get_world_size(),
+            feats.size(0),
+            feats.size(1),
+            dtype=feats.dtype,
+            device=feats.device,
+        )
+        output_l = list(feats_all.unbind(0))
+        output_all_reduce = torch.distributed.all_gather(output_l, feats, async_op=True)
+        output_all_reduce.wait()
+
+        # update storage feature matrix
+        if dist.get_rank() == 0:
+            if use_cuda:
+                features.index_copy_(0, index_all, torch.cat(output_l))
+            else:
+                features.index_copy_(0, index_all.cpu(), torch.cat(output_l).cpu())
+    if pca_model is None:
+        features = features.detach().numpy()
+        pca = faiss.PCAMatrix(features.shape[-1], k)
+        pca.train(features)
+        trans_features = pca.apply_py(features)
+        return torch.from_numpy(trans_features), pca
+    else:
+        return features, None
+
+
+# saving features into numpy files
+def save_embeddings_numpy(embeddings, filenames, savepath):
+    os.makedirs(savepath, exist_ok=True)
+    for c, fname in enumerate(filenames):
+        np_emb = np.asarray(embeddings[c, :].cpu().detach(), dtype=np.float16)
+        np.save(f'{savepath}/{fname}.npy', np_emb)

README.md

@@ -1,3 +1,11 @@
 ---
 license: mit
 ---
+
+## Measuring Style Similarity in Diffusion Models
+
+Cloned from [learn2phoenix/CSD](https://github.com/learn2phoenix/CSD?tab=readme-ov-file).
+
+Their model (`csd-vit-l.pth`) downloaded from their [Google Drive](https://drive.google.com/file/d/1FX0xs8p-C7Ob-h5Y4cUhTeOepHzXv_46/view?usp=sharing).
+
+The original Git Repo is in the `CSD` folder.

csd-vit-l.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:40e92fad63a361b8136100cd234c42d401ef9b34ff1748234318929ebcc7e7a1
+size 2438228893