Spaces:
Sleeping
Sleeping
| """ Convolution with Weight Standardization (StdConv and ScaledStdConv) | |
| StdConv: | |
| @article{weightstandardization, | |
| author = {Siyuan Qiao and Huiyu Wang and Chenxi Liu and Wei Shen and Alan Yuille}, | |
| title = {Weight Standardization}, | |
| journal = {arXiv preprint arXiv:1903.10520}, | |
| year = {2019}, | |
| } | |
| Code: https://github.com/joe-siyuan-qiao/WeightStandardization | |
| ScaledStdConv: | |
| Paper: `Characterizing signal propagation to close the performance gap in unnormalized ResNets` | |
| - https://arxiv.org/abs/2101.08692 | |
| Official Deepmind JAX code: https://github.com/deepmind/deepmind-research/tree/master/nfnets | |
| Hacked together by / copyright Ross Wightman, 2021. | |
| """ | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| from .padding import get_padding, get_padding_value, pad_same | |
| class StdConv2d(nn.Conv2d): | |
| """Conv2d with Weight Standardization. Used for BiT ResNet-V2 models. | |
| Paper: `Micro-Batch Training with Batch-Channel Normalization and Weight Standardization` - | |
| https://arxiv.org/abs/1903.10520v2 | |
| """ | |
| def __init__( | |
| self, in_channel, out_channels, kernel_size, stride=1, padding=None, | |
| dilation=1, groups=1, bias=False, eps=1e-6): | |
| if padding is None: | |
| padding = get_padding(kernel_size, stride, dilation) | |
| super().__init__( | |
| in_channel, out_channels, kernel_size, stride=stride, | |
| padding=padding, dilation=dilation, groups=groups, bias=bias) | |
| self.eps = eps | |
| def forward(self, x): | |
| weight = F.batch_norm( | |
| self.weight.view(1, self.out_channels, -1), None, None, | |
| training=True, momentum=0., eps=self.eps).reshape_as(self.weight) | |
| x = F.conv2d(x, weight, self.bias, self.stride, self.padding, self.dilation, self.groups) | |
| return x | |
| class StdConv2dSame(nn.Conv2d): | |
| """Conv2d with Weight Standardization. TF compatible SAME padding. Used for ViT Hybrid model. | |
| Paper: `Micro-Batch Training with Batch-Channel Normalization and Weight Standardization` - | |
| https://arxiv.org/abs/1903.10520v2 | |
| """ | |
| def __init__( | |
| self, in_channel, out_channels, kernel_size, stride=1, padding='SAME', | |
| dilation=1, groups=1, bias=False, eps=1e-6): | |
| padding, is_dynamic = get_padding_value(padding, kernel_size, stride=stride, dilation=dilation) | |
| super().__init__( | |
| in_channel, out_channels, kernel_size, stride=stride, padding=padding, dilation=dilation, | |
| groups=groups, bias=bias) | |
| self.same_pad = is_dynamic | |
| self.eps = eps | |
| def forward(self, x): | |
| if self.same_pad: | |
| x = pad_same(x, self.kernel_size, self.stride, self.dilation) | |
| weight = F.batch_norm( | |
| self.weight.view(1, self.out_channels, -1), None, None, | |
| training=True, momentum=0., eps=self.eps).reshape_as(self.weight) | |
| x = F.conv2d(x, weight, self.bias, self.stride, self.padding, self.dilation, self.groups) | |
| return x | |
| class ScaledStdConv2d(nn.Conv2d): | |
| """Conv2d layer with Scaled Weight Standardization. | |
| Paper: `Characterizing signal propagation to close the performance gap in unnormalized ResNets` - | |
| https://arxiv.org/abs/2101.08692 | |
| NOTE: the operations used in this impl differ slightly from the DeepMind Haiku impl. The impact is minor. | |
| """ | |
| def __init__( | |
| self, in_channels, out_channels, kernel_size, stride=1, padding=None, | |
| dilation=1, groups=1, bias=True, gamma=1.0, eps=1e-6, gain_init=1.0): | |
| if padding is None: | |
| padding = get_padding(kernel_size, stride, dilation) | |
| super().__init__( | |
| in_channels, out_channels, kernel_size, stride=stride, padding=padding, dilation=dilation, | |
| groups=groups, bias=bias) | |
| self.gain = nn.Parameter(torch.full((self.out_channels, 1, 1, 1), gain_init)) | |
| self.scale = gamma * self.weight[0].numel() ** -0.5 # gamma * 1 / sqrt(fan-in) | |
| self.eps = eps | |
| def forward(self, x): | |
| weight = F.batch_norm( | |
| self.weight.view(1, self.out_channels, -1), None, None, | |
| weight=(self.gain * self.scale).view(-1), | |
| training=True, momentum=0., eps=self.eps).reshape_as(self.weight) | |
| return F.conv2d(x, weight, self.bias, self.stride, self.padding, self.dilation, self.groups) | |
| class ScaledStdConv2dSame(nn.Conv2d): | |
| """Conv2d layer with Scaled Weight Standardization and Tensorflow-like SAME padding support | |
| Paper: `Characterizing signal propagation to close the performance gap in unnormalized ResNets` - | |
| https://arxiv.org/abs/2101.08692 | |
| NOTE: the operations used in this impl differ slightly from the DeepMind Haiku impl. The impact is minor. | |
| """ | |
| def __init__( | |
| self, in_channels, out_channels, kernel_size, stride=1, padding='SAME', | |
| dilation=1, groups=1, bias=True, gamma=1.0, eps=1e-6, gain_init=1.0): | |
| padding, is_dynamic = get_padding_value(padding, kernel_size, stride=stride, dilation=dilation) | |
| super().__init__( | |
| in_channels, out_channels, kernel_size, stride=stride, padding=padding, dilation=dilation, | |
| groups=groups, bias=bias) | |
| self.gain = nn.Parameter(torch.full((self.out_channels, 1, 1, 1), gain_init)) | |
| self.scale = gamma * self.weight[0].numel() ** -0.5 | |
| self.same_pad = is_dynamic | |
| self.eps = eps | |
| def forward(self, x): | |
| if self.same_pad: | |
| x = pad_same(x, self.kernel_size, self.stride, self.dilation) | |
| weight = F.batch_norm( | |
| self.weight.view(1, self.out_channels, -1), None, None, | |
| weight=(self.gain * self.scale).view(-1), | |
| training=True, momentum=0., eps=self.eps).reshape_as(self.weight) | |
| return F.conv2d(x, weight, self.bias, self.stride, self.padding, self.dilation, self.groups) | |