Spaces:
Runtime error
Runtime error
| from functools import partial | |
| import numpy as np | |
| import torch | |
| from timm.models.efficientnet import tf_efficientnet_b4_ns, tf_efficientnet_b3_ns, \ | |
| tf_efficientnet_b5_ns, tf_efficientnet_b2_ns, tf_efficientnet_b6_ns, tf_efficientnet_b7_ns | |
| from torch import nn | |
| from torch.nn.modules.dropout import Dropout | |
| from torch.nn.modules.linear import Linear | |
| from torch.nn.modules.pooling import AdaptiveAvgPool2d | |
| encoder_params = { | |
| "tf_efficientnet_b3_ns": { | |
| "features": 1536, | |
| "init_op": partial(tf_efficientnet_b3_ns, pretrained=True, drop_path_rate=0.2) | |
| }, | |
| "tf_efficientnet_b2_ns": { | |
| "features": 1408, | |
| "init_op": partial(tf_efficientnet_b2_ns, pretrained=False, drop_path_rate=0.2) | |
| }, | |
| "tf_efficientnet_b4_ns": { | |
| "features": 1792, | |
| "init_op": partial(tf_efficientnet_b4_ns, pretrained=True, drop_path_rate=0.5) | |
| }, | |
| "tf_efficientnet_b5_ns": { | |
| "features": 2048, | |
| "init_op": partial(tf_efficientnet_b5_ns, pretrained=True, drop_path_rate=0.2) | |
| }, | |
| "tf_efficientnet_b4_ns_03d": { | |
| "features": 1792, | |
| "init_op": partial(tf_efficientnet_b4_ns, pretrained=True, drop_path_rate=0.3) | |
| }, | |
| "tf_efficientnet_b5_ns_03d": { | |
| "features": 2048, | |
| "init_op": partial(tf_efficientnet_b5_ns, pretrained=True, drop_path_rate=0.3) | |
| }, | |
| "tf_efficientnet_b5_ns_04d": { | |
| "features": 2048, | |
| "init_op": partial(tf_efficientnet_b5_ns, pretrained=True, drop_path_rate=0.4) | |
| }, | |
| "tf_efficientnet_b6_ns": { | |
| "features": 2304, | |
| "init_op": partial(tf_efficientnet_b6_ns, pretrained=True, drop_path_rate=0.2) | |
| }, | |
| "tf_efficientnet_b7_ns": { | |
| "features": 2560, | |
| "init_op": partial(tf_efficientnet_b7_ns, pretrained=True, drop_path_rate=0.2) | |
| }, | |
| "tf_efficientnet_b6_ns_04d": { | |
| "features": 2304, | |
| "init_op": partial(tf_efficientnet_b6_ns, pretrained=True, drop_path_rate=0.4) | |
| }, | |
| } | |
| def setup_srm_weights(input_channels: int = 3) -> torch.Tensor: | |
| """Creates the SRM kernels for noise analysis.""" | |
| # note: values taken from Zhou et al., "Learning Rich Features for Image Manipulation Detection", CVPR2018 | |
| srm_kernel = torch.from_numpy(np.array([ | |
| [ # srm 1/2 horiz | |
| [0., 0., 0., 0., 0.], # noqa: E241,E201 | |
| [0., 0., 0., 0., 0.], # noqa: E241,E201 | |
| [0., 1., -2., 1., 0.], # noqa: E241,E201 | |
| [0., 0., 0., 0., 0.], # noqa: E241,E201 | |
| [0., 0., 0., 0., 0.], # noqa: E241,E201 | |
| ], [ # srm 1/4 | |
| [0., 0., 0., 0., 0.], # noqa: E241,E201 | |
| [0., -1., 2., -1., 0.], # noqa: E241,E201 | |
| [0., 2., -4., 2., 0.], # noqa: E241,E201 | |
| [0., -1., 2., -1., 0.], # noqa: E241,E201 | |
| [0., 0., 0., 0., 0.], # noqa: E241,E201 | |
| ], [ # srm 1/12 | |
| [-1., 2., -2., 2., -1.], # noqa: E241,E201 | |
| [2., -6., 8., -6., 2.], # noqa: E241,E201 | |
| [-2., 8., -12., 8., -2.], # noqa: E241,E201 | |
| [2., -6., 8., -6., 2.], # noqa: E241,E201 | |
| [-1., 2., -2., 2., -1.], # noqa: E241,E201 | |
| ] | |
| ])).float() | |
| srm_kernel[0] /= 2 | |
| srm_kernel[1] /= 4 | |
| srm_kernel[2] /= 12 | |
| return srm_kernel.view(3, 1, 5, 5).repeat(1, input_channels, 1, 1) | |
| def setup_srm_layer(input_channels: int = 3) -> torch.nn.Module: | |
| """Creates a SRM convolution layer for noise analysis.""" | |
| weights = setup_srm_weights(input_channels) | |
| conv = torch.nn.Conv2d(input_channels, out_channels=3, kernel_size=5, stride=1, padding=2, bias=False) | |
| with torch.no_grad(): | |
| conv.weight = torch.nn.Parameter(weights, requires_grad=False) | |
| return conv | |
| class DeepFakeClassifierSRM(nn.Module): | |
| def __init__(self, encoder, dropout_rate=0.5) -> None: | |
| super().__init__() | |
| self.encoder = encoder_params[encoder]["init_op"]() | |
| self.avg_pool = AdaptiveAvgPool2d((1, 1)) | |
| self.srm_conv = setup_srm_layer(3) | |
| self.dropout = Dropout(dropout_rate) | |
| self.fc = Linear(encoder_params[encoder]["features"], 1) | |
| def forward(self, x): | |
| noise = self.srm_conv(x) | |
| x = self.encoder.forward_features(noise) | |
| x = self.avg_pool(x).flatten(1) | |
| x = self.dropout(x) | |
| x = self.fc(x) | |
| return x | |
| class GlobalWeightedAvgPool2d(nn.Module): | |
| """ | |
| Global Weighted Average Pooling from paper "Global Weighted Average | |
| Pooling Bridges Pixel-level Localization and Image-level Classification" | |
| """ | |
| def __init__(self, features: int, flatten=False): | |
| super().__init__() | |
| self.conv = nn.Conv2d(features, 1, kernel_size=1, bias=True) | |
| self.flatten = flatten | |
| def fscore(self, x): | |
| m = self.conv(x) | |
| m = m.sigmoid().exp() | |
| return m | |
| def norm(self, x: torch.Tensor): | |
| return x / x.sum(dim=[2, 3], keepdim=True) | |
| def forward(self, x): | |
| input_x = x | |
| x = self.fscore(x) | |
| x = self.norm(x) | |
| x = x * input_x | |
| x = x.sum(dim=[2, 3], keepdim=not self.flatten) | |
| return x | |
| class DeepFakeClassifier(nn.Module): | |
| def __init__(self, encoder, dropout_rate=0.0) -> None: | |
| super().__init__() | |
| self.encoder = encoder_params[encoder]["init_op"]() | |
| self.avg_pool = AdaptiveAvgPool2d((1, 1)) | |
| self.dropout = Dropout(dropout_rate) | |
| self.fc = Linear(encoder_params[encoder]["features"], 1) | |
| def forward(self, x): | |
| x = self.encoder.forward_features(x) | |
| x = self.avg_pool(x).flatten(1) | |
| x = self.dropout(x) | |
| x = self.fc(x) | |
| return x | |
| class DeepFakeClassifierGWAP(nn.Module): | |
| def __init__(self, encoder, dropout_rate=0.5) -> None: | |
| super().__init__() | |
| self.encoder = encoder_params[encoder]["init_op"]() | |
| self.avg_pool = GlobalWeightedAvgPool2d(encoder_params[encoder]["features"]) | |
| self.dropout = Dropout(dropout_rate) | |
| self.fc = Linear(encoder_params[encoder]["features"], 1) | |
| def forward(self, x): | |
| x = self.encoder.forward_features(x) | |
| x = self.avg_pool(x).flatten(1) | |
| x = self.dropout(x) | |
| x = self.fc(x) | |
| return x |