File size: 4,863 Bytes
a749dfd |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 |
try:
import bitsandbytes as bnb
from bitsandbytes.nn.modules import Params4bit, Int8Params
except ImportError:
pass
import torch
def Params4bitCuda(self, device):
self.data = self.data.cuda(device)
self.quant_state[0] = self.quant_state[0].cuda(device)
self.quant_state[4][0] = self.quant_state[4][0].cuda(device)
self.quant_state[4][1][0] = self.quant_state[4][1][0].cuda(device)
self.quant_state[4][1][1] = self.quant_state[4][1][1].cuda(device)
self.quant_state[6] = self.quant_state[6].cuda(device)
return self
class Linear4bitOnline(torch.nn.Module):
def __init__(self, weight, bias, quant_type):
super().__init__()
self.weight = Params4bit(
weight.data, requires_grad=False, compress_statistics=True, quant_type=quant_type
)
self.compute_dtype = None
#self.weight.cuda(weight.device)
self.bias = bias
def forward(self, x: torch.Tensor):
# weights are cast automatically as Int8Params, but the bias has to be cast manually
if self.bias is not None and self.bias.dtype != x.dtype:
self.bias.data = self.bias.data.to(x.dtype)
if getattr(self.weight, "quant_state", None) is None:
print(
"FP4 quantization state not initialized. Please call .cuda() or .to(device) on the LinearFP4 layer first."
)
inp_dtype = x.dtype
if self.compute_dtype is not None:
x = x.to(self.compute_dtype)
bias = None if self.bias is None else self.bias.to(self.compute_dtype)
out = bnb.matmul_4bit(
x, self.weight.t(), bias=bias, quant_state=self.weight.quant_state
)
out = out.to(inp_dtype)
return out
class Linear8bitLtOnline(torch.nn.Module):
def __init__(
self,
weight,
bias,
has_fp16_weights=True,
memory_efficient_backward=False,
threshold=0.0,
index=None,
):
super().__init__()
assert (
not memory_efficient_backward
), "memory_efficient_backward is no longer required and the argument is deprecated in 0.37.0 and will be removed in 0.39.0"
self.state = bnb.MatmulLtState()
self.index = index
# Necessary for stacked layers
self.state.threshold = threshold
self.state.has_fp16_weights = has_fp16_weights
self.state.memory_efficient_backward = memory_efficient_backward
if threshold > 0.0 and not has_fp16_weights:
self.state.use_pool = True
self.weight = Int8Params(
weight.data,
has_fp16_weights=has_fp16_weights,
requires_grad=has_fp16_weights,
)
self.bias = bias
def init_8bit_state(self):
self.state.CB = self.weight.CB
self.state.SCB = self.weight.SCB
self.weight.CB = None
self.weight.SCB = None
def forward(self, x: torch.Tensor):
self.state.is_training = self.training
if self.weight.CB is not None:
self.init_8bit_state()
# weights are cast automatically as Int8Params, but the bias has to be cast manually
if self.bias is not None and self.bias.dtype != x.dtype:
self.bias.data = self.bias.data.to(x.dtype)
out = bnb.matmul(x, self.weight, bias=self.bias, state=self.state)
if not self.state.has_fp16_weights:
if self.state.CB is not None and self.state.CxB is not None:
# we converted 8-bit row major to turing/ampere format in the first inference pass
# we no longer need the row-major weight
del self.state.CB
self.weight.data = self.state.CxB
return out
def quantize_online(model, bits: int):
def quant(weight, bias=None):
if bits == 8:
linear = Linear8bitLtOnline(
weight,
bias,
has_fp16_weights=False,
threshold=6.0,
)
if bias is not None:
linear.bias = torch.nn.Parameter(bias)
elif bits == 4:
linear = Linear4bitOnline(
weight,
bias,
quant_type="nf4", #fp4/nf4
)
else:
raise ValueError("quantize only support 4/8 bit")
return linear
def auto_quant(layer):
if hasattr(layer,"bias"):
linear = quant(layer.weight,bias=layer.bias)
else:
linear = quant(layer.weight)
return linear
for i,layer in enumerate(model.transformer.h):
layer.mlp.c_fc = auto_quant(layer.mlp.c_fc)
layer.mlp.c_proj = auto_quant(layer.mlp.c_proj)
layer.attn.c_attn=auto_quant(layer.attn.c_attn)
layer.attn.c_proj=auto_quant(layer.attn.c_proj)
return model |