kaiokendev commited on
Commit
b71cff8
1 Parent(s): 198b278

Create llama_rope_scaled_monkey_patch.py

Browse files
Files changed (1) hide show
  1. llama_rope_scaled_monkey_patch.py +64 -0
llama_rope_scaled_monkey_patch.py ADDED
@@ -0,0 +1,64 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ import transformers
3
+ import transformers.models.llama.modeling_llama
4
+ from einops import rearrange
5
+ import random
6
+
7
+
8
+ class ScaledRotaryEmbedding(torch.nn.Module):
9
+ def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None):
10
+ super().__init__()
11
+ inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2).float().to(device) / dim))
12
+ self.register_buffer("inv_freq", inv_freq)
13
+
14
+ max_position_embeddings = 8192
15
+
16
+ # Build here to make `torch.jit.trace` work.
17
+ self.max_seq_len_cached = max_position_embeddings
18
+ t = torch.arange(
19
+ self.max_seq_len_cached,
20
+ device=self.inv_freq.device,
21
+ dtype=self.inv_freq.dtype,
22
+ )
23
+
24
+ self.scale = 1 / 4
25
+ t *= self.scale
26
+
27
+ freqs = torch.einsum("i,j->ij", t, self.inv_freq)
28
+ # Different from paper, but it uses a different permutation in order to obtain the same calculation
29
+ emb = torch.cat((freqs, freqs), dim=-1)
30
+ self.register_buffer(
31
+ "cos_cached", emb.cos()[None, None, :, :], persistent=False
32
+ )
33
+ self.register_buffer(
34
+ "sin_cached", emb.sin()[None, None, :, :], persistent=False
35
+ )
36
+
37
+ def forward(self, x, seq_len=None):
38
+ # x: [bs, num_attention_heads, seq_len, head_size]
39
+ # This `if` block is unlikely to be run after we build sin/cos in `__init__`. Keep the logic here just in case.
40
+ if seq_len > self.max_seq_len_cached:
41
+ self.max_seq_len_cached = seq_len
42
+ t = torch.arange(
43
+ self.max_seq_len_cached, device=x.device, dtype=self.inv_freq.dtype
44
+ )
45
+ t *= self.scale
46
+ freqs = torch.einsum("i,j->ij", t, self.inv_freq)
47
+ # Different from paper, but it uses a different permutation in order to obtain the same calculation
48
+ emb = torch.cat((freqs, freqs), dim=-1).to(x.device)
49
+ self.register_buffer(
50
+ "cos_cached", emb.cos()[None, None, :, :], persistent=False
51
+ )
52
+ self.register_buffer(
53
+ "sin_cached", emb.sin()[None, None, :, :], persistent=False
54
+ )
55
+ return (
56
+ self.cos_cached[:, :, :seq_len, ...].to(dtype=x.dtype),
57
+ self.sin_cached[:, :, :seq_len, ...].to(dtype=x.dtype),
58
+ )
59
+
60
+
61
+ def replace_llama_rope_with_scaled_rope():
62
+ transformers.models.llama.modeling_llama.LlamaRotaryEmbedding = (
63
+ ScaledRotaryEmbedding
64
+ )