Update patch.diff

patch.diff

@@ -1,21 +1,23 @@
 diff --git a/src/transformers/models/llama/convert_llama_weights_to_hf.py b/src/transformers/models/llama/convert_llama_weights_to_hf.py
-index a0fbe4680..50c7ed738 100644
+index a0fbe4680a..a6de9fe3ef 100644
 --- a/src/transformers/models/llama/convert_llama_weights_to_hf.py
 +++ b/src/transformers/models/llama/convert_llama_weights_to_hf.py
-@@ -17,10 +17,10 @@ import json
+@@ -17,10 +17,12 @@ import json
  import os
  import shutil
  import warnings
 -
++from turtledemo.sorting_animate import instructions1
 +from typing import List
  import torch
  
 -from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer, PreTrainedTokenizerFast
++from setup import install_requires
 +from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer, PreTrainedTokenizerFast, GenerationConfig
  from transformers.convert_slow_tokenizer import TikTokenConverter
  
  
-@@ -85,8 +85,12 @@ NUM_SHARDS = {
+@@ -85,8 +87,12 @@ NUM_SHARDS = {
      "65B": 8,
      "70B": 8,
      "70Bf": 8,
@@ -28,7 +30,7 @@ index a0fbe4680..50c7ed738 100644
  
  def compute_intermediate_size(n, ffn_dim_multiplier=1, multiple_of=256):
      return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3)) + multiple_of - 1) // multiple_of)
-@@ -107,9 +111,10 @@ def write_model(
+@@ -107,9 +113,10 @@ def write_model(
      input_base_path,
      model_size=None,
      safe_serialization=True,
@@ -40,7 +42,7 @@ index a0fbe4680..50c7ed738 100644
  ):
      os.makedirs(model_path, exist_ok=True)
      tmp_model_path = os.path.join(model_path, "tmp")
-@@ -125,18 +130,11 @@ def write_model(
+@@ -125,18 +132,11 @@ def write_model(
      dims_per_head = dim // n_heads
      base = params.get("rope_theta", 10000.0)
      inv_freq = 1.0 / (base ** (torch.arange(0, dims_per_head, 2).float() / dims_per_head))
@@ -62,7 +64,7 @@ index a0fbe4680..50c7ed738 100644
      if params.get("n_kv_heads", None) is not None:
          num_key_value_heads = params["n_kv_heads"]  # for GQA / MQA
          num_key_value_heads_per_shard = num_key_value_heads // num_shards
-@@ -144,8 +142,7 @@ def write_model(
+@@ -144,8 +144,7 @@ def write_model(
      else:  # compatibility with other checkpoints
          num_key_value_heads = n_heads
          num_key_value_heads_per_shard = n_heads_per_shard
@@ -72,7 +74,7 @@ index a0fbe4680..50c7ed738 100644
  
      # permute for sliced rotary
      def permute(w, n_heads, dim1=dim, dim2=dim):
-@@ -159,11 +156,9 @@ def write_model(
+@@ -159,11 +158,9 @@ def write_model(
          loaded = torch.load(os.path.join(input_base_path, "consolidated.00.pth"), map_location="cpu")
      else:
          # Sharded
@@ -87,7 +89,7 @@ index a0fbe4680..50c7ed738 100644
      param_count = 0
      index_dict = {"weight_map": {}}
      for layer_i in range(n_layers):
-@@ -263,7 +258,7 @@ def write_model(
+@@ -263,7 +260,7 @@ def write_model(
              "lm_head.weight": loaded["output.weight"],
          }
      else:
@@ -96,7 +98,7 @@ index a0fbe4680..50c7ed738 100644
          state_dict = {
              "model.norm.weight": loaded[0]["norm.weight"],
              "model.embed_tokens.weight": torch.cat(
-@@ -282,6 +277,18 @@ def write_model(
+@@ -282,6 +279,18 @@ def write_model(
      write_json(index_dict, os.path.join(tmp_model_path, "pytorch_model.bin.index.json"))
      ffn_dim_multiplier = params["ffn_dim_multiplier"] if "ffn_dim_multiplier" in params else 1
      multiple_of = params["multiple_of"] if "multiple_of" in params else 256
@@ -115,7 +117,7 @@ index a0fbe4680..50c7ed738 100644
      config = LlamaConfig(
          hidden_size=dim,
          intermediate_size=compute_intermediate_size(dim, ffn_dim_multiplier, multiple_of),
-@@ -292,11 +299,21 @@ def write_model(
+@@ -292,11 +301,21 @@ def write_model(
          vocab_size=vocab_size,
          rope_theta=base,
          max_position_embeddings=max_position_embeddings,
@@ -139,7 +141,7 @@ index a0fbe4680..50c7ed738 100644
      # Make space so we can load the model properly now.
      del state_dict
      del loaded
-@@ -313,7 +330,7 @@ def write_model(
+@@ -313,7 +332,7 @@ def write_model(
  
  
  class Llama3Converter(TikTokenConverter):
@@ -148,7 +150,7 @@ index a0fbe4680..50c7ed738 100644
          super().__init__(vocab_file, **kwargs)
          tokenizer = self.converted()
          chat_template = (
-@@ -327,34 +344,29 @@ class Llama3Converter(TikTokenConverter):
+@@ -327,34 +346,27 @@ class Llama3Converter(TikTokenConverter):
              "{% endfor %}"
              "{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}"
          )
@@ -167,14 +169,13 @@ index a0fbe4680..50c7ed738 100644
 -        ] + [f"<|reserved_special_token_{i}|>" for i in range(5, num_reserved_special_tokens - 5)]
          tokenizer.add_special_tokens(special_tokens)
  
-+        print("MODEL MAX LENGTH", model_max_length)
-+
          self.tokenizer = PreTrainedTokenizerFast(
              tokenizer_object=tokenizer,
              bos_token="<|begin_of_text|>",
 -            eos_token="<|end_of_text|>",
+-            chat_template=chat_template,
 +            eos_token="<|end_of_text|>" if not instruct else "<|eot_id|>",
-             chat_template=chat_template,
++            chat_template=chat_template if instruct else None,
              model_input_names=["input_ids", "attention_mask"],
 +            model_max_length=model_max_length,
          )
@@ -290,45 +291,3 @@ index a0fbe4680..50c7ed738 100644
          )
  
  
-diff --git a/src/transformers/models/llama/modeling_llama.py b/src/transformers/models/llama/modeling_llama.py
-index 8cbe8fe35..65b4bb56b 100644
---- a/src/transformers/models/llama/modeling_llama.py
-+++ b/src/transformers/models/llama/modeling_llama.py
-@@ -90,6 +90,29 @@ class LlamaRMSNorm(nn.Module):
- 
- ALL_LAYERNORM_LAYERS.append(LlamaRMSNorm)
- 
-+def apply_scaling(freqs: torch.Tensor):
-+    # Values obtained from grid search
-+    scale_factor = 8
-+    low_freq_factor = 1
-+    high_freq_factor = 4
-+    old_context_len = 8192  # original llama3 length
-+
-+    low_freq_wavelen = old_context_len / low_freq_factor
-+    high_freq_wavelen = old_context_len / high_freq_factor
-+    new_freqs = []
-+    for freq in freqs:
-+        wavelen = 2 * math.pi / freq
-+        if wavelen < high_freq_wavelen:
-+            new_freqs.append(freq)
-+        elif wavelen > low_freq_wavelen:
-+            new_freqs.append(freq / scale_factor)
-+        else:
-+            assert low_freq_wavelen != high_freq_wavelen
-+            smooth = (old_context_len / wavelen - low_freq_factor) / (
-+                high_freq_factor - low_freq_factor
-+            )
-+            new_freqs.append((1 - smooth) * freq / scale_factor + smooth * freq)
-+    return torch.tensor(new_freqs, dtype=freqs.dtype, device=freqs.device)
- 
- class LlamaRotaryEmbedding(nn.Module):
-     def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None, scaling_factor=1.0):
-@@ -99,6 +122,7 @@ class LlamaRotaryEmbedding(nn.Module):
-         self.max_position_embeddings = max_position_embeddings
-         self.base = base
-         inv_freq = 1.0 / (self.base ** (torch.arange(0, self.dim, 2, dtype=torch.int64).float().to(device) / self.dim))
-+        inv_freq = apply_scaling(inv_freq)
-         self.register_buffer("inv_freq", inv_freq, persistent=False)
-         # For BC we register cos and sin cached
-         self.max_seq_len_cached = max_position_embeddings