Duplicate from jbochi/madlad400-8b-lm

Co-authored-by: J Bochi <jbochi@users.noreply.huggingface.co>

.gitattributes

@@ -0,0 +1,36 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,462 @@
+---
+license: apache-2.0
+language:
+- en
+- ru
+- es
+- fr
+- de
+- it
+- pt
+- pl
+- nl
+- vi
+- tr
+- sv
+- id
+- ro
+- cs
+- zh
+- hu
+- ja
+- th
+- fi
+- fa
+- uk
+- da
+- el
+- "no"
+- bg
+- sk
+- ko
+- ar
+- lt
+- ca
+- sl
+- he
+- et
+- lv
+- hi
+- sq
+- ms
+- az
+- sr
+- ta
+- hr
+- kk
+- is
+- ml
+- mr
+- te
+- af
+- gl
+- fil
+- be
+- mk
+- eu
+- bn
+- ka
+- mn
+- bs
+- uz
+- ur
+- sw
+- yue
+- ne
+- kn
+- kaa
+- gu
+- si
+- cy
+- eo
+- la
+- hy
+- ky
+- tg
+- ga
+- mt
+- my
+- km
+- tt
+- so
+- ku
+- ps
+- pa
+- rw
+- lo
+- ha
+- dv
+- fy
+- lb
+- ckb
+- mg
+- gd
+- am
+- ug
+- ht
+- grc
+- hmn
+- sd
+- jv
+- mi
+- tk
+- ceb
+- yi
+- ba
+- fo
+- or
+- xh
+- su
+- kl
+- ny
+- sm
+- sn
+- co
+- zu
+- ig
+- yo
+- pap
+- st
+- haw
+- as
+- oc
+- cv
+- lus
+- tet
+- gsw
+- sah
+- br
+- rm
+- sa
+- bo
+- om
+- se
+- ce
+- cnh
+- ilo
+- hil
+- udm
+- os
+- lg
+- ti
+- vec
+- ts
+- tyv
+- kbd
+- ee
+- iba
+- av
+- kha
+- to
+- tn
+- nso
+- fj
+- zza
+- ak
+- ada
+- otq
+- dz
+- bua
+- cfm
+- ln
+- chm
+- gn
+- krc
+- wa
+- hif
+- yua
+- srn
+- war
+- rom
+- bik
+- pam
+- sg
+- lu
+- ady
+- kbp
+- syr
+- ltg
+- myv
+- iso
+- kac
+- bho
+- ay
+- kum
+- qu
+- za
+- pag
+- ngu
+- ve
+- pck
+- zap
+- tyz
+- hui
+- bbc
+- tzo
+- tiv
+- ksd
+- gom
+- min
+- ang
+- nhe
+- bgp
+- nzi
+- nnb
+- nv
+- zxx
+- bci
+- kv
+- new
+- mps
+- alt
+- meu
+- bew
+- fon
+- iu
+- abt
+- mgh
+- mnw
+- tvl
+- dov
+- tlh
+- ho
+- kw
+- mrj
+- meo
+- crh
+- mbt
+- emp
+- ace
+- ium
+- mam
+- gym
+- mai
+- crs
+- pon
+- ubu
+- fip
+- quc
+- gv
+- kj
+- btx
+- ape
+- chk
+- rcf
+- shn
+- tzh
+- mdf
+- ppk
+- ss
+- gag
+- cab
+- kri
+- seh
+- ibb
+- tbz
+- bru
+- enq
+- ach
+- cuk
+- kmb
+- wo
+- kek
+- qub
+- tab
+- bts
+- kos
+- rwo
+- cak
+- tuc
+- bum
+- cjk
+- gil
+- stq
+- tsg
+- quh
+- mak
+- arn
+- ban
+- jiv
+- sja
+- yap
+- tcy
+- toj
+- twu
+- xal
+- amu
+- rmc
+- hus
+- nia
+- kjh
+- bm
+- guh
+- mas
+- acf
+- dtp
+- ksw
+- bzj
+- din
+- zne
+- mad
+- msi
+- mag
+- mkn
+- kg
+- lhu
+- ch
+- qvi
+- mh
+- djk
+- sus
+- mfe
+- srm
+- dyu
+- ctu
+- gui
+- pau
+- inb
+- bi
+- mni
+- guc
+- jam
+- wal
+- jac
+- bas
+- gor
+- skr
+- nyu
+- noa
+- sda
+- gub
+- nog
+- cni
+- teo
+- tdx
+- sxn
+- rki
+- nr
+- frp
+- alz
+- taj
+- lrc
+- cce
+- rn
+- jvn
+- hvn
+- nij
+- dwr
+- izz
+- msm
+- bus
+- ktu
+- chr
+- maz
+- tzj
+- suz
+- knj
+- bim
+- gvl
+- bqc
+- tca
+- pis
+- prk
+- laj
+- mel
+- qxr
+- niq
+- ahk
+- shp
+- hne
+- spp
+- koi
+- krj
+- quf
+- luz
+- agr
+- tsc
+- mqy
+- gof
+- gbm
+- miq
+- dje
+- awa
+- bjj
+- qvz
+- sjp
+- tll
+- raj
+- kjg
+- bgz
+- quy
+- cbk
+- akb
+- oj
+- ify
+- mey
+- ks
+- cac
+- brx
+- qup
+- syl
+- jax
+- ff
+- ber
+- tks
+- trp
+- mrw
+- adh
+- smt
+- srr
+- ffm
+- qvc
+- mtr
+- ann
+- kaa
+- aa
+- noe
+- nut
+- gyn
+- kwi
+- xmm
+- msb
+library_name: transformers
+tags:
+- text-generation-inference
+datasets:
+- allenai/MADLAD-400
+---
+
+This model has the safetensors weights for the [Madlad-400](https://github.com/google-research/google-research/tree/master/madlad_400) 8B param **language model**.
+
+The HF transformers code to run inference is not ready yet. The [original implementation](https://github.com/google/flaxformer/blob/ea17eb012a1d340ddff017b7a534c2162aaec34c/flaxformer/architectures/t5/t5_architecture.py#L1484) is in JAX/Flaxformer.
+
+The model architecture is the same as [Palm 8B](https://arxiv.org/pdf/2204.02311.pdf).
+
+It's a decoder-only T5 with 32 layers, 16 query heads, 1 KV head, and 4096 embedding size.
+
+These are the main differences relative to the original T5 architecture:
+
+- SwiGLU Activation
+- Parallel Layers
+- Multi-Query Attention
+- RoPE Embeddings
+- Shared Input-Output Embeddings
+- No biases
+- Bidirectional attention
+- Layer Norm with `center_scale_at_zero` and final layer with `use_scale=False`
+
+If you are looking for the language models models, here are the available versions:
+- [3B](https://huggingface.co/jbochi/madlad400-3b-mt)
+- [7B](https://huggingface.co/jbochi/madlad400-7b-mt)
+- [7B-BT](https://huggingface.co/jbochi/madlad400-7b-mt-bt)
+- [10B](https://huggingface.co/jbochi/madlad400-10b-mt)
+
+
+Article: [MADLAD-400: A Multilingual And Document-Level Large Audited Dataset](https://arxiv.org/abs/2309.04662)
+
+Abstract:
+
+> We introduce MADLAD-400, a manually audited, general domain 3T token monolingual dataset based on CommonCrawl, spanning 419 languages. We discuss the limitations revealed by self-auditing MADLAD-400, and the role data auditing had in the dataset creation process. We then train and release a 10.7B-parameter multilingual machine translation model on 250 billion tokens covering over 450 languages using publicly available data, and find that it is competitive with models that are significantly larger, and report the results on different domains. In addition, we train a 8B-parameter language model, and assess the results on few-shot translation. We make the baseline models available to the research community.
+
+
+

added_tokens.json

@@ -0,0 +1,2 @@
+{
+}

config.json

@@ -0,0 +1,39 @@
+{
+    "architectures": [
+      "DecoderOnlyT5Model"
+    ],
+    "auto_map": {
+      "AutoConfig": "decoderonlyt5_config.DecoderOnlyT5Config",
+      "AutoModelForCausalLM": "decoderonlyt5_modeling.DecoderOnlyT5Model"
+    },
+    "d_ff": 16384,
+    "d_kv": 256,
+    "d_model": 4096,
+    "dropout_rate": 0.0,
+    "decoder_start_token_id": 0,
+    "pad_token_id": 1,
+    "eos_token_id": 3,
+    "feed_forward_proj": "gated-swish",
+    "initializer_factor": 1.0,
+    "is_encoder_decoder": false,
+    "is_decoder_only": true,
+    "layer_norm_epsilon": 1e-06,
+    "model_type": "t5",
+    "n_positions": 512,
+    "num_layers": 0,
+    "num_decoder_layers": 32,
+    "num_heads": 16,
+    "output_past": true,
+    "relative_attention_max_distance": 128,
+    "relative_attention_num_buckets": 32,
+    "task_specific_params": {},
+    "tie_word_embeddings": true,
+    "transformers_version": "4.23.1",
+    "use_cache": true,
+    "vocab_size": 256512,
+    "parallel_layers": true,
+    "has_relative_attention_bias": false,
+    "multi_query_attention": true,
+    "use_rotary_embedding": true,
+    "rotary_embedding_max_timescale": 1000
+}

decoderonlyt5_config.py

@@ -0,0 +1,11 @@
+from transformers.models.t5.configuration_t5 import T5Config
+
+
+class DecoderOnlyT5Config(T5Config):
+    is_decoder_only = True
+    # whether to call attention and mlp in parallel.
+    # https://github.com/google/flaxformer/blob/ea17eb012a1d340ddff017b7a534c2162aaec34c/flaxformer/architectures/t5/t5_architecture.py#L384
+    parallel_layers = True
+    has_relative_attention_bias = False
+    # https://arxiv.org/abs/1911.02150
+    multi_query_attention = True

decoderonlyt5_modeling.py

@@ -0,0 +1,840 @@
+import copy
+import math
+from typing import Optional, Tuple, Union
+
+import torch
+from torch import nn
+from torch.nn import CrossEntropyLoss
+from transformers.models.t5 import modeling_t5
+from transformers.modeling_outputs import CausalLMOutputWithPast
+from transformers.utils import (
+    add_start_docstrings_to_model_forward,
+    logging,
+    replace_return_docstrings,
+)
+
+from .decoderonlyt5_config import DecoderOnlyT5Config
+
+
+logger = logging.get_logger(__name__)
+_CONFIG_FOR_DOC = "DecoderOnlyT5Config"
+
+
+class DecoderOnlyT5LayerNorm(nn.Module):
+    def __init__(self, hidden_size, eps=1e-6, use_scale=True, center_scale_at_zero=False):
+        """
+        Construct a layernorm module in the T5 style No bias and no subtraction of mean.
+        """
+        super().__init__()
+        if use_scale:
+            self.weight = nn.Parameter(torch.ones(hidden_size))
+        else:
+            assert not center_scale_at_zero
+            self.weight = None
+        self.center_scale_at_zero = center_scale_at_zero
+        self.variance_epsilon = eps
+
+    def forward(self, hidden_states):
+        # https://github.com/google/flaxformer/blob/ea17eb012a1d340ddff017b7a534c2162aaec34c/flaxformer/components/layer_norm.py#L30
+
+        # layer norm should always be calculated in float32
+        mean2 = hidden_states.to(torch.float32).pow(2).mean(-1, keepdim=True)
+        hidden_states = hidden_states * torch.rsqrt(mean2 + self.variance_epsilon)
+
+        # convert into float16 if necessary
+        if self.weight is None:
+            return hidden_states        
+        if self.weight.dtype == torch.float16:
+            hidden_states = hidden_states.to(torch.float16)
+        if self.center_scale_at_zero:
+            return (self.weight + 1.0) * hidden_states
+        else:
+            return self.weight * hidden_states
+            
+
+
+class DecoderOnlyT5LayerFF(modeling_t5.T5LayerFF):
+    def __init__(self, config: DecoderOnlyT5Config):
+        super(modeling_t5.T5LayerFF, self).__init__()
+        if config.is_gated_act:
+            self.DenseReluDense = modeling_t5.T5DenseGatedActDense(config)
+        else:
+            self.DenseReluDense = modeling_t5.T5DenseActDense(config)
+
+        if not config.parallel_layers:
+            self.layer_norm = modeling_t5.DecoderOnlyT5LayerNorm(
+                config.d_model, eps=config.layer_norm_epsilon
+            )
+        else:
+            self.layer_norm = nn.Identity()
+        self.dropout = nn.Dropout(config.dropout_rate)
+
+
+# LlamaRotaryEmbedding
+class DecoderOnlyT5RotaryEmbedding(nn.Module):
+    def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None):
+        super().__init__()
+
+        self.dim = dim
+        self.max_position_embeddings = max_position_embeddings
+        self.base = base
+        inv_freq = 1.0 / (
+            self.base ** (torch.arange(0, self.dim, 2).float().to(device) / self.dim)
+        )
+        self.register_buffer("inv_freq", inv_freq, persistent=False)
+
+        # Build here to make `torch.jit.trace` work.
+        self._set_cos_sin_cache(
+            seq_len=max_position_embeddings,
+            device=self.inv_freq.device,
+            dtype=torch.get_default_dtype(),
+        )
+
+    def _set_cos_sin_cache(self, seq_len, device, dtype):
+        self.max_seq_len_cached = seq_len
+        t = torch.arange(
+            self.max_seq_len_cached, device=device, dtype=self.inv_freq.dtype
+        )
+
+        freqs = torch.einsum("i,j->ij", t, self.inv_freq)
+        # Different from paper, but it uses a different permutation in order to obtain the same calculation
+        emb = torch.cat((freqs, freqs), dim=-1)
+        self.register_buffer("cos_cached", emb.cos().to(dtype), persistent=False)
+        self.register_buffer("sin_cached", emb.sin().to(dtype), persistent=False)
+
+    def forward(self, x, seq_len=None):
+        # x: [bs, num_attention_heads, seq_len, head_size]
+        if seq_len > self.max_seq_len_cached:
+            self._set_cos_sin_cache(seq_len=seq_len, device=x.device, dtype=x.dtype)
+
+        return (
+            self.cos_cached[:seq_len].to(dtype=x.dtype),
+            self.sin_cached[:seq_len].to(dtype=x.dtype),
+        )
+
+
+def rotate_half(x):
+    """Rotates half the hidden dims of the input."""
+    x1 = x[..., : x.shape[-1] // 2]
+    x2 = x[..., x.shape[-1] // 2 :]
+    return torch.cat((-x2, x1), dim=-1)
+
+
+def apply_rotary_pos_emb(q, k, cos, sin, position_ids, unsqueeze_dim=1):
+    """Applies Rotary Position Embedding to the query and key tensors.
+
+    Args:
+        q (`torch.Tensor`): The query tensor.
+        k (`torch.Tensor`): The key tensor.
+        cos (`torch.Tensor`): The cosine part of the rotary embedding.
+        sin (`torch.Tensor`): The sine part of the rotary embedding.
+        position_ids (`torch.Tensor`):
+            The position indices of the tokens corresponding to the query and key tensors. For example, this can be
+            used to pass offsetted position ids when working with a KV-cache.
+        unsqueeze_dim (`int`, *optional*, defaults to 1):
+            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
+            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
+            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
+            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
+            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
+            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
+    Returns:
+        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
+    """
+    cos = cos[position_ids].unsqueeze(unsqueeze_dim)
+    sin = sin[position_ids].unsqueeze(unsqueeze_dim)
+    q_embed = (q * cos) + (rotate_half(q) * sin)
+    k_embed = (k * cos) + (rotate_half(k) * sin)
+    return q_embed, k_embed
+
+
+# https://github.com/huggingface/transformers/blob/7ee995fd9c692761c4601ddbffa2ac2ec9f27b0b/src/transformers/models/llama/modeling_llama.py#L263
+def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
+    """
+    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
+    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
+    """
+    batch, num_key_value_heads, slen, head_dim = hidden_states.shape
+    if n_rep == 1:
+        return hidden_states
+    hidden_states = hidden_states[:, :, None, :, :].expand(
+        batch, num_key_value_heads, n_rep, slen, head_dim
+    )
+    return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim)
+
+
+class DecoderOnlyT5Attention(modeling_t5.T5Attention):
+    """
+    Supports both multi-head and multi-query attention.
+    https://arxiv.org/abs/1911.02150
+    https://github.com/google/flaxformer/blob/ea17eb012a1d340ddff017b7a534c2162aaec34c/flaxformer/components/attention/dense_attention.py#L292
+    """
+
+    def __init__(self, config: DecoderOnlyT5Config, has_relative_attention_bias=False):
+        super(modeling_t5.T5Attention, self).__init__()
+        self.is_decoder = config.is_decoder
+        assert not has_relative_attention_bias
+        assert config.use_rotary_embedding
+        self.d_model = config.d_model
+        self.head_dim = config.d_kv
+        self.num_heads = config.num_heads
+        self.num_key_value_heads = 1 if config.multi_query_attention else self.n_heads
+        self.num_key_value_groups = self.num_heads // self.num_key_value_heads
+        self.attention_dropout = config.dropout_rate
+        self.inner_dim = self.num_heads * self.head_dim
+        self.kv_inner_dim = self.num_key_value_heads * self.head_dim    
+        self.rotary_emb = DecoderOnlyT5RotaryEmbedding(
+            self.head_dim,
+            max_position_embeddings=config.relative_attention_max_distance,
+            base=config.rotary_embedding_max_timescale,
+        )
+
+        # Mesh TensorFlow initialization to avoid scaling before softmax
+        self.q = nn.Linear(self.d_model, self.inner_dim, bias=False)
+        self.k = nn.Linear(self.d_model, self.kv_inner_dim, bias=False)
+        self.v = nn.Linear(self.d_model, self.kv_inner_dim, bias=False)
+        self.o = nn.Linear(self.inner_dim, self.d_model, bias=False)
+
+        self.pruned_heads = set()
+        self.gradient_checkpointing = False
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        key_value_states=None,
+        position_bias=None,
+        mask: Optional[torch.Tensor] = None,
+        layer_head_mask=None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_value: Optional[Tuple[torch.Tensor]] = None,
+        output_attentions: bool = False,
+        use_cache: bool = False,
+    ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
+        assert key_value_states is None
+        assert position_bias is None
+        assert layer_head_mask is None
+
+        bsz, q_len, _ = hidden_states.size()
+
+        query_states = self.q(hidden_states)
+        key_states = self.k(hidden_states)
+        value_states = self.v(hidden_states)
+
+        query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
+        key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
+        value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
+
+        kv_seq_len = key_states.shape[-2]
+        if past_key_value is not None:
+            kv_seq_len += past_key_value[0].shape[-2]
+        cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len)
+        query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids)
+
+        if past_key_value is not None:
+            # reuse k, v, self_attention
+            key_states = torch.cat([past_key_value[0], key_states], dim=2)
+            value_states = torch.cat([past_key_value[1], value_states], dim=2)
+
+        past_key_value = (key_states, value_states) if use_cache else None
+
+        key_states = repeat_kv(key_states, self.num_key_value_groups)
+        value_states = repeat_kv(value_states, self.num_key_value_groups)
+
+        attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim)
+
+        if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len):
+            raise ValueError(
+                f"Attention weights should be of size {(bsz, self.num_heads, q_len, kv_seq_len)}, but is"
+                f" {attn_weights.size()}"
+            )
+
+        if mask is not None:
+            if mask.size() != (bsz, 1, q_len, kv_seq_len):
+                raise ValueError(
+                    f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {mask.size()}"
+                )
+            attn_weights = attn_weights + mask
+
+        # upcast attention to fp32
+        attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype)
+        attn_weights = nn.functional.dropout(attn_weights, p=self.attention_dropout)
+        attn_output = torch.matmul(attn_weights, value_states)
+
+        if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim):
+            raise ValueError(
+                f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is"
+                f" {attn_output.size()}"
+            )
+
+        attn_output = attn_output.transpose(1, 2).contiguous()
+        attn_output = attn_output.reshape(bsz, q_len, self.inner_dim)
+        attn_output = self.o(attn_output)
+
+        present_key_value_state = (
+            (key_states, value_states) if (self.is_decoder and use_cache) else None
+        )
+        outputs = (attn_output,) + (present_key_value_state,) + (position_bias,)
+
+        if output_attentions:
+            outputs = outputs + (attn_weights,)
+        return outputs
+
+
+class DecoderOnlyT5LayerSelfAttention(modeling_t5.T5LayerSelfAttention):
+    def __init__(self, config, has_relative_attention_bias=False):
+        super(modeling_t5.T5LayerSelfAttention, self).__init__()
+        self.SelfAttention = DecoderOnlyT5Attention(
+            config, has_relative_attention_bias=has_relative_attention_bias
+        )
+        self.layer_norm = DecoderOnlyT5LayerNorm(
+            config.d_model,
+            eps=config.layer_norm_epsilon,
+            use_scale=True,
+            center_scale_at_zero=True,
+        )
+        self.dropout = nn.Dropout(config.dropout_rate)
+        self.parallel_layers = config.parallel_layers
+
+    def forward(
+        self,
+        hidden_states,
+        attention_mask=None,
+        position_bias=None,
+        position_ids=None,
+        layer_head_mask=None,
+        past_key_value=None,
+        use_cache=False,
+        output_attentions=False,
+    ):
+        if not self.parallel_layers:
+            x = self.layer_norm(hidden_states)
+        else:
+            x = hidden_states
+        attention_output = self.SelfAttention(
+            x,
+            mask=attention_mask,
+            position_bias=position_bias,
+            position_ids=position_ids,
+            layer_head_mask=layer_head_mask,
+            past_key_value=past_key_value,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+        )
+        if not self.parallel_layers:
+            # When parallel_layers is True, the residual connection is applied
+            # in the decoder block instead of here.
+            hidden_states = hidden_states + self.dropout(attention_output[0])
+        else:
+            hidden_states = attention_output[0]
+        outputs = (hidden_states,) + attention_output[
+            1:
+        ]  # add attentions if we output them
+        return outputs
+
+
+class DecoderOnlyT5Block(modeling_t5.T5Block):
+    def __init__(self, config, has_relative_attention_bias=False):
+        super(modeling_t5.T5Block, self).__init__()
+        self.is_decoder = config.is_decoder
+        self.is_decoder_only = config.is_decoder_only
+        self.layer = nn.ModuleList()
+        self.layer.append(
+            DecoderOnlyT5LayerSelfAttention(
+                config, has_relative_attention_bias=has_relative_attention_bias
+            )
+        )
+        if self.is_decoder:
+            if config.is_decoder_only:
+                self.layer.append(nn.Identity())
+            else:
+                self.layer.append(modeling_t5.T5LayerCrossAttention(config))
+        self.parallel_layers = config.parallel_layers
+        self.layer.append(DecoderOnlyT5LayerFF(config))
+
+    def forward(
+        self,
+        hidden_states,
+        attention_mask=None,
+        position_bias=None,
+        position_ids=None,
+        encoder_hidden_states=None,
+        layer_head_mask=None,
+        past_key_value=None,
+        use_cache=False,
+        output_attentions=False,
+        encoder_attention_mask=None,
+        encoder_decoder_position_bias=None,
+        cross_attn_layer_head_mask=None,
+        return_dict=True,
+    ):
+        assert encoder_attention_mask is None
+        assert encoder_decoder_position_bias is None
+        assert cross_attn_layer_head_mask is None
+        if past_key_value is not None:
+            expected_num_past_key_values = 2 if encoder_hidden_states is None else 4
+
+            if len(past_key_value) != expected_num_past_key_values:
+                raise ValueError(
+                    f"There should be {expected_num_past_key_values} past states. "
+                    f"{'2 (past / key) for cross attention. ' if expected_num_past_key_values == 4 else ''}"
+                    f"Got {len(past_key_value)} past key / value states"
+                )
+            self_attn_past_key_value = past_key_value[:2]
+        else:
+            self_attn_past_key_value = None
+
+        ff_layer = self.layer[-1]
+        if self.parallel_layers:
+            # https://github.com/google/flaxformer/blob/ea17eb012a1d340ddff017b7a534c2162aaec34c/flaxformer/architectures/t5/t5_architecture.py#L563-L568
+            x = self.layer[0].layer_norm(hidden_states)
+            ff_output = ff_layer(x)
+        else:
+            x = hidden_states
+
+        self_attention_outputs = self.layer[0](
+            x,
+            attention_mask=attention_mask,
+            position_bias=position_bias,
+            position_ids=position_ids,
+            layer_head_mask=layer_head_mask,
+            past_key_value=self_attn_past_key_value,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+        )
+        x, present_key_value_state = self_attention_outputs[:2]
+        attention_outputs = self_attention_outputs[
+            2:
+        ]  # Keep self-attention outputs and relative position weights
+
+        # clamp inf values to enable fp16 training
+        if x.dtype == torch.float16:
+            clamp_value = torch.where(
+                torch.isinf(x).any(),
+                torch.finfo(x.dtype).max - 1000,
+                torch.finfo(x.dtype).max,
+            )
+            x = torch.clamp(x, min=-clamp_value, max=clamp_value)
+
+        do_cross_attention = (
+            self.is_decoder
+            and not self.is_decoder_only
+            and encoder_hidden_states is not None
+        )
+        assert not do_cross_attention
+
+        if self.parallel_layers:
+            # https://github.com/google/flaxformer/blob/ea17eb012a1d340ddff017b7a534c2162aaec34c/flaxformer/architectures/t5/t5_architecture.py#L534-L578
+            x = x + ff_output
+            x *= 2**-0.5
+            hidden_states = hidden_states + self.layer[0].dropout(x)
+        else:
+            hidden_states = ff_layer(x)
+
+        # clamp inf values to enable fp16 training
+        if hidden_states.dtype == torch.float16:
+            clamp_value = torch.where(
+                torch.isinf(hidden_states).any(),
+                torch.finfo(hidden_states.dtype).max - 1000,
+                torch.finfo(hidden_states.dtype).max,
+            )
+            hidden_states = torch.clamp(
+                hidden_states, min=-clamp_value, max=clamp_value
+            )
+
+        outputs = (hidden_states,)
+
+        if use_cache:
+            outputs = outputs + (present_key_value_state,) + attention_outputs
+        else:
+            outputs = outputs + attention_outputs
+
+        return outputs  # hidden-states, present_key_value_states, (self-attention position bias), (self-attention weights), (cross-attention position bias), (cross-attention weights)
+
+
+class DecoderOnlyT5Stack(modeling_t5.T5Stack):
+    def __init__(self, config, embed_tokens=None):
+        super(modeling_t5.T5Stack, self).__init__(config)
+
+        self.embed_tokens = embed_tokens
+        self.is_decoder = config.is_decoder
+
+        self.block = nn.ModuleList(
+            [
+                DecoderOnlyT5Block(
+                    config,
+                    has_relative_attention_bias=(
+                        config.has_relative_attention_bias and bool(i == 0)
+                    ),
+                )
+                for i in range(config.num_layers)
+            ]
+        )
+        self.final_layer_norm = DecoderOnlyT5LayerNorm(
+            config.d_model,
+            eps=config.layer_norm_epsilon,
+            use_scale=False,
+            center_scale_at_zero=False,
+        )
+        self.dropout = nn.Dropout(config.dropout_rate)
+
+        # Initialize weights and apply final processing
+        self.post_init()
+        # Model parallel
+        self.model_parallel = False
+        self.device_map = None
+        self.gradient_checkpointing = False
+
+    def forward(
+        self,
+        input_ids=None,
+        position_ids=None,
+        attention_mask=None,
+        encoder_hidden_states=None,
+        encoder_attention_mask=None,
+        inputs_embeds=None,
+        head_mask=None,
+        cross_attn_head_mask=None,
+        past_key_values=None,
+        use_cache=None,
+        output_attentions=None,
+        output_hidden_states=None,
+        return_dict=None,
+    ):
+        # Model parallel
+        if self.model_parallel:
+            torch.cuda.set_device(self.first_device)
+            self.embed_tokens = self.embed_tokens.to(self.first_device)
+        use_cache = use_cache if use_cache is not None else self.config.use_cache
+        output_attentions = (
+            output_attentions
+            if output_attentions is not None
+            else self.config.output_attentions
+        )
+        output_hidden_states = (
+            output_hidden_states
+            if output_hidden_states is not None
+            else self.config.output_hidden_states
+        )
+        return_dict = (
+            return_dict if return_dict is not None else self.config.use_return_dict
+        )
+
+        if input_ids is not None and inputs_embeds is not None:
+            err_msg_prefix = "decoder_" if self.is_decoder else ""
+            raise ValueError(
+                f"You cannot specify both {err_msg_prefix}input_ids and {err_msg_prefix}inputs_embeds at the same time"
+            )
+        elif input_ids is not None:
+            input_shape = input_ids.size()
+            input_ids = input_ids.view(-1, input_shape[-1])
+        elif inputs_embeds is not None:
+            input_shape = inputs_embeds.size()[:-1]
+        else:
+            err_msg_prefix = "decoder_" if self.is_decoder else ""
+            raise ValueError(
+                f"You have to specify either {err_msg_prefix}input_ids or {err_msg_prefix}inputs_embeds"
+            )
+
+        if position_ids is None:
+            seq_length = input_ids.shape[1]
+            past_key_values_length = (
+                0 if past_key_values is None else past_key_values[0][0].shape[2]
+            )
+            device = input_ids.device if input_ids is not None else inputs_embeds.device
+            position_ids = torch.arange(
+                past_key_values_length,
+                seq_length + past_key_values_length,
+                dtype=torch.long,
+                device=device,
+            ).unsqueeze(0)
+
+        if inputs_embeds is None:
+            if self.embed_tokens is None:
+                raise ValueError(
+                    "You have to initialize the model with valid token embeddings"
+                )
+            inputs_embeds = self.embed_tokens(input_ids)
+
+        batch_size, seq_length = input_shape
+
+        # required mask seq length can be calculated via length of past
+        mask_seq_length = (
+            past_key_values[0][0].shape[2] + seq_length
+            if past_key_values is not None
+            else seq_length
+        )
+
+        if use_cache is True:
+            if not self.is_decoder:
+                raise ValueError(
+                    f"`use_cache` can only be set to `True` if {self} is used as a decoder"
+                )
+
+        if attention_mask is None:
+            attention_mask = torch.ones(
+                batch_size, mask_seq_length, device=inputs_embeds.device
+            )
+
+        # initialize past_key_values with `None` if past does not exist
+        if past_key_values is None:
+            past_key_values = [None] * len(self.block)
+
+        # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
+        # ourselves in which case we just need to make it broadcastable to all heads.
+        extended_attention_mask = self.get_extended_attention_mask(
+            attention_mask, input_shape
+        )
+
+        if self.gradient_checkpointing and self.training:
+            if use_cache:
+                logger.warning_once(
+                    "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..."
+                )
+                use_cache = False
+
+        # Prepare head mask if needed
+        head_mask = self.get_head_mask(head_mask, self.config.num_layers)
+        cross_attn_head_mask = self.get_head_mask(
+            cross_attn_head_mask, self.config.num_layers
+        )
+        present_key_value_states = () if use_cache else None
+        all_hidden_states = () if output_hidden_states else None
+        all_attentions = () if output_attentions else None
+        all_cross_attentions = () if (output_attentions and self.is_decoder) else None
+        position_bias = None
+
+        hidden_states = self.dropout(inputs_embeds)
+
+        for i, (layer_module, past_key_value) in enumerate(
+            zip(self.block, past_key_values)
+        ):
+            layer_head_mask = head_mask[i]
+            cross_attn_layer_head_mask = cross_attn_head_mask[i]
+            # Model parallel
+            if self.model_parallel:
+                torch.cuda.set_device(hidden_states.device)
+                # Ensure that attention_mask is always on the same device as hidden_states
+                if attention_mask is not None:
+                    attention_mask = attention_mask.to(hidden_states.device)
+                if position_bias is not None:
+                    position_bias = position_bias.to(hidden_states.device)                
+                if layer_head_mask is not None:
+                    layer_head_mask = layer_head_mask.to(hidden_states.device)
+
+            if output_hidden_states:
+                all_hidden_states = all_hidden_states + (hidden_states,)
+
+            if self.gradient_checkpointing and self.training:
+                layer_outputs = self._gradient_checkpointing_func(
+                    layer_module.forward,
+                    hidden_states,
+                    extended_attention_mask,
+                    position_bias,
+                    None,
+                    None,
+                    None,
+                    layer_head_mask,
+                    cross_attn_layer_head_mask,
+                    None,  # past_key_value is always None with gradient checkpointing
+                    use_cache,
+                    output_attentions,
+                )
+            else:
+                layer_outputs = layer_module(
+                    hidden_states,
+                    attention_mask=extended_attention_mask,
+                    position_bias=position_bias,
+                    position_ids=position_ids,
+                    encoder_hidden_states=None,
+                    encoder_attention_mask=None,
+                    encoder_decoder_position_bias=None,
+                    layer_head_mask=layer_head_mask,
+                    cross_attn_layer_head_mask=cross_attn_layer_head_mask,
+                    past_key_value=past_key_value,
+                    use_cache=use_cache,
+                    output_attentions=output_attentions,
+                )
+
+            # layer_outputs is a tuple with:
+            # hidden-states, key-value-states, (self-attention position bias), (self-attention weights), (cross-attention position bias), (cross-attention weights)
+            if use_cache is False:
+                layer_outputs = layer_outputs[:1] + (None,) + layer_outputs[1:]
+
+            hidden_states, present_key_value_state = layer_outputs[:2]
+
+            # We share the position biases between the layers - the first layer store them
+            # layer_outputs = hidden-states, key-value-states (self-attention position bias), (self-attention weights),
+            # (cross-attention position bias), (cross-attention weights)
+            position_bias = layer_outputs[2]
+            # append next layer key value states
+            if use_cache:
+                present_key_value_states = present_key_value_states + (
+                    present_key_value_state,
+                )
+
+            if output_attentions:
+                all_attentions = all_attentions + (layer_outputs[3],)
+                if self.is_decoder:
+                    all_cross_attentions = all_cross_attentions + (layer_outputs[5],)
+
+            # Model Parallel: If it's the last layer for that device, put things on the next device
+            if self.model_parallel:
+                for k, v in self.device_map.items():
+                    if i == v[-1] and "cuda:" + str(k) != self.last_device:
+                        hidden_states = hidden_states.to("cuda:" + str(k + 1))
+
+        hidden_states = self.final_layer_norm(hidden_states)
+        hidden_states = self.dropout(hidden_states)
+
+        # Add last layer
+        if output_hidden_states:
+            all_hidden_states = all_hidden_states + (hidden_states,)
+
+        if not return_dict:
+            return tuple(
+                v
+                for v in [
+                    hidden_states,
+                    present_key_value_states,
+                    all_hidden_states,
+                    all_attentions,
+                    all_cross_attentions,
+                ]
+                if v is not None
+            )
+        return modeling_t5.BaseModelOutputWithPastAndCrossAttentions(
+            last_hidden_state=hidden_states,
+            past_key_values=present_key_value_states,
+            hidden_states=all_hidden_states,
+            attentions=all_attentions,
+            cross_attentions=all_cross_attentions,
+        )
+
+
+class DecoderOnlyT5Model(modeling_t5.T5ForConditionalGeneration):
+    def __init__(self, config: DecoderOnlyT5Config):
+        super(modeling_t5.T5ForConditionalGeneration, self).__init__(config)
+        self.model_dim = config.d_model
+
+        self.shared = nn.Embedding(config.vocab_size, config.d_model)
+        assert (
+            self.config.num_layers == 0
+        ), "Decoder only model cannot have encoder layers"
+        self.encoder = None
+
+        decoder_config = copy.deepcopy(config)
+        decoder_config.is_decoder = True
+        decoder_config.is_encoder_decoder = False
+        decoder_config.num_layers = config.num_decoder_layers
+        self.decoder = DecoderOnlyT5Stack(decoder_config, self.shared)
+
+        self.lm_head = nn.Linear(config.d_model, config.vocab_size, bias=False)
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+        # Model parallel
+        self.model_parallel = False
+        self.device_map = None
+
+    def _tie_weights(self):
+        if not self.config.tie_word_embeddings:
+            return
+        if self.decoder:
+            self._tie_or_clone_weights(self.decoder.embed_tokens, self.shared)
+
+    @add_start_docstrings_to_model_forward(modeling_t5.T5_INPUTS_DOCSTRING)
+    @replace_return_docstrings(
+        output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC
+    )
+    def forward(
+        self,
+        input_ids: Optional[torch.LongTensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        attention_mask: Optional[torch.FloatTensor] = None,
+        past_key_values: Optional[Tuple[Tuple[torch.Tensor]]] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        labels: Optional[torch.LongTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+    ) -> Union[Tuple, CausalLMOutputWithPast]:
+        r"""
+        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
+            Labels for computing the sequence classification/regression loss. Indices should be in `[-100, 0, ...,
+            config.vocab_size - 1]`. All labels set to `-100` are ignored (masked), the loss is only computed for
+            labels in `[0, ..., config.vocab_size]`
+
+        Returns:
+
+        Examples:
+
+        ```"""
+        use_cache = use_cache if use_cache is not None else self.config.use_cache
+        return_dict = (
+            return_dict if return_dict is not None else self.config.use_return_dict
+        )
+
+        if self.model_parallel:
+            torch.cuda.set_device(self.decoder.first_device)
+
+        # Set device for model parallelism
+        if self.model_parallel:
+            torch.cuda.set_device(self.decoder.first_device)
+            if input_ids is not None:
+                input_ids = input_ids.to(self.decoder.first_device)
+            if attention_mask is not None:
+                attention_mask = attention_mask.to(self.decoder.first_device)
+
+        # Decode
+        outputs = self.decoder(
+            input_ids=input_ids,
+            position_ids=position_ids,
+            attention_mask=attention_mask,
+            inputs_embeds=inputs_embeds,
+            past_key_values=past_key_values,
+            encoder_hidden_states=None,
+            encoder_attention_mask=None,
+            head_mask=None,
+            cross_attn_head_mask=None,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+
+        sequence_output = outputs[0]
+
+        # Set device for model parallelism
+        if self.model_parallel:
+            torch.cuda.set_device(self.decoder.first_device)
+            self.lm_head = self.lm_head.to(self.decoder.first_device)
+            sequence_output = sequence_output.to(self.lm_head.weight.device)
+
+        if self.config.tie_word_embeddings:
+            # Rescale output before projecting on vocab
+            # See https://github.com/tensorflow/mesh/blob/fa19d69eafc9a482aff0b59ddd96b025c0cb207d/mesh_tensorflow/transformer/transformer.py#L586
+            sequence_output = sequence_output * (self.model_dim**-0.5)
+
+        lm_logits = self.lm_head(sequence_output)
+
+        loss = None
+        if labels is not None:
+            loss_fct = CrossEntropyLoss(ignore_index=-100)
+            # move labels to correct device to enable PP
+            labels = labels.to(lm_logits.device)
+            loss = loss_fct(lm_logits.view(-1, lm_logits.size(-1)), labels.view(-1))
+            # TODO(thom): Add z_loss https://github.com/tensorflow/mesh/blob/fa19d69eafc9a482aff0b59ddd96b025c0cb207d/mesh_tensorflow/layers.py#L666
+
+        if not return_dict:
+            output = (lm_logits,) + outputs[1:]
+            return ((loss,) + output) if loss is not None else output
+
+        return CausalLMOutputWithPast(
+            loss=loss,
+            logits=lm_logits,
+            past_key_values=outputs.past_key_values,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )

model-00000-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eee3b0c4eef668152f9f1106f18bf0a892bd04ba8b26017d7d5865f49dec5f3c
+size 5150622792

model-00001-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29b44a655d9261522e705d963d1fa7ca1717c3e6bcfb9402fa69cb8ee6156c6f
+size 4739650416

model-00002-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1dbd16405fa07722d953ba5c99aeb8ae05c1068cb4018a9622e5828336c1b9c8
+size 4739650424

model-00003-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7444d7670f145f12706970a191b260b609a82626c5e417f08f1039c05fbdda75
+size 4739650456

model-00004-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:471e481023c3f4622d2ff0b1031a34c55469fb10fe1252f5c5c62e8f95418b4b
+size 4739650456

model-00005-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e934d8fcf5bc54649e82b29c1322b7ce3d13e8915d2cac205e65660e0a4cdbbb
+size 4739650456

model-00006-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f3f82de95453b49cb5a78ec517fac3219556f83edd9075a20a7ac95a577b5e93
+size 4739650456

model-00007-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f816e875186339eaf9afa8964d34952c80e697ade369154512f9900ea0a33553
+size 947930104

model.safetensors.index.json

@@ -0,0 +1,262 @@
+{
+    "metadata": {},
+    "weight_map": {
+        "shared.weight": "model-00000-of-00007.safetensors",
+        "decoder.block.0.layer.0.layer_norm.weight": "model-00000-of-00007.safetensors",
+        "decoder.block.0.layer.0.SelfAttention.k.weight": "model-00000-of-00007.safetensors",
+        "decoder.block.0.layer.0.SelfAttention.o.weight": "model-00000-of-00007.safetensors",
+        "decoder.block.0.layer.0.SelfAttention.q.weight": "model-00000-of-00007.safetensors",
+        "decoder.block.0.layer.0.SelfAttention.v.weight": "model-00000-of-00007.safetensors",
+        "decoder.block.0.layer.2.DenseReluDense.wi_0.weight": "model-00000-of-00007.safetensors",
+        "decoder.block.0.layer.2.DenseReluDense.wi_1.weight": "model-00000-of-00007.safetensors",
+        "decoder.block.0.layer.2.DenseReluDense.wo.weight": "model-00000-of-00007.safetensors",
+        "decoder.block.1.layer.0.layer_norm.weight": "model-00001-of-00007.safetensors",
+        "decoder.block.1.layer.0.SelfAttention.k.weight": "model-00001-of-00007.safetensors",
+        "decoder.block.1.layer.0.SelfAttention.o.weight": "model-00001-of-00007.safetensors",
+        "decoder.block.1.layer.0.SelfAttention.q.weight": "model-00001-of-00007.safetensors",
+        "decoder.block.1.layer.0.SelfAttention.v.weight": "model-00001-of-00007.safetensors",
+        "decoder.block.1.layer.2.DenseReluDense.wi_0.weight": "model-00001-of-00007.safetensors",
+        "decoder.block.1.layer.2.DenseReluDense.wi_1.weight": "model-00001-of-00007.safetensors",
+        "decoder.block.1.layer.2.DenseReluDense.wo.weight": "model-00001-of-00007.safetensors",
+        "decoder.block.2.layer.0.layer_norm.weight": "model-00001-of-00007.safetensors",
+        "decoder.block.2.layer.0.SelfAttention.k.weight": "model-00001-of-00007.safetensors",
+        "decoder.block.2.layer.0.SelfAttention.o.weight": "model-00001-of-00007.safetensors",
+        "decoder.block.2.layer.0.SelfAttention.q.weight": "model-00001-of-00007.safetensors",
+        "decoder.block.2.layer.0.SelfAttention.v.weight": "model-00001-of-00007.safetensors",
+        "decoder.block.2.layer.2.DenseReluDense.wi_0.weight": "model-00001-of-00007.safetensors",
+        "decoder.block.2.layer.2.DenseReluDense.wi_1.weight": "model-00001-of-00007.safetensors",
+        "decoder.block.2.layer.2.DenseReluDense.wo.weight": "model-00001-of-00007.safetensors",
+        "decoder.block.3.layer.0.layer_norm.weight": "model-00001-of-00007.safetensors",
+        "decoder.block.3.layer.0.SelfAttention.k.weight": "model-00001-of-00007.safetensors",
+        "decoder.block.3.layer.0.SelfAttention.o.weight": "model-00001-of-00007.safetensors",
+        "decoder.block.3.layer.0.SelfAttention.q.weight": "model-00001-of-00007.safetensors",
+        "decoder.block.3.layer.0.SelfAttention.v.weight": "model-00001-of-00007.safetensors",
+        "decoder.block.3.layer.2.DenseReluDense.wi_0.weight": "model-00001-of-00007.safetensors",
+        "decoder.block.3.layer.2.DenseReluDense.wi_1.weight": "model-00001-of-00007.safetensors",
+        "decoder.block.3.layer.2.DenseReluDense.wo.weight": "model-00001-of-00007.safetensors",
+        "decoder.block.4.layer.0.layer_norm.weight": "model-00001-of-00007.safetensors",
+        "decoder.block.4.layer.0.SelfAttention.k.weight": "model-00001-of-00007.safetensors",
+        "decoder.block.4.layer.0.SelfAttention.o.weight": "model-00001-of-00007.safetensors",
+        "decoder.block.4.layer.0.SelfAttention.q.weight": "model-00001-of-00007.safetensors",
+        "decoder.block.4.layer.0.SelfAttention.v.weight": "model-00001-of-00007.safetensors",
+        "decoder.block.4.layer.2.DenseReluDense.wi_0.weight": "model-00001-of-00007.safetensors",
+        "decoder.block.4.layer.2.DenseReluDense.wi_1.weight": "model-00001-of-00007.safetensors",
+        "decoder.block.4.layer.2.DenseReluDense.wo.weight": "model-00001-of-00007.safetensors",
+        "decoder.block.5.layer.0.layer_norm.weight": "model-00001-of-00007.safetensors",
+        "decoder.block.5.layer.0.SelfAttention.k.weight": "model-00001-of-00007.safetensors",
+        "decoder.block.5.layer.0.SelfAttention.o.weight": "model-00001-of-00007.safetensors",
+        "decoder.block.5.layer.0.SelfAttention.q.weight": "model-00001-of-00007.safetensors",
+        "decoder.block.5.layer.0.SelfAttention.v.weight": "model-00001-of-00007.safetensors",
+        "decoder.block.5.layer.2.DenseReluDense.wi_0.weight": "model-00001-of-00007.safetensors",
+        "decoder.block.5.layer.2.DenseReluDense.wi_1.weight": "model-00001-of-00007.safetensors",
+        "decoder.block.5.layer.2.DenseReluDense.wo.weight": "model-00001-of-00007.safetensors",
+        "decoder.block.6.layer.0.layer_norm.weight": "model-00002-of-00007.safetensors",
+        "decoder.block.6.layer.0.SelfAttention.k.weight": "model-00002-of-00007.safetensors",
+        "decoder.block.6.layer.0.SelfAttention.o.weight": "model-00002-of-00007.safetensors",
+        "decoder.block.6.layer.0.SelfAttention.q.weight": "model-00002-of-00007.safetensors",
+        "decoder.block.6.layer.0.SelfAttention.v.weight": "model-00002-of-00007.safetensors",
+        "decoder.block.6.layer.2.DenseReluDense.wi_0.weight": "model-00002-of-00007.safetensors",
+        "decoder.block.6.layer.2.DenseReluDense.wi_1.weight": "model-00002-of-00007.safetensors",
+        "decoder.block.6.layer.2.DenseReluDense.wo.weight": "model-00002-of-00007.safetensors",
+        "decoder.block.7.layer.0.layer_norm.weight": "model-00002-of-00007.safetensors",
+        "decoder.block.7.layer.0.SelfAttention.k.weight": "model-00002-of-00007.safetensors",
+        "decoder.block.7.layer.0.SelfAttention.o.weight": "model-00002-of-00007.safetensors",
+        "decoder.block.7.layer.0.SelfAttention.q.weight": "model-00002-of-00007.safetensors",
+        "decoder.block.7.layer.0.SelfAttention.v.weight": "model-00002-of-00007.safetensors",
+        "decoder.block.7.layer.2.DenseReluDense.wi_0.weight": "model-00002-of-00007.safetensors",
+        "decoder.block.7.layer.2.DenseReluDense.wi_1.weight": "model-00002-of-00007.safetensors",
+        "decoder.block.7.layer.2.DenseReluDense.wo.weight": "model-00002-of-00007.safetensors",
+        "decoder.block.8.layer.0.layer_norm.weight": "model-00002-of-00007.safetensors",
+        "decoder.block.8.layer.0.SelfAttention.k.weight": "model-00002-of-00007.safetensors",
+        "decoder.block.8.layer.0.SelfAttention.o.weight": "model-00002-of-00007.safetensors",
+        "decoder.block.8.layer.0.SelfAttention.q.weight": "model-00002-of-00007.safetensors",
+        "decoder.block.8.layer.0.SelfAttention.v.weight": "model-00002-of-00007.safetensors",
+        "decoder.block.8.layer.2.DenseReluDense.wi_0.weight": "model-00002-of-00007.safetensors",
+        "decoder.block.8.layer.2.DenseReluDense.wi_1.weight": "model-00002-of-00007.safetensors",
+        "decoder.block.8.layer.2.DenseReluDense.wo.weight": "model-00002-of-00007.safetensors",
+        "decoder.block.9.layer.0.layer_norm.weight": "model-00002-of-00007.safetensors",
+        "decoder.block.9.layer.0.SelfAttention.k.weight": "model-00002-of-00007.safetensors",
+        "decoder.block.9.layer.0.SelfAttention.o.weight": "model-00002-of-00007.safetensors",
+        "decoder.block.9.layer.0.SelfAttention.q.weight": "model-00002-of-00007.safetensors",
+        "decoder.block.9.layer.0.SelfAttention.v.weight": "model-00002-of-00007.safetensors",
+        "decoder.block.9.layer.2.DenseReluDense.wi_0.weight": "model-00002-of-00007.safetensors",
+        "decoder.block.9.layer.2.DenseReluDense.wi_1.weight": "model-00002-of-00007.safetensors",
+        "decoder.block.9.layer.2.DenseReluDense.wo.weight": "model-00002-of-00007.safetensors",
+        "decoder.block.10.layer.0.layer_norm.weight": "model-00002-of-00007.safetensors",
+        "decoder.block.10.layer.0.SelfAttention.k.weight": "model-00002-of-00007.safetensors",
+        "decoder.block.10.layer.0.SelfAttention.o.weight": "model-00002-of-00007.safetensors",
+        "decoder.block.10.layer.0.SelfAttention.q.weight": "model-00002-of-00007.safetensors",
+        "decoder.block.10.layer.0.SelfAttention.v.weight": "model-00002-of-00007.safetensors",
+        "decoder.block.10.layer.2.DenseReluDense.wi_0.weight": "model-00002-of-00007.safetensors",
+        "decoder.block.10.layer.2.DenseReluDense.wi_1.weight": "model-00002-of-00007.safetensors",
+        "decoder.block.10.layer.2.DenseReluDense.wo.weight": "model-00002-of-00007.safetensors",
+        "decoder.block.11.layer.0.layer_norm.weight": "model-00003-of-00007.safetensors",
+        "decoder.block.11.layer.0.SelfAttention.k.weight": "model-00003-of-00007.safetensors",
+        "decoder.block.11.layer.0.SelfAttention.o.weight": "model-00003-of-00007.safetensors",
+        "decoder.block.11.layer.0.SelfAttention.q.weight": "model-00003-of-00007.safetensors",
+        "decoder.block.11.layer.0.SelfAttention.v.weight": "model-00003-of-00007.safetensors",
+        "decoder.block.11.layer.2.DenseReluDense.wi_0.weight": "model-00003-of-00007.safetensors",
+        "decoder.block.11.layer.2.DenseReluDense.wi_1.weight": "model-00003-of-00007.safetensors",
+        "decoder.block.11.layer.2.DenseReluDense.wo.weight": "model-00003-of-00007.safetensors",
+        "decoder.block.12.layer.0.layer_norm.weight": "model-00003-of-00007.safetensors",
+        "decoder.block.12.layer.0.SelfAttention.k.weight": "model-00003-of-00007.safetensors",
+        "decoder.block.12.layer.0.SelfAttention.o.weight": "model-00003-of-00007.safetensors",
+        "decoder.block.12.layer.0.SelfAttention.q.weight": "model-00003-of-00007.safetensors",
+        "decoder.block.12.layer.0.SelfAttention.v.weight": "model-00003-of-00007.safetensors",
+        "decoder.block.12.layer.2.DenseReluDense.wi_0.weight": "model-00003-of-00007.safetensors",
+        "decoder.block.12.layer.2.DenseReluDense.wi_1.weight": "model-00003-of-00007.safetensors",
+        "decoder.block.12.layer.2.DenseReluDense.wo.weight": "model-00003-of-00007.safetensors",
+        "decoder.block.13.layer.0.layer_norm.weight": "model-00003-of-00007.safetensors",
+        "decoder.block.13.layer.0.SelfAttention.k.weight": "model-00003-of-00007.safetensors",
+        "decoder.block.13.layer.0.SelfAttention.o.weight": "model-00003-of-00007.safetensors",
+        "decoder.block.13.layer.0.SelfAttention.q.weight": "model-00003-of-00007.safetensors",
+        "decoder.block.13.layer.0.SelfAttention.v.weight": "model-00003-of-00007.safetensors",
+        "decoder.block.13.layer.2.DenseReluDense.wi_0.weight": "model-00003-of-00007.safetensors",
+        "decoder.block.13.layer.2.DenseReluDense.wi_1.weight": "model-00003-of-00007.safetensors",
+        "decoder.block.13.layer.2.DenseReluDense.wo.weight": "model-00003-of-00007.safetensors",
+        "decoder.block.14.layer.0.layer_norm.weight": "model-00003-of-00007.safetensors",
+        "decoder.block.14.layer.0.SelfAttention.k.weight": "model-00003-of-00007.safetensors",
+        "decoder.block.14.layer.0.SelfAttention.o.weight": "model-00003-of-00007.safetensors",
+        "decoder.block.14.layer.0.SelfAttention.q.weight": "model-00003-of-00007.safetensors",
+        "decoder.block.14.layer.0.SelfAttention.v.weight": "model-00003-of-00007.safetensors",
+        "decoder.block.14.layer.2.DenseReluDense.wi_0.weight": "model-00003-of-00007.safetensors",
+        "decoder.block.14.layer.2.DenseReluDense.wi_1.weight": "model-00003-of-00007.safetensors",
+        "decoder.block.14.layer.2.DenseReluDense.wo.weight": "model-00003-of-00007.safetensors",
+        "decoder.block.15.layer.0.layer_norm.weight": "model-00003-of-00007.safetensors",
+        "decoder.block.15.layer.0.SelfAttention.k.weight": "model-00003-of-00007.safetensors",
+        "decoder.block.15.layer.0.SelfAttention.o.weight": "model-00003-of-00007.safetensors",
+        "decoder.block.15.layer.0.SelfAttention.q.weight": "model-00003-of-00007.safetensors",
+        "decoder.block.15.layer.0.SelfAttention.v.weight": "model-00003-of-00007.safetensors",
+        "decoder.block.15.layer.2.DenseReluDense.wi_0.weight": "model-00003-of-00007.safetensors",
+        "decoder.block.15.layer.2.DenseReluDense.wi_1.weight": "model-00003-of-00007.safetensors",
+        "decoder.block.15.layer.2.DenseReluDense.wo.weight": "model-00003-of-00007.safetensors",
+        "decoder.block.16.layer.0.layer_norm.weight": "model-00004-of-00007.safetensors",
+        "decoder.block.16.layer.0.SelfAttention.k.weight": "model-00004-of-00007.safetensors",
+        "decoder.block.16.layer.0.SelfAttention.o.weight": "model-00004-of-00007.safetensors",
+        "decoder.block.16.layer.0.SelfAttention.q.weight": "model-00004-of-00007.safetensors",
+        "decoder.block.16.layer.0.SelfAttention.v.weight": "model-00004-of-00007.safetensors",
+        "decoder.block.16.layer.2.DenseReluDense.wi_0.weight": "model-00004-of-00007.safetensors",
+        "decoder.block.16.layer.2.DenseReluDense.wi_1.weight": "model-00004-of-00007.safetensors",
+        "decoder.block.16.layer.2.DenseReluDense.wo.weight": "model-00004-of-00007.safetensors",
+        "decoder.block.17.layer.0.layer_norm.weight": "model-00004-of-00007.safetensors",
+        "decoder.block.17.layer.0.SelfAttention.k.weight": "model-00004-of-00007.safetensors",
+        "decoder.block.17.layer.0.SelfAttention.o.weight": "model-00004-of-00007.safetensors",
+        "decoder.block.17.layer.0.SelfAttention.q.weight": "model-00004-of-00007.safetensors",
+        "decoder.block.17.layer.0.SelfAttention.v.weight": "model-00004-of-00007.safetensors",
+        "decoder.block.17.layer.2.DenseReluDense.wi_0.weight": "model-00004-of-00007.safetensors",
+        "decoder.block.17.layer.2.DenseReluDense.wi_1.weight": "model-00004-of-00007.safetensors",
+        "decoder.block.17.layer.2.DenseReluDense.wo.weight": "model-00004-of-00007.safetensors",
+        "decoder.block.18.layer.0.layer_norm.weight": "model-00004-of-00007.safetensors",
+        "decoder.block.18.layer.0.SelfAttention.k.weight": "model-00004-of-00007.safetensors",
+        "decoder.block.18.layer.0.SelfAttention.o.weight": "model-00004-of-00007.safetensors",
+        "decoder.block.18.layer.0.SelfAttention.q.weight": "model-00004-of-00007.safetensors",
+        "decoder.block.18.layer.0.SelfAttention.v.weight": "model-00004-of-00007.safetensors",
+        "decoder.block.18.layer.2.DenseReluDense.wi_0.weight": "model-00004-of-00007.safetensors",
+        "decoder.block.18.layer.2.DenseReluDense.wi_1.weight": "model-00004-of-00007.safetensors",
+        "decoder.block.18.layer.2.DenseReluDense.wo.weight": "model-00004-of-00007.safetensors",
+        "decoder.block.19.layer.0.layer_norm.weight": "model-00004-of-00007.safetensors",
+        "decoder.block.19.layer.0.SelfAttention.k.weight": "model-00004-of-00007.safetensors",
+        "decoder.block.19.layer.0.SelfAttention.o.weight": "model-00004-of-00007.safetensors",
+        "decoder.block.19.layer.0.SelfAttention.q.weight": "model-00004-of-00007.safetensors",
+        "decoder.block.19.layer.0.SelfAttention.v.weight": "model-00004-of-00007.safetensors",
+        "decoder.block.19.layer.2.DenseReluDense.wi_0.weight": "model-00004-of-00007.safetensors",
+        "decoder.block.19.layer.2.DenseReluDense.wi_1.weight": "model-00004-of-00007.safetensors",
+        "decoder.block.19.layer.2.DenseReluDense.wo.weight": "model-00004-of-00007.safetensors",
+        "decoder.block.20.layer.0.layer_norm.weight": "model-00004-of-00007.safetensors",
+        "decoder.block.20.layer.0.SelfAttention.k.weight": "model-00004-of-00007.safetensors",
+        "decoder.block.20.layer.0.SelfAttention.o.weight": "model-00004-of-00007.safetensors",
+        "decoder.block.20.layer.0.SelfAttention.q.weight": "model-00004-of-00007.safetensors",
+        "decoder.block.20.layer.0.SelfAttention.v.weight": "model-00004-of-00007.safetensors",
+        "decoder.block.20.layer.2.DenseReluDense.wi_0.weight": "model-00004-of-00007.safetensors",
+        "decoder.block.20.layer.2.DenseReluDense.wi_1.weight": "model-00004-of-00007.safetensors",
+        "decoder.block.20.layer.2.DenseReluDense.wo.weight": "model-00004-of-00007.safetensors",
+        "decoder.block.21.layer.0.layer_norm.weight": "model-00005-of-00007.safetensors",
+        "decoder.block.21.layer.0.SelfAttention.k.weight": "model-00005-of-00007.safetensors",
+        "decoder.block.21.layer.0.SelfAttention.o.weight": "model-00005-of-00007.safetensors",
+        "decoder.block.21.layer.0.SelfAttention.q.weight": "model-00005-of-00007.safetensors",
+        "decoder.block.21.layer.0.SelfAttention.v.weight": "model-00005-of-00007.safetensors",
+        "decoder.block.21.layer.2.DenseReluDense.wi_0.weight": "model-00005-of-00007.safetensors",
+        "decoder.block.21.layer.2.DenseReluDense.wi_1.weight": "model-00005-of-00007.safetensors",
+        "decoder.block.21.layer.2.DenseReluDense.wo.weight": "model-00005-of-00007.safetensors",
+        "decoder.block.22.layer.0.layer_norm.weight": "model-00005-of-00007.safetensors",
+        "decoder.block.22.layer.0.SelfAttention.k.weight": "model-00005-of-00007.safetensors",
+        "decoder.block.22.layer.0.SelfAttention.o.weight": "model-00005-of-00007.safetensors",
+        "decoder.block.22.layer.0.SelfAttention.q.weight": "model-00005-of-00007.safetensors",
+        "decoder.block.22.layer.0.SelfAttention.v.weight": "model-00005-of-00007.safetensors",
+        "decoder.block.22.layer.2.DenseReluDense.wi_0.weight": "model-00005-of-00007.safetensors",
+        "decoder.block.22.layer.2.DenseReluDense.wi_1.weight": "model-00005-of-00007.safetensors",
+        "decoder.block.22.layer.2.DenseReluDense.wo.weight": "model-00005-of-00007.safetensors",
+        "decoder.block.23.layer.0.layer_norm.weight": "model-00005-of-00007.safetensors",
+        "decoder.block.23.layer.0.SelfAttention.k.weight": "model-00005-of-00007.safetensors",
+        "decoder.block.23.layer.0.SelfAttention.o.weight": "model-00005-of-00007.safetensors",
+        "decoder.block.23.layer.0.SelfAttention.q.weight": "model-00005-of-00007.safetensors",
+        "decoder.block.23.layer.0.SelfAttention.v.weight": "model-00005-of-00007.safetensors",
+        "decoder.block.23.layer.2.DenseReluDense.wi_0.weight": "model-00005-of-00007.safetensors",
+        "decoder.block.23.layer.2.DenseReluDense.wi_1.weight": "model-00005-of-00007.safetensors",
+        "decoder.block.23.layer.2.DenseReluDense.wo.weight": "model-00005-of-00007.safetensors",
+        "decoder.block.24.layer.0.layer_norm.weight": "model-00005-of-00007.safetensors",
+        "decoder.block.24.layer.0.SelfAttention.k.weight": "model-00005-of-00007.safetensors",
+        "decoder.block.24.layer.0.SelfAttention.o.weight": "model-00005-of-00007.safetensors",
+        "decoder.block.24.layer.0.SelfAttention.q.weight": "model-00005-of-00007.safetensors",
+        "decoder.block.24.layer.0.SelfAttention.v.weight": "model-00005-of-00007.safetensors",
+        "decoder.block.24.layer.2.DenseReluDense.wi_0.weight": "model-00005-of-00007.safetensors",
+        "decoder.block.24.layer.2.DenseReluDense.wi_1.weight": "model-00005-of-00007.safetensors",
+        "decoder.block.24.layer.2.DenseReluDense.wo.weight": "model-00005-of-00007.safetensors",
+        "decoder.block.25.layer.0.layer_norm.weight": "model-00005-of-00007.safetensors",
+        "decoder.block.25.layer.0.SelfAttention.k.weight": "model-00005-of-00007.safetensors",
+        "decoder.block.25.layer.0.SelfAttention.o.weight": "model-00005-of-00007.safetensors",
+        "decoder.block.25.layer.0.SelfAttention.q.weight": "model-00005-of-00007.safetensors",
+        "decoder.block.25.layer.0.SelfAttention.v.weight": "model-00005-of-00007.safetensors",
+        "decoder.block.25.layer.2.DenseReluDense.wi_0.weight": "model-00005-of-00007.safetensors",
+        "decoder.block.25.layer.2.DenseReluDense.wi_1.weight": "model-00005-of-00007.safetensors",
+        "decoder.block.25.layer.2.DenseReluDense.wo.weight": "model-00005-of-00007.safetensors",
+        "decoder.block.26.layer.0.layer_norm.weight": "model-00006-of-00007.safetensors",
+        "decoder.block.26.layer.0.SelfAttention.k.weight": "model-00006-of-00007.safetensors",
+        "decoder.block.26.layer.0.SelfAttention.o.weight": "model-00006-of-00007.safetensors",
+        "decoder.block.26.layer.0.SelfAttention.q.weight": "model-00006-of-00007.safetensors",
+        "decoder.block.26.layer.0.SelfAttention.v.weight": "model-00006-of-00007.safetensors",
+        "decoder.block.26.layer.2.DenseReluDense.wi_0.weight": "model-00006-of-00007.safetensors",
+        "decoder.block.26.layer.2.DenseReluDense.wi_1.weight": "model-00006-of-00007.safetensors",
+        "decoder.block.26.layer.2.DenseReluDense.wo.weight": "model-00006-of-00007.safetensors",
+        "decoder.block.27.layer.0.layer_norm.weight": "model-00006-of-00007.safetensors",
+        "decoder.block.27.layer.0.SelfAttention.k.weight": "model-00006-of-00007.safetensors",
+        "decoder.block.27.layer.0.SelfAttention.o.weight": "model-00006-of-00007.safetensors",
+        "decoder.block.27.layer.0.SelfAttention.q.weight": "model-00006-of-00007.safetensors",
+        "decoder.block.27.layer.0.SelfAttention.v.weight": "model-00006-of-00007.safetensors",
+        "decoder.block.27.layer.2.DenseReluDense.wi_0.weight": "model-00006-of-00007.safetensors",
+        "decoder.block.27.layer.2.DenseReluDense.wi_1.weight": "model-00006-of-00007.safetensors",
+        "decoder.block.27.layer.2.DenseReluDense.wo.weight": "model-00006-of-00007.safetensors",
+        "decoder.block.28.layer.0.layer_norm.weight": "model-00006-of-00007.safetensors",
+        "decoder.block.28.layer.0.SelfAttention.k.weight": "model-00006-of-00007.safetensors",
+        "decoder.block.28.layer.0.SelfAttention.o.weight": "model-00006-of-00007.safetensors",
+        "decoder.block.28.layer.0.SelfAttention.q.weight": "model-00006-of-00007.safetensors",
+        "decoder.block.28.layer.0.SelfAttention.v.weight": "model-00006-of-00007.safetensors",
+        "decoder.block.28.layer.2.DenseReluDense.wi_0.weight": "model-00006-of-00007.safetensors",
+        "decoder.block.28.layer.2.DenseReluDense.wi_1.weight": "model-00006-of-00007.safetensors",
+        "decoder.block.28.layer.2.DenseReluDense.wo.weight": "model-00006-of-00007.safetensors",
+        "decoder.block.29.layer.0.layer_norm.weight": "model-00006-of-00007.safetensors",
+        "decoder.block.29.layer.0.SelfAttention.k.weight": "model-00006-of-00007.safetensors",
+        "decoder.block.29.layer.0.SelfAttention.o.weight": "model-00006-of-00007.safetensors",
+        "decoder.block.29.layer.0.SelfAttention.q.weight": "model-00006-of-00007.safetensors",
+        "decoder.block.29.layer.0.SelfAttention.v.weight": "model-00006-of-00007.safetensors",
+        "decoder.block.29.layer.2.DenseReluDense.wi_0.weight": "model-00006-of-00007.safetensors",
+        "decoder.block.29.layer.2.DenseReluDense.wi_1.weight": "model-00006-of-00007.safetensors",
+        "decoder.block.29.layer.2.DenseReluDense.wo.weight": "model-00006-of-00007.safetensors",
+        "decoder.block.30.layer.0.layer_norm.weight": "model-00006-of-00007.safetensors",
+        "decoder.block.30.layer.0.SelfAttention.k.weight": "model-00006-of-00007.safetensors",
+        "decoder.block.30.layer.0.SelfAttention.o.weight": "model-00006-of-00007.safetensors",
+        "decoder.block.30.layer.0.SelfAttention.q.weight": "model-00006-of-00007.safetensors",
+        "decoder.block.30.layer.0.SelfAttention.v.weight": "model-00006-of-00007.safetensors",
+        "decoder.block.30.layer.2.DenseReluDense.wi_0.weight": "model-00006-of-00007.safetensors",
+        "decoder.block.30.layer.2.DenseReluDense.wi_1.weight": "model-00006-of-00007.safetensors",
+        "decoder.block.30.layer.2.DenseReluDense.wo.weight": "model-00006-of-00007.safetensors",
+        "decoder.block.31.layer.0.layer_norm.weight": "model-00007-of-00007.safetensors",
+        "decoder.block.31.layer.0.SelfAttention.k.weight": "model-00007-of-00007.safetensors",
+        "decoder.block.31.layer.0.SelfAttention.o.weight": "model-00007-of-00007.safetensors",
+        "decoder.block.31.layer.0.SelfAttention.q.weight": "model-00007-of-00007.safetensors",
+        "decoder.block.31.layer.0.SelfAttention.v.weight": "model-00007-of-00007.safetensors",
+        "decoder.block.31.layer.2.DenseReluDense.wi_0.weight": "model-00007-of-00007.safetensors",
+        "decoder.block.31.layer.2.DenseReluDense.wi_1.weight": "model-00007-of-00007.safetensors",
+        "decoder.block.31.layer.2.DenseReluDense.wo.weight": "model-00007-of-00007.safetensors"
+    }
+}

special_tokens_map.json

@@ -0,0 +1,23 @@
+{
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

spiece.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef11ac9a22c7503492f56d48dce53be20e339b63605983e9f27d2cd0e0f3922c
+size 4427844

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2799ccc696b752ba00c34f58726bfe253a04921ceb6cfc620400f560474790b
+size 16629031

tokenizer_config.json

@@ -0,0 +1,38 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "</s>",
+  "extra_ids": 0,
+  "legacy": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<s>",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "T5Tokenizer",
+  "unk_token": "<unk>"
+}