Update license

LICENSE.txt

@@ -0,0 +1,72 @@
+TII Falcon LLM License Version 1.0
+May 2023
+falconllm.tii.ae
+
+INTRODUCTORY NOTE
+This license is, in part, based on the Apache License Version 2.0 (available at http://www.apache.org/licenses/), with a series of modifications. The contribution of the Apache License 2.0 to the framing of this document is acknowledged. Please read this license carefully, as it is different to other ‘open source’ licenses you may have encountered previously. In particular, note that this license contains obligations on those of you who are commercially exploiting Falcon LLM or any Derivative Work to make royalty payments.
+
+TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 
+1	Definitions.
+"Commercial Application Address” means Falconllm.sales@tii.ae.
+“Commercial Use” means use where there is, or, in relation to a new use case, a reasonable expectation that there will be revenue directly attributable to the use of the Work for that use case.
+“Commercial User” means someone who has applied to make Commercial Use of the Work and been granted permission by the Licensor to make such Commercial Use in accordance with Section 8. 
+“Contribution” shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, “submitted” means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as “Not a Contribution.” 
+“Contributor” shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work.
+“Derivative Works” shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall include machine learning models trained using outputs from the Falcon LLM or any other Derivative Work, but shall not otherwise include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof.
+“Falcon LLM” shall mean only the following releases of TII’s Falcon large language models: (i) Falcon-RW-1B; (ii) Falcon-RW-7B; (iii) Falcon-7B; (iv) Falcon-40B; (v) Falcon-7B-Instruct; or (vi) Falcon-40B-Instruct; each of which is initially made available in Object form only under this license at FalconLLM.tii.ae. No other sizes or versions of the ‘Falcon’ family of large language models is made available under this license. 
+“Legal Entity” shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, “control” means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity.
+“License” shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 to 11 of this document.
+“Licensor” shall mean the copyright owner or entity authorized by the copyright owner that is granting the License.
+“Object” form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, a trained and/or fine-tuned machine learning model or artificial intelligence model, generated documentation, and conversions to other media types.
+“Source” form shall mean the preferred form for making modifications, including but not limited to software source code, training datasets used for training or fine tuning a machine learning model or artificial intelligence model, documentation source, and configuration files.
+“TII” shall mean the Technology Innovation Institute – Sole Proprietorship L.L.C., or any party nominated in writing by Technology Innovation Institute – Sole Proprietorship L.L.C. as its successor for the purposes of this License, or any party nominated in writing to be a successor to any successor for the purposes of this license.
+“Work” shall mean the work of authorship, which in relation to the initial release of Falcon LLM is in Object form only, but in the case of any and all Derivative Works means the work of authorship whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). 
+“You” (or “Your”) shall mean an individual or Legal Entity exercising permissions granted by this License.
+2	Grant of Copyright License. 
+2.1	Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form.
+2.2	Other than where you are a Commercial User in accordance with Section 8, Your copyright license to use the Work shall be royalty free and without charge. 
+3	Grant of Patent License. 
+3.1	Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed.
+3.2	Other than where you are a Commercial User in accordance with Section 8, Your patent license to use the Work shall be royalty free and without charge. 
+4	Redistribution. 
+4.1	Irrespective of whether you are a Commercial User, You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: 
+(a)	You must give any other recipients of the Work or Derivative Works a copy of this License; and 
+(b)	You must cause any modified files to carry prominent notices stating that You changed the files; and 
+(c)	You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and 
+(d)	If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. 
+4.2	Subject to Sections 5 and 8 below, where you are a Commercial User , then You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License.
+4.3	Where You are not a Commercial User, then: 
+(a)	You must apply this license in an unmodified form as the terms under which you distribute Your modifications or any Derivative Work. Making the Work or Derivative Work available to any third party users via any means will constitute distribution of the Work or Derivative Work for these purposes. 
+(b)	Other than in respect of Falcon LLM, which may only be distributed in Object form, where you distribute Your modifications or any Derivative Work then You must provide both Object and Source versions of the modifications or Derivative Work, provided that this obligation may be satisfied by providing users of the Object version of Your modifications or Derivative Work a link to a location where they may freely download a copy of the Source version.
+(c)	You may include a copyright statement recording the nature of your modifications or Derivative Work, but such copyright statement shall be based on the copyright statement provided with the Work and you shall make only such changes as are strictly necessary to record your contribution.  
+5	Publication 
+5.1	You shall include prominently in any public statement regarding a Derivative Work the following statement: 
+“[name of relevant Derivative Work] is built using Falcon LLM technology from the Technology Innovation Institute”. 
+5.2	You may request from TII a reasonably adjusted version of the above statement to suit the publication the relevant statement is being made in. TII will not unreasonably withhold or delay approval of such a request. 
+5.3	You shall comply with any reasonable request from TII regarding the prominence and phrasing of any reference to the ‘Falcon LLM technology’ or the ‘Technology Innovation Institute’.
+6	Submission of Contributions. 
+6.1	Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions.
+6.2	Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions.
+7	Trademarks. 
+7.1	Except as required for compliance with Section 5 of this License, this License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file.
+8	Commercial Use
+8.1	Where You wish to make Commercial Use of Falcon LLM or any Derivative Work, You must apply to TII for permission to make Commercial Use of that Work in writing via the means specified from time to time at the Commercial Application Address, providing such information as may be required.
+8.2	Where TII grants permission for You to make Commercial Use of the relevant Work, then for that purpose You shall be considered a Commercial User, and:
+(a)	In its written grant of permission, TII shall set the royalty rate that will apply to you as a Commercial User as a percentage of revenue ( “Relevant Percentage”), where, unless otherwise specified in the grant of permission, the Relevant Percentage shall be 10%; and 
+(b)	Each year on the anniversary of the date upon which you were granted permission by TII to make Commercial Use of the relevant Work (the “Anniversary Date") You shall account to TII in writing in full for all revenue you have received in the previous 12 months which is attributable (whether directly or indirectly) to Your use of the relevant Work (“Attributable Revenue”); and
+(c)	Where, on the Anniversary Date, the Attributable Revenue for the preceding 12 months is greater than $1m or its equivalent in the currency or currencies in which the revenue has been earned (the “Royalty Threshold”) then You shall make a payment of the Relevant Percentage of the relevant Attributable Revenue that exceeds the Royalty Threshold in full in cleared funds to TII into the account specified by TII from time to time in writing for such purpose within 30 days of that Anniversary Date.
+9	Disclaimer of Warranty. 
+9.1	Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License.
+10	Limitation of Liability. 
+10.1	In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages.
+11	Accepting Warranty or Additional Liability. 
+11.1	While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability.
+
+END OF TERMS AND CONDITIONS 
+
+APPENDIX: How to apply the TII Falcon LLM License to your work.
+To apply the TII Falcon LLM License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives.
+
+Copyright [yyyy] [name of copyright owner] Licensed under the TII Falcon LLM License, Version 1.0 (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at FalconLLM.tii.ae. Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and limitations under the License.

README.md

@@ -12,19 +12,16 @@ license: apache-2.0
 
 # 🚀 Falcon-40B
 
-**Falcon-40B is a 40B parameters causal decoder-only model built by [TII](https://www.tii.ae) and trained on 1,000B tokens of [RefinedWeb](https://huggingface.co/datasets/tiiuae/falcon-refinedweb) enhanced with curated corpora. It is made available under the Apache 2.0 license.**
+**Falcon-40B is a 40B parameters causal decoder-only model built by [TII](https://www.tii.ae) and trained on 1,000B tokens of [RefinedWeb](https://huggingface.co/datasets/tiiuae/falcon-refinedweb) enhanced with curated corpora. It is made available under the [TII Falcon LLM License](https://huggingface.co/tiiuae/falcon-40b/blob/main/LICENSE.txt).**
 
 *Paper coming soon 😊.*
 
-
-🤗 To get started with Falcon (inference, finetuning, quantization, etc.), we recommend reading [this great blogpost fron HF](https://huggingface.co/blog/falcon)!
-
 ## Why use Falcon-40B?
 
 * **It is the best open-source model currently available.** Falcon-40B outperforms [LLaMA](https://github.com/facebookresearch/llama), [StableLM](https://github.com/Stability-AI/StableLM), [RedPajama](https://huggingface.co/togethercomputer/RedPajama-INCITE-Base-7B-v0.1), [MPT](https://huggingface.co/mosaicml/mpt-7b), etc. See the [OpenLLM Leaderboard](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard).
 * **It features an architecture optimized for inference**, with FlashAttention ([Dao et al., 2022](https://arxiv.org/abs/2205.14135)) and multiquery ([Shazeer et al., 2019](https://arxiv.org/abs/1911.02150)). 
-* **It is made available under a permissive Apache 2.0 license allowing for commercial use**, without any royalties or restrictions.
-* 
+* **It is made available under a license allowing commercial use**, see the details of the [TII Falcon LLM License](https://huggingface.co/tiiuae/falcon-40b/blob/main/LICENSE.txt) below.
+
 ⚠️ **This is a raw, pretrained model, which should be further finetuned for most usecases.** If you are looking for a version better suited to taking generic instructions in a chat format, we recommend taking a look at [Falcon-40B-Instruct](https://huggingface.co/tiiuae/falcon-40b-instruct). 
 
 💸 **Looking for a smaller, less expensive model?** [Falcon-7B](https://huggingface.co/tiiuae/falcon-7b) is Falcon-40B's little brother!
@@ -60,9 +57,6 @@ for seq in sequences:
 
 💥 **Falcon LLMs require PyTorch 2.0 for use with `transformers`!**
 
-For fast inference with Falcon, check-out [Text Generation Inference](https://github.com/huggingface/text-generation-inference)! Read more in this [blogpost]((https://huggingface.co/blog/falcon). 
-
-You will need **at least 85-100GB of memory** to swiftly run inference with Falcon-40B.
 
 # Model Card for Falcon-40B
 
@@ -73,7 +67,7 @@ You will need **at least 85-100GB of memory** to swiftly run inference with Falc
 - **Developed by:** [https://www.tii.ae](https://www.tii.ae);
 - **Model type:** Causal decoder-only;
 - **Language(s) (NLP):** English, German, Spanish, French (and limited capabilities in Italian, Portuguese, Polish, Dutch, Romanian, Czech, Swedish);
-- **License:** Apache 2.0 license.
+- **License:** [TII Falcon LLM License](https://huggingface.co/tiiuae/falcon-40b/blob/main/LICENSE.txt).
 
 ### Model Source
 
@@ -142,7 +136,7 @@ Falcon-40B was trained on 1,000B tokens of [RefinedWeb](https://huggingface.co/d
 | Books  | 6%           | 60B        |                  |
 | Conversations      | 5%           | 50B        | Reddit, StackOverflow, HackerNews |
 | Code               | 5%           | 50B        |                                   |
-| Technical          | 2%           | 20B        | arXiv, PubMed, USPTO, etc.        |
+| Technical          | 2%           | 20B        | arXiv, PubMed, UPSTO, etc.        |
 
 RefinedWeb-Europe is made of the following languages:
 
@@ -225,33 +219,15 @@ Falcon-40B was trained a custom distributed training codebase, Gigatron. It uses
 
 ## Citation
 
-*Paper coming soon* 😊. In the meanwhile, you can use the following information to cite: 
-```
-@article{falcon40b,
-  title={{Falcon-40B}: an open large language model with state-of-the-art performance},
-  author={Almazrouei, Ebtesam and Alobeidli, Hamza and Alshamsi, Abdulaziz and Cappelli, Alessandro and Cojocaru, Ruxandra and Debbah, Merouane and Goffinet, Etienne and Heslow, Daniel and Launay, Julien and Malartic, Quentin and Noune, Badreddine and Pannier, Baptiste and Penedo, Guilherme},
-  year={2023}
-}
-```
-
-To learn more about the pretraining dataset, see the 📓 [RefinedWeb paper](https://arxiv.org/abs/2306.01116).
-
-```
-@article{refinedweb,
-  title={The {R}efined{W}eb dataset for {F}alcon {LLM}: outperforming curated corpora with web data, and web data only},
-  author={Guilherme Penedo and Quentin Malartic and Daniel Hesslow and Ruxandra Cojocaru and Alessandro Cappelli and Hamza Alobeidli and Baptiste Pannier and Ebtesam Almazrouei and Julien Launay},
-  journal={arXiv preprint arXiv:2306.01116},
-  eprint={2306.01116},
-  eprinttype = {arXiv},
-  url={https://arxiv.org/abs/2306.01116},
-  year={2023}
-}
-```
-
+*Paper coming soon 😊.*
 
 ## License
 
-Falcon-40B is made available under the Apache 2.0 license.
+Falcon-40B is made available under the [TII Falcon LLM License](https://huggingface.co/tiiuae/falcon-40b/blob/main/LICENSE.txt). Broadly speaking,
+* You can freely use our models for research and/or personal purpose;
+* You are allowed to share and build derivatives of these models, but you are required to give attribution and to share-alike with the same license;
+* For commercial use, you are exempt from royalties payment if the attributable revenues are inferior to $1M/year, otherwise you should enter in a commercial agreement with TII.
+
 
 ## Contact
 falconllm@tii.ae

config.json

@@ -2,16 +2,16 @@
   "alibi": false,
   "apply_residual_connection_post_layernorm": false,
   "architectures": [
-    "FalconForCausalLM"
+    "RWForCausalLM"
   ],
   "attention_dropout": 0.0,
   "auto_map": {
-    "AutoConfig": "configuration_falcon.FalconConfig",
-    "AutoModel": "modeling_falcon.FalconModel",
-    "AutoModelForSequenceClassification": "modeling_falcon.FalconForSequenceClassification",
-    "AutoModelForTokenClassification": "modeling_falcon.FalconForTokenClassification",
-    "AutoModelForQuestionAnswering": "modeling_falcon.FalconForQuestionAnswering",
-    "AutoModelForCausalLM": "modeling_falcon.FalconForCausalLM"
+    "AutoConfig": "configuration_RW.RWConfig",
+    "AutoModel": "modelling_RW.RWModel",
+    "AutoModelForSequenceClassification": "modelling_RW.RWForSequenceClassification",
+    "AutoModelForTokenClassification": "modelling_RW.RWForTokenClassification",
+    "AutoModelForQuestionAnswering": "modelling_RW.RWForQuestionAnswering",
+    "AutoModelForCausalLM": "modelling_RW.RWForCausalLM"
   },
   "bias": false,
   "bos_token_id": 11,
@@ -20,11 +20,10 @@
   "hidden_size": 8192,
   "initializer_range": 0.02,
   "layer_norm_epsilon": 1e-05,
-  "model_type": "falcon",
-  "new_decoder_architecture": true,
-  "num_attention_heads": 128,
-  "num_hidden_layers": 60,
-  "num_kv_heads": 8,
+  "model_type": "RefinedWeb",
+  "n_head": 128,
+  "n_head_kv": 8,
+  "n_layer": 60,
   "parallel_attn": true,
   "torch_dtype": "bfloat16",
   "transformers_version": "4.27.4",

configuration_RW.py

@@ -0,0 +1,75 @@
+# coding=utf-8
+# Copyright 2022 the Big Science Workshop and HuggingFace Inc. team.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" Bloom configuration"""
+from transformers.configuration_utils import PretrainedConfig
+from transformers.utils import logging
+
+
+logger = logging.get_logger(__name__)
+
+
+class RWConfig(PretrainedConfig):
+    model_type = "RefinedWeb"
+    keys_to_ignore_at_inference = ["past_key_values"]
+    attribute_map = {
+        "num_hidden_layers": "n_layer",
+        "num_attention_heads": "n_head",
+    }
+
+    def __init__(
+        self,
+        vocab_size=250880,
+        hidden_size=64,
+        n_layer=2,
+        n_head=8,
+        layer_norm_epsilon=1e-5,
+        initializer_range=0.02,
+        use_cache=True,
+        bos_token_id=1,
+        eos_token_id=2,
+        apply_residual_connection_post_layernorm=False,
+        hidden_dropout=0.0,
+        attention_dropout=0.0,
+        n_head_kv=None,
+        alibi=False,
+        **kwargs,
+    ):
+        self.vocab_size = vocab_size
+        # Backward compatibility with n_embed kwarg
+        n_embed = kwargs.pop("n_embed", None)
+        self.hidden_size = hidden_size if n_embed is None else n_embed
+        self.n_layer = n_layer
+        self.n_head = n_head
+        self.layer_norm_epsilon = layer_norm_epsilon
+        self.initializer_range = initializer_range
+        self.use_cache = use_cache
+        self.apply_residual_connection_post_layernorm = apply_residual_connection_post_layernorm
+        self.hidden_dropout = hidden_dropout
+        self.attention_dropout = attention_dropout
+
+        self.bos_token_id = bos_token_id
+        self.eos_token_id = eos_token_id
+        self.n_head_kv = n_head if n_head_kv is None else n_head_kv
+        self.alibi = alibi
+
+        super().__init__(bos_token_id=bos_token_id, eos_token_id=eos_token_id, **kwargs)
+
+    @property
+    def head_dim(self):
+        return self.hidden_size // self.n_head
+
+    @property
+    def rotary(self):
+        return not self.alibi

configuration_falcon.py

@@ -1,152 +0,0 @@
-# coding=utf-8
-# Copyright 2023 the Falcon authors and HuggingFace Inc. team.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-""" Falcon configuration"""
-from transformers.configuration_utils import PretrainedConfig
-from transformers.utils import logging
-
-
-logger = logging.get_logger(__name__)
-
-FALCON_PRETRAINED_CONFIG_ARCHIVE_MAP = {
-    "tiiuae/falcon-40b": "https://huggingface.co/tiiuae/falcon-40b/resolve/main/config.json",
-    "tiiuae/falcon-7b": "https://huggingface.co/tiiuae/falcon-7b/resolve/main/config.json",
-}
-
-
-class FalconConfig(PretrainedConfig):
-    r"""
-    This is the configuration class to store the configuration of a [`FalconModel`]. It is used to instantiate a Falcon
-    model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
-    defaults will yield a similar configuration to that of the
-    [tiiuae/falcon-7b](https://huggingface.co/tiiuae/falcon-7b) architecture.
-
-    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
-    documentation from [`PretrainedConfig`] for more information.
-
-
-    Args:
-        vocab_size (`int`, *optional*, defaults to 65024):
-            Vocabulary size of the Falcon model. Defines the number of different tokens that can be represented by the
-            `inputs_ids` passed when calling [`FalconModel`]
-        hidden_size (`int`, *optional*, defaults to 4544):
-            Dimension of the hidden representations.
-        num_hidden_layers (`int`, *optional*, defaults to 32):
-            Number of hidden layers in the Transformer decoder.
-        num_attention_heads (`int`, *optional*, defaults to 71):
-            Number of attention heads for each attention layer in the Transformer encoder.
-        initializer_range (`float`, *optional*, defaults to 0.02):
-            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
-        use_cache (`bool`, *optional*, defaults to `True`):
-            Whether the model should return the last key/values attentions (not used by all models). Only relevant if
-            `config.is_decoder=True`.
-        layer_norm_epsilon (`float`, *optional*, defaults to 1e-5):
-            The epsilon used by the layer normalization layers.
-        hidden_dropout (`float`, *optional*, defaults to 0.0):
-            The dropout probability for MLP layers.
-        attention_dropout (`float`, *optional*, defaults to 0.0):
-            The dropout probability for attention layers.
-        num_kv_heads (`int`, *optional*):
-            Number of key-value heads to use per attention layer. If unset, defaults to the same value as
-            `num_attention_heads`.
-        alibi (`bool`, *optional*, defaults to `False`):
-            Whether to use ALiBi positional biases during self-attention.
-        new_decoder_architecture (`bool`, *optional*, defaults to `False`):
-            Whether to use the new (Falcon-40B) decoder architecture. If `True`, the `multi_query` and `parallel_attn`
-            arguments are ignored, as the new decoder always uses parallel attention.
-        multi_query (`bool`, *optional*, defaults to `True`):
-            Whether to use multi-query attention in the decoder. Ignored when `new_decoder_architecture` is `True`.
-        parallel_attn (`bool`, *optional*, defaults to `True`):
-            Whether to compute attention in parallel with the feedforward layer. If False, they are consecutive
-            instead, as in the original Transformer architecture. Ignored when `new_decoder_architecture` is `True`.
-        bias (`bool`, *optional*, defaults to `False`):
-            Whether to use bias on Linear layers.
-        bos_token_id (`int`, *optional*, defaults to 11):
-            The id of the "beginning-of-sequence" token.
-        eos_token_id (`int`, *optional*, defaults to 11):
-            The id of the "end-of-sequence" token.
-
-    Example:
-
-    ```python
-    >>> from transformers import FalconModel, FalconConfig
-
-    >>> # Initializing a small (2-layer) Falcon configuration
-    >>> configuration = FalconConfig(num_hidden_layers=2)
-
-    >>> # Initializing a model from the small configuration
-    >>> model = FalconModel(configuration)
-
-    >>> # Accessing the model configuration
-    >>> configuration = model.config
-    ```"""
-    model_type = "falcon"
-    keys_to_ignore_at_inference = ["past_key_values"]
-
-    def __init__(
-        self,
-        vocab_size=65024,
-        hidden_size=4544,
-        num_hidden_layers=32,
-        num_attention_heads=71,
-        layer_norm_epsilon=1e-5,
-        initializer_range=0.02,
-        use_cache=True,
-        hidden_dropout=0.0,
-        attention_dropout=0.0,
-        num_kv_heads=None,
-        alibi=False,
-        new_decoder_architecture=False,
-        multi_query=True,
-        parallel_attn=True,
-        bias=False,
-        bos_token_id=11,
-        eos_token_id=11,
-        **kwargs,
-    ):
-        logger.warning_once(
-            "\nWARNING: You are currently loading Falcon using legacy code contained in the model repository. Falcon has now been fully ported into the Hugging Face transformers library. "
-            "For the most up-to-date and high-performance version of the Falcon model code, please update to the latest version of transformers and then load the model "
-            "without the trust_remote_code=True argument.\n"
-        )
-        self.vocab_size = vocab_size
-        # Backward compatibility with n_embed kwarg
-        n_embed = kwargs.pop("n_embed", None)
-        self.hidden_size = hidden_size if n_embed is None else n_embed
-        self.num_hidden_layers = num_hidden_layers
-        self.num_attention_heads = num_attention_heads
-        self.layer_norm_epsilon = layer_norm_epsilon
-        self.initializer_range = initializer_range
-        self.use_cache = use_cache
-        self.hidden_dropout = hidden_dropout
-        self.attention_dropout = attention_dropout
-
-        self.bos_token_id = bos_token_id
-        self.eos_token_id = eos_token_id
-        self.num_kv_heads = num_attention_heads if num_kv_heads is None else num_kv_heads
-        self.alibi = alibi
-        self.new_decoder_architecture = new_decoder_architecture
-        self.multi_query = multi_query  # Ignored when new_decoder_architecture is True
-        self.parallel_attn = parallel_attn
-        self.bias = bias
-
-        super().__init__(bos_token_id=bos_token_id, eos_token_id=eos_token_id, **kwargs)
-
-    @property
-    def head_dim(self):
-        return self.hidden_size // self.num_attention_heads
-
-    @property
-    def rotary(self):
-        return not self.alibi

generation_config.json

@@ -1,6 +1,6 @@
 {
   "_from_model_config": true,
-  "bos_token_id": 11,
-  "eos_token_id": 11,
-  "transformers_version": "4.33.0.dev0"
-}
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.27.4"
+}

model-00001-of-00009.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:33d1225539f0a027f946e4d2a170317daee0c2a4e680e806f7fd876b0e4c012f
-size 9504757808

model-00002-of-00009.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:127ddcef2f52d868e9b0385a216d207ca35d6acce249c18e059b81bd528cd390
-size 9513146576

model-00003-of-00009.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:2abce4a0bb07ffd9050f22eb0311a18b00b1cce61910bd9ad5d288ea86551ffc
-size 9513146640

model-00004-of-00009.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:6e69018268ff0152d9430a4777e8906cb57ad1c229a4adfa39e88da5dbf7fe2e
-size 9513146640

model-00005-of-00009.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:36d3265bcfd8f1e5764c6542da075e5e443785846d31fe9f06f5176a99bbeeac
-size 9513146640

model-00006-of-00009.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:1082f1034b65909b1c4cb97a404740743a6b506902eb02ce3fbb8e44724bc400
-size 9513146640

model-00007-of-00009.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:aff64377e3e9c9e3c9f7b60059dfbee5655cb1cd4b2b7107fd049eb95180c946
-size 9513146640

model-00008-of-00009.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:9897d9458495ae577b5a90d7c10e02edcab6bf7bd154b1405ccb13b73c58fe15
-size 9513146640

model-00009-of-00009.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:751428d13d93719f003ec02426a2a75ed6d3e8d6754e640a1e8780734c0cd039
-size 7575212144

model.safetensors.index.json

@@ -1,491 +0,0 @@
-{
-    "metadata": {
-        "total_size": 83671941120
-    },
-    "weight_map": {
-        "lm_head.weight": "model-00009-of-00009.safetensors",
-        "transformer.h.0.ln_attn.bias": "model-00001-of-00009.safetensors",
-        "transformer.h.0.ln_attn.weight": "model-00001-of-00009.safetensors",
-        "transformer.h.0.ln_mlp.bias": "model-00001-of-00009.safetensors",
-        "transformer.h.0.ln_mlp.weight": "model-00001-of-00009.safetensors",
-        "transformer.h.0.mlp.dense_4h_to_h.weight": "model-00001-of-00009.safetensors",
-        "transformer.h.0.mlp.dense_h_to_4h.weight": "model-00001-of-00009.safetensors",
-        "transformer.h.0.self_attention.dense.weight": "model-00001-of-00009.safetensors",
-        "transformer.h.0.self_attention.query_key_value.weight": "model-00001-of-00009.safetensors",
-        "transformer.h.1.ln_attn.bias": "model-00001-of-00009.safetensors",
-        "transformer.h.1.ln_attn.weight": "model-00001-of-00009.safetensors",
-        "transformer.h.1.ln_mlp.bias": "model-00001-of-00009.safetensors",
-        "transformer.h.1.ln_mlp.weight": "model-00001-of-00009.safetensors",
-        "transformer.h.1.mlp.dense_4h_to_h.weight": "model-00001-of-00009.safetensors",
-        "transformer.h.1.mlp.dense_h_to_4h.weight": "model-00001-of-00009.safetensors",
-        "transformer.h.1.self_attention.dense.weight": "model-00001-of-00009.safetensors",
-        "transformer.h.1.self_attention.query_key_value.weight": "model-00001-of-00009.safetensors",
-        "transformer.h.10.ln_attn.bias": "model-00002-of-00009.safetensors",
-        "transformer.h.10.ln_attn.weight": "model-00002-of-00009.safetensors",
-        "transformer.h.10.ln_mlp.bias": "model-00002-of-00009.safetensors",
-        "transformer.h.10.ln_mlp.weight": "model-00002-of-00009.safetensors",
-        "transformer.h.10.mlp.dense_4h_to_h.weight": "model-00002-of-00009.safetensors",
-        "transformer.h.10.mlp.dense_h_to_4h.weight": "model-00002-of-00009.safetensors",
-        "transformer.h.10.self_attention.dense.weight": "model-00002-of-00009.safetensors",
-        "transformer.h.10.self_attention.query_key_value.weight": "model-00002-of-00009.safetensors",
-        "transformer.h.11.ln_attn.bias": "model-00002-of-00009.safetensors",
-        "transformer.h.11.ln_attn.weight": "model-00002-of-00009.safetensors",
-        "transformer.h.11.ln_mlp.bias": "model-00002-of-00009.safetensors",
-        "transformer.h.11.ln_mlp.weight": "model-00002-of-00009.safetensors",
-        "transformer.h.11.mlp.dense_4h_to_h.weight": "model-00002-of-00009.safetensors",
-        "transformer.h.11.mlp.dense_h_to_4h.weight": "model-00002-of-00009.safetensors",
-        "transformer.h.11.self_attention.dense.weight": "model-00002-of-00009.safetensors",
-        "transformer.h.11.self_attention.query_key_value.weight": "model-00002-of-00009.safetensors",
-        "transformer.h.12.ln_attn.bias": "model-00002-of-00009.safetensors",
-        "transformer.h.12.ln_attn.weight": "model-00002-of-00009.safetensors",
-        "transformer.h.12.ln_mlp.bias": "model-00002-of-00009.safetensors",
-        "transformer.h.12.ln_mlp.weight": "model-00002-of-00009.safetensors",
-        "transformer.h.12.mlp.dense_4h_to_h.weight": "model-00002-of-00009.safetensors",
-        "transformer.h.12.mlp.dense_h_to_4h.weight": "model-00002-of-00009.safetensors",
-        "transformer.h.12.self_attention.dense.weight": "model-00002-of-00009.safetensors",
-        "transformer.h.12.self_attention.query_key_value.weight": "model-00002-of-00009.safetensors",
-        "transformer.h.13.ln_attn.bias": "model-00002-of-00009.safetensors",
-        "transformer.h.13.ln_attn.weight": "model-00002-of-00009.safetensors",
-        "transformer.h.13.ln_mlp.bias": "model-00002-of-00009.safetensors",
-        "transformer.h.13.ln_mlp.weight": "model-00002-of-00009.safetensors",
-        "transformer.h.13.mlp.dense_4h_to_h.weight": "model-00003-of-00009.safetensors",
-        "transformer.h.13.mlp.dense_h_to_4h.weight": "model-00003-of-00009.safetensors",
-        "transformer.h.13.self_attention.dense.weight": "model-00002-of-00009.safetensors",
-        "transformer.h.13.self_attention.query_key_value.weight": "model-00002-of-00009.safetensors",
-        "transformer.h.14.ln_attn.bias": "model-00003-of-00009.safetensors",
-        "transformer.h.14.ln_attn.weight": "model-00003-of-00009.safetensors",
-        "transformer.h.14.ln_mlp.bias": "model-00003-of-00009.safetensors",
-        "transformer.h.14.ln_mlp.weight": "model-00003-of-00009.safetensors",
-        "transformer.h.14.mlp.dense_4h_to_h.weight": "model-00003-of-00009.safetensors",
-        "transformer.h.14.mlp.dense_h_to_4h.weight": "model-00003-of-00009.safetensors",
-        "transformer.h.14.self_attention.dense.weight": "model-00003-of-00009.safetensors",
-        "transformer.h.14.self_attention.query_key_value.weight": "model-00003-of-00009.safetensors",
-        "transformer.h.15.ln_attn.bias": "model-00003-of-00009.safetensors",
-        "transformer.h.15.ln_attn.weight": "model-00003-of-00009.safetensors",
-        "transformer.h.15.ln_mlp.bias": "model-00003-of-00009.safetensors",
-        "transformer.h.15.ln_mlp.weight": "model-00003-of-00009.safetensors",
-        "transformer.h.15.mlp.dense_4h_to_h.weight": "model-00003-of-00009.safetensors",
-        "transformer.h.15.mlp.dense_h_to_4h.weight": "model-00003-of-00009.safetensors",
-        "transformer.h.15.self_attention.dense.weight": "model-00003-of-00009.safetensors",
-        "transformer.h.15.self_attention.query_key_value.weight": "model-00003-of-00009.safetensors",
-        "transformer.h.16.ln_attn.bias": "model-00003-of-00009.safetensors",
-        "transformer.h.16.ln_attn.weight": "model-00003-of-00009.safetensors",
-        "transformer.h.16.ln_mlp.bias": "model-00003-of-00009.safetensors",
-        "transformer.h.16.ln_mlp.weight": "model-00003-of-00009.safetensors",
-        "transformer.h.16.mlp.dense_4h_to_h.weight": "model-00003-of-00009.safetensors",
-        "transformer.h.16.mlp.dense_h_to_4h.weight": "model-00003-of-00009.safetensors",
-        "transformer.h.16.self_attention.dense.weight": "model-00003-of-00009.safetensors",
-        "transformer.h.16.self_attention.query_key_value.weight": "model-00003-of-00009.safetensors",
-        "transformer.h.17.ln_attn.bias": "model-00003-of-00009.safetensors",
-        "transformer.h.17.ln_attn.weight": "model-00003-of-00009.safetensors",
-        "transformer.h.17.ln_mlp.bias": "model-00003-of-00009.safetensors",
-        "transformer.h.17.ln_mlp.weight": "model-00003-of-00009.safetensors",
-        "transformer.h.17.mlp.dense_4h_to_h.weight": "model-00003-of-00009.safetensors",
-        "transformer.h.17.mlp.dense_h_to_4h.weight": "model-00003-of-00009.safetensors",
-        "transformer.h.17.self_attention.dense.weight": "model-00003-of-00009.safetensors",
-        "transformer.h.17.self_attention.query_key_value.weight": "model-00003-of-00009.safetensors",
-        "transformer.h.18.ln_attn.bias": "model-00003-of-00009.safetensors",
-        "transformer.h.18.ln_attn.weight": "model-00003-of-00009.safetensors",
-        "transformer.h.18.ln_mlp.bias": "model-00003-of-00009.safetensors",
-        "transformer.h.18.ln_mlp.weight": "model-00003-of-00009.safetensors",
-        "transformer.h.18.mlp.dense_4h_to_h.weight": "model-00003-of-00009.safetensors",
-        "transformer.h.18.mlp.dense_h_to_4h.weight": "model-00003-of-00009.safetensors",
-        "transformer.h.18.self_attention.dense.weight": "model-00003-of-00009.safetensors",
-        "transformer.h.18.self_attention.query_key_value.weight": "model-00003-of-00009.safetensors",
-        "transformer.h.19.ln_attn.bias": "model-00003-of-00009.safetensors",
-        "transformer.h.19.ln_attn.weight": "model-00003-of-00009.safetensors",
-        "transformer.h.19.ln_mlp.bias": "model-00003-of-00009.safetensors",
-        "transformer.h.19.ln_mlp.weight": "model-00003-of-00009.safetensors",
-        "transformer.h.19.mlp.dense_4h_to_h.weight": "model-00003-of-00009.safetensors",
-        "transformer.h.19.mlp.dense_h_to_4h.weight": "model-00003-of-00009.safetensors",
-        "transformer.h.19.self_attention.dense.weight": "model-00003-of-00009.safetensors",
-        "transformer.h.19.self_attention.query_key_value.weight": "model-00003-of-00009.safetensors",
-        "transformer.h.2.ln_attn.bias": "model-00001-of-00009.safetensors",
-        "transformer.h.2.ln_attn.weight": "model-00001-of-00009.safetensors",
-        "transformer.h.2.ln_mlp.bias": "model-00001-of-00009.safetensors",
-        "transformer.h.2.ln_mlp.weight": "model-00001-of-00009.safetensors",
-        "transformer.h.2.mlp.dense_4h_to_h.weight": "model-00001-of-00009.safetensors",
-        "transformer.h.2.mlp.dense_h_to_4h.weight": "model-00001-of-00009.safetensors",
-        "transformer.h.2.self_attention.dense.weight": "model-00001-of-00009.safetensors",
-        "transformer.h.2.self_attention.query_key_value.weight": "model-00001-of-00009.safetensors",
-        "transformer.h.20.ln_attn.bias": "model-00003-of-00009.safetensors",
-        "transformer.h.20.ln_attn.weight": "model-00003-of-00009.safetensors",
-        "transformer.h.20.ln_mlp.bias": "model-00003-of-00009.safetensors",
-        "transformer.h.20.ln_mlp.weight": "model-00003-of-00009.safetensors",
-        "transformer.h.20.mlp.dense_4h_to_h.weight": "model-00004-of-00009.safetensors",
-        "transformer.h.20.mlp.dense_h_to_4h.weight": "model-00004-of-00009.safetensors",
-        "transformer.h.20.self_attention.dense.weight": "model-00003-of-00009.safetensors",
-        "transformer.h.20.self_attention.query_key_value.weight": "model-00003-of-00009.safetensors",
-        "transformer.h.21.ln_attn.bias": "model-00004-of-00009.safetensors",
-        "transformer.h.21.ln_attn.weight": "model-00004-of-00009.safetensors",
-        "transformer.h.21.ln_mlp.bias": "model-00004-of-00009.safetensors",
-        "transformer.h.21.ln_mlp.weight": "model-00004-of-00009.safetensors",
-        "transformer.h.21.mlp.dense_4h_to_h.weight": "model-00004-of-00009.safetensors",
-        "transformer.h.21.mlp.dense_h_to_4h.weight": "model-00004-of-00009.safetensors",
-        "transformer.h.21.self_attention.dense.weight": "model-00004-of-00009.safetensors",
-        "transformer.h.21.self_attention.query_key_value.weight": "model-00004-of-00009.safetensors",
-        "transformer.h.22.ln_attn.bias": "model-00004-of-00009.safetensors",
-        "transformer.h.22.ln_attn.weight": "model-00004-of-00009.safetensors",
-        "transformer.h.22.ln_mlp.bias": "model-00004-of-00009.safetensors",
-        "transformer.h.22.ln_mlp.weight": "model-00004-of-00009.safetensors",
-        "transformer.h.22.mlp.dense_4h_to_h.weight": "model-00004-of-00009.safetensors",
-        "transformer.h.22.mlp.dense_h_to_4h.weight": "model-00004-of-00009.safetensors",
-        "transformer.h.22.self_attention.dense.weight": "model-00004-of-00009.safetensors",
-        "transformer.h.22.self_attention.query_key_value.weight": "model-00004-of-00009.safetensors",
-        "transformer.h.23.ln_attn.bias": "model-00004-of-00009.safetensors",
-        "transformer.h.23.ln_attn.weight": "model-00004-of-00009.safetensors",
-        "transformer.h.23.ln_mlp.bias": "model-00004-of-00009.safetensors",
-        "transformer.h.23.ln_mlp.weight": "model-00004-of-00009.safetensors",
-        "transformer.h.23.mlp.dense_4h_to_h.weight": "model-00004-of-00009.safetensors",
-        "transformer.h.23.mlp.dense_h_to_4h.weight": "model-00004-of-00009.safetensors",
-        "transformer.h.23.self_attention.dense.weight": "model-00004-of-00009.safetensors",
-        "transformer.h.23.self_attention.query_key_value.weight": "model-00004-of-00009.safetensors",
-        "transformer.h.24.ln_attn.bias": "model-00004-of-00009.safetensors",
-        "transformer.h.24.ln_attn.weight": "model-00004-of-00009.safetensors",
-        "transformer.h.24.ln_mlp.bias": "model-00004-of-00009.safetensors",
-        "transformer.h.24.ln_mlp.weight": "model-00004-of-00009.safetensors",
-        "transformer.h.24.mlp.dense_4h_to_h.weight": "model-00004-of-00009.safetensors",
-        "transformer.h.24.mlp.dense_h_to_4h.weight": "model-00004-of-00009.safetensors",
-        "transformer.h.24.self_attention.dense.weight": "model-00004-of-00009.safetensors",
-        "transformer.h.24.self_attention.query_key_value.weight": "model-00004-of-00009.safetensors",
-        "transformer.h.25.ln_attn.bias": "model-00004-of-00009.safetensors",
-        "transformer.h.25.ln_attn.weight": "model-00004-of-00009.safetensors",
-        "transformer.h.25.ln_mlp.bias": "model-00004-of-00009.safetensors",
-        "transformer.h.25.ln_mlp.weight": "model-00004-of-00009.safetensors",
-        "transformer.h.25.mlp.dense_4h_to_h.weight": "model-00004-of-00009.safetensors",
-        "transformer.h.25.mlp.dense_h_to_4h.weight": "model-00004-of-00009.safetensors",
-        "transformer.h.25.self_attention.dense.weight": "model-00004-of-00009.safetensors",
-        "transformer.h.25.self_attention.query_key_value.weight": "model-00004-of-00009.safetensors",
-        "transformer.h.26.ln_attn.bias": "model-00004-of-00009.safetensors",
-        "transformer.h.26.ln_attn.weight": "model-00004-of-00009.safetensors",
-        "transformer.h.26.ln_mlp.bias": "model-00004-of-00009.safetensors",
-        "transformer.h.26.ln_mlp.weight": "model-00004-of-00009.safetensors",
-        "transformer.h.26.mlp.dense_4h_to_h.weight": "model-00004-of-00009.safetensors",
-        "transformer.h.26.mlp.dense_h_to_4h.weight": "model-00004-of-00009.safetensors",
-        "transformer.h.26.self_attention.dense.weight": "model-00004-of-00009.safetensors",
-        "transformer.h.26.self_attention.query_key_value.weight": "model-00004-of-00009.safetensors",
-        "transformer.h.27.ln_attn.bias": "model-00004-of-00009.safetensors",
-        "transformer.h.27.ln_attn.weight": "model-00004-of-00009.safetensors",
-        "transformer.h.27.ln_mlp.bias": "model-00004-of-00009.safetensors",
-        "transformer.h.27.ln_mlp.weight": "model-00004-of-00009.safetensors",
-        "transformer.h.27.mlp.dense_4h_to_h.weight": "model-00005-of-00009.safetensors",
-        "transformer.h.27.mlp.dense_h_to_4h.weight": "model-00005-of-00009.safetensors",
-        "transformer.h.27.self_attention.dense.weight": "model-00004-of-00009.safetensors",
-        "transformer.h.27.self_attention.query_key_value.weight": "model-00004-of-00009.safetensors",
-        "transformer.h.28.ln_attn.bias": "model-00005-of-00009.safetensors",
-        "transformer.h.28.ln_attn.weight": "model-00005-of-00009.safetensors",
-        "transformer.h.28.ln_mlp.bias": "model-00005-of-00009.safetensors",
-        "transformer.h.28.ln_mlp.weight": "model-00005-of-00009.safetensors",
-        "transformer.h.28.mlp.dense_4h_to_h.weight": "model-00005-of-00009.safetensors",
-        "transformer.h.28.mlp.dense_h_to_4h.weight": "model-00005-of-00009.safetensors",
-        "transformer.h.28.self_attention.dense.weight": "model-00005-of-00009.safetensors",
-        "transformer.h.28.self_attention.query_key_value.weight": "model-00005-of-00009.safetensors",
-        "transformer.h.29.ln_attn.bias": "model-00005-of-00009.safetensors",
-        "transformer.h.29.ln_attn.weight": "model-00005-of-00009.safetensors",
-        "transformer.h.29.ln_mlp.bias": "model-00005-of-00009.safetensors",
-        "transformer.h.29.ln_mlp.weight": "model-00005-of-00009.safetensors",
-        "transformer.h.29.mlp.dense_4h_to_h.weight": "model-00005-of-00009.safetensors",
-        "transformer.h.29.mlp.dense_h_to_4h.weight": "model-00005-of-00009.safetensors",
-        "transformer.h.29.self_attention.dense.weight": "model-00005-of-00009.safetensors",
-        "transformer.h.29.self_attention.query_key_value.weight": "model-00005-of-00009.safetensors",
-        "transformer.h.3.ln_attn.bias": "model-00001-of-00009.safetensors",
-        "transformer.h.3.ln_attn.weight": "model-00001-of-00009.safetensors",
-        "transformer.h.3.ln_mlp.bias": "model-00001-of-00009.safetensors",
-        "transformer.h.3.ln_mlp.weight": "model-00001-of-00009.safetensors",
-        "transformer.h.3.mlp.dense_4h_to_h.weight": "model-00001-of-00009.safetensors",
-        "transformer.h.3.mlp.dense_h_to_4h.weight": "model-00001-of-00009.safetensors",
-        "transformer.h.3.self_attention.dense.weight": "model-00001-of-00009.safetensors",
-        "transformer.h.3.self_attention.query_key_value.weight": "model-00001-of-00009.safetensors",
-        "transformer.h.30.ln_attn.bias": "model-00005-of-00009.safetensors",
-        "transformer.h.30.ln_attn.weight": "model-00005-of-00009.safetensors",
-        "transformer.h.30.ln_mlp.bias": "model-00005-of-00009.safetensors",
-        "transformer.h.30.ln_mlp.weight": "model-00005-of-00009.safetensors",
-        "transformer.h.30.mlp.dense_4h_to_h.weight": "model-00005-of-00009.safetensors",
-        "transformer.h.30.mlp.dense_h_to_4h.weight": "model-00005-of-00009.safetensors",
-        "transformer.h.30.self_attention.dense.weight": "model-00005-of-00009.safetensors",
-        "transformer.h.30.self_attention.query_key_value.weight": "model-00005-of-00009.safetensors",
-        "transformer.h.31.ln_attn.bias": "model-00005-of-00009.safetensors",
-        "transformer.h.31.ln_attn.weight": "model-00005-of-00009.safetensors",
-        "transformer.h.31.ln_mlp.bias": "model-00005-of-00009.safetensors",
-        "transformer.h.31.ln_mlp.weight": "model-00005-of-00009.safetensors",
-        "transformer.h.31.mlp.dense_4h_to_h.weight": "model-00005-of-00009.safetensors",
-        "transformer.h.31.mlp.dense_h_to_4h.weight": "model-00005-of-00009.safetensors",
-        "transformer.h.31.self_attention.dense.weight": "model-00005-of-00009.safetensors",
-        "transformer.h.31.self_attention.query_key_value.weight": "model-00005-of-00009.safetensors",
-        "transformer.h.32.ln_attn.bias": "model-00005-of-00009.safetensors",
-        "transformer.h.32.ln_attn.weight": "model-00005-of-00009.safetensors",
-        "transformer.h.32.ln_mlp.bias": "model-00005-of-00009.safetensors",
-        "transformer.h.32.ln_mlp.weight": "model-00005-of-00009.safetensors",
-        "transformer.h.32.mlp.dense_4h_to_h.weight": "model-00005-of-00009.safetensors",
-        "transformer.h.32.mlp.dense_h_to_4h.weight": "model-00005-of-00009.safetensors",
-        "transformer.h.32.self_attention.dense.weight": "model-00005-of-00009.safetensors",
-        "transformer.h.32.self_attention.query_key_value.weight": "model-00005-of-00009.safetensors",
-        "transformer.h.33.ln_attn.bias": "model-00005-of-00009.safetensors",
-        "transformer.h.33.ln_attn.weight": "model-00005-of-00009.safetensors",
-        "transformer.h.33.ln_mlp.bias": "model-00005-of-00009.safetensors",
-        "transformer.h.33.ln_mlp.weight": "model-00005-of-00009.safetensors",
-        "transformer.h.33.mlp.dense_4h_to_h.weight": "model-00005-of-00009.safetensors",
-        "transformer.h.33.mlp.dense_h_to_4h.weight": "model-00005-of-00009.safetensors",
-        "transformer.h.33.self_attention.dense.weight": "model-00005-of-00009.safetensors",
-        "transformer.h.33.self_attention.query_key_value.weight": "model-00005-of-00009.safetensors",
-        "transformer.h.34.ln_attn.bias": "model-00005-of-00009.safetensors",
-        "transformer.h.34.ln_attn.weight": "model-00005-of-00009.safetensors",
-        "transformer.h.34.ln_mlp.bias": "model-00005-of-00009.safetensors",
-        "transformer.h.34.ln_mlp.weight": "model-00005-of-00009.safetensors",
-        "transformer.h.34.mlp.dense_4h_to_h.weight": "model-00006-of-00009.safetensors",
-        "transformer.h.34.mlp.dense_h_to_4h.weight": "model-00006-of-00009.safetensors",
-        "transformer.h.34.self_attention.dense.weight": "model-00005-of-00009.safetensors",
-        "transformer.h.34.self_attention.query_key_value.weight": "model-00005-of-00009.safetensors",
-        "transformer.h.35.ln_attn.bias": "model-00006-of-00009.safetensors",
-        "transformer.h.35.ln_attn.weight": "model-00006-of-00009.safetensors",
-        "transformer.h.35.ln_mlp.bias": "model-00006-of-00009.safetensors",
-        "transformer.h.35.ln_mlp.weight": "model-00006-of-00009.safetensors",
-        "transformer.h.35.mlp.dense_4h_to_h.weight": "model-00006-of-00009.safetensors",
-        "transformer.h.35.mlp.dense_h_to_4h.weight": "model-00006-of-00009.safetensors",
-        "transformer.h.35.self_attention.dense.weight": "model-00006-of-00009.safetensors",
-        "transformer.h.35.self_attention.query_key_value.weight": "model-00006-of-00009.safetensors",
-        "transformer.h.36.ln_attn.bias": "model-00006-of-00009.safetensors",
-        "transformer.h.36.ln_attn.weight": "model-00006-of-00009.safetensors",
-        "transformer.h.36.ln_mlp.bias": "model-00006-of-00009.safetensors",
-        "transformer.h.36.ln_mlp.weight": "model-00006-of-00009.safetensors",
-        "transformer.h.36.mlp.dense_4h_to_h.weight": "model-00006-of-00009.safetensors",
-        "transformer.h.36.mlp.dense_h_to_4h.weight": "model-00006-of-00009.safetensors",
-        "transformer.h.36.self_attention.dense.weight": "model-00006-of-00009.safetensors",
-        "transformer.h.36.self_attention.query_key_value.weight": "model-00006-of-00009.safetensors",
-        "transformer.h.37.ln_attn.bias": "model-00006-of-00009.safetensors",
-        "transformer.h.37.ln_attn.weight": "model-00006-of-00009.safetensors",
-        "transformer.h.37.ln_mlp.bias": "model-00006-of-00009.safetensors",
-        "transformer.h.37.ln_mlp.weight": "model-00006-of-00009.safetensors",
-        "transformer.h.37.mlp.dense_4h_to_h.weight": "model-00006-of-00009.safetensors",
-        "transformer.h.37.mlp.dense_h_to_4h.weight": "model-00006-of-00009.safetensors",
-        "transformer.h.37.self_attention.dense.weight": "model-00006-of-00009.safetensors",
-        "transformer.h.37.self_attention.query_key_value.weight": "model-00006-of-00009.safetensors",
-        "transformer.h.38.ln_attn.bias": "model-00006-of-00009.safetensors",
-        "transformer.h.38.ln_attn.weight": "model-00006-of-00009.safetensors",
-        "transformer.h.38.ln_mlp.bias": "model-00006-of-00009.safetensors",
-        "transformer.h.38.ln_mlp.weight": "model-00006-of-00009.safetensors",
-        "transformer.h.38.mlp.dense_4h_to_h.weight": "model-00006-of-00009.safetensors",
-        "transformer.h.38.mlp.dense_h_to_4h.weight": "model-00006-of-00009.safetensors",
-        "transformer.h.38.self_attention.dense.weight": "model-00006-of-00009.safetensors",
-        "transformer.h.38.self_attention.query_key_value.weight": "model-00006-of-00009.safetensors",
-        "transformer.h.39.ln_attn.bias": "model-00006-of-00009.safetensors",
-        "transformer.h.39.ln_attn.weight": "model-00006-of-00009.safetensors",
-        "transformer.h.39.ln_mlp.bias": "model-00006-of-00009.safetensors",
-        "transformer.h.39.ln_mlp.weight": "model-00006-of-00009.safetensors",
-        "transformer.h.39.mlp.dense_4h_to_h.weight": "model-00006-of-00009.safetensors",
-        "transformer.h.39.mlp.dense_h_to_4h.weight": "model-00006-of-00009.safetensors",
-        "transformer.h.39.self_attention.dense.weight": "model-00006-of-00009.safetensors",
-        "transformer.h.39.self_attention.query_key_value.weight": "model-00006-of-00009.safetensors",
-        "transformer.h.4.ln_attn.bias": "model-00001-of-00009.safetensors",
-        "transformer.h.4.ln_attn.weight": "model-00001-of-00009.safetensors",
-        "transformer.h.4.ln_mlp.bias": "model-00001-of-00009.safetensors",
-        "transformer.h.4.ln_mlp.weight": "model-00001-of-00009.safetensors",
-        "transformer.h.4.mlp.dense_4h_to_h.weight": "model-00001-of-00009.safetensors",
-        "transformer.h.4.mlp.dense_h_to_4h.weight": "model-00001-of-00009.safetensors",
-        "transformer.h.4.self_attention.dense.weight": "model-00001-of-00009.safetensors",
-        "transformer.h.4.self_attention.query_key_value.weight": "model-00001-of-00009.safetensors",
-        "transformer.h.40.ln_attn.bias": "model-00006-of-00009.safetensors",
-        "transformer.h.40.ln_attn.weight": "model-00006-of-00009.safetensors",
-        "transformer.h.40.ln_mlp.bias": "model-00006-of-00009.safetensors",
-        "transformer.h.40.ln_mlp.weight": "model-00006-of-00009.safetensors",
-        "transformer.h.40.mlp.dense_4h_to_h.weight": "model-00006-of-00009.safetensors",
-        "transformer.h.40.mlp.dense_h_to_4h.weight": "model-00006-of-00009.safetensors",
-        "transformer.h.40.self_attention.dense.weight": "model-00006-of-00009.safetensors",
-        "transformer.h.40.self_attention.query_key_value.weight": "model-00006-of-00009.safetensors",
-        "transformer.h.41.ln_attn.bias": "model-00006-of-00009.safetensors",
-        "transformer.h.41.ln_attn.weight": "model-00006-of-00009.safetensors",
-        "transformer.h.41.ln_mlp.bias": "model-00006-of-00009.safetensors",
-        "transformer.h.41.ln_mlp.weight": "model-00006-of-00009.safetensors",
-        "transformer.h.41.mlp.dense_4h_to_h.weight": "model-00007-of-00009.safetensors",
-        "transformer.h.41.mlp.dense_h_to_4h.weight": "model-00007-of-00009.safetensors",
-        "transformer.h.41.self_attention.dense.weight": "model-00006-of-00009.safetensors",
-        "transformer.h.41.self_attention.query_key_value.weight": "model-00006-of-00009.safetensors",
-        "transformer.h.42.ln_attn.bias": "model-00007-of-00009.safetensors",
-        "transformer.h.42.ln_attn.weight": "model-00007-of-00009.safetensors",
-        "transformer.h.42.ln_mlp.bias": "model-00007-of-00009.safetensors",
-        "transformer.h.42.ln_mlp.weight": "model-00007-of-00009.safetensors",
-        "transformer.h.42.mlp.dense_4h_to_h.weight": "model-00007-of-00009.safetensors",
-        "transformer.h.42.mlp.dense_h_to_4h.weight": "model-00007-of-00009.safetensors",
-        "transformer.h.42.self_attention.dense.weight": "model-00007-of-00009.safetensors",
-        "transformer.h.42.self_attention.query_key_value.weight": "model-00007-of-00009.safetensors",
-        "transformer.h.43.ln_attn.bias": "model-00007-of-00009.safetensors",
-        "transformer.h.43.ln_attn.weight": "model-00007-of-00009.safetensors",
-        "transformer.h.43.ln_mlp.bias": "model-00007-of-00009.safetensors",
-        "transformer.h.43.ln_mlp.weight": "model-00007-of-00009.safetensors",
-        "transformer.h.43.mlp.dense_4h_to_h.weight": "model-00007-of-00009.safetensors",
-        "transformer.h.43.mlp.dense_h_to_4h.weight": "model-00007-of-00009.safetensors",
-        "transformer.h.43.self_attention.dense.weight": "model-00007-of-00009.safetensors",
-        "transformer.h.43.self_attention.query_key_value.weight": "model-00007-of-00009.safetensors",
-        "transformer.h.44.ln_attn.bias": "model-00007-of-00009.safetensors",
-        "transformer.h.44.ln_attn.weight": "model-00007-of-00009.safetensors",
-        "transformer.h.44.ln_mlp.bias": "model-00007-of-00009.safetensors",
-        "transformer.h.44.ln_mlp.weight": "model-00007-of-00009.safetensors",
-        "transformer.h.44.mlp.dense_4h_to_h.weight": "model-00007-of-00009.safetensors",
-        "transformer.h.44.mlp.dense_h_to_4h.weight": "model-00007-of-00009.safetensors",
-        "transformer.h.44.self_attention.dense.weight": "model-00007-of-00009.safetensors",
-        "transformer.h.44.self_attention.query_key_value.weight": "model-00007-of-00009.safetensors",
-        "transformer.h.45.ln_attn.bias": "model-00007-of-00009.safetensors",
-        "transformer.h.45.ln_attn.weight": "model-00007-of-00009.safetensors",
-        "transformer.h.45.ln_mlp.bias": "model-00007-of-00009.safetensors",
-        "transformer.h.45.ln_mlp.weight": "model-00007-of-00009.safetensors",
-        "transformer.h.45.mlp.dense_4h_to_h.weight": "model-00007-of-00009.safetensors",
-        "transformer.h.45.mlp.dense_h_to_4h.weight": "model-00007-of-00009.safetensors",
-        "transformer.h.45.self_attention.dense.weight": "model-00007-of-00009.safetensors",
-        "transformer.h.45.self_attention.query_key_value.weight": "model-00007-of-00009.safetensors",
-        "transformer.h.46.ln_attn.bias": "model-00007-of-00009.safetensors",
-        "transformer.h.46.ln_attn.weight": "model-00007-of-00009.safetensors",
-        "transformer.h.46.ln_mlp.bias": "model-00007-of-00009.safetensors",
-        "transformer.h.46.ln_mlp.weight": "model-00007-of-00009.safetensors",
-        "transformer.h.46.mlp.dense_4h_to_h.weight": "model-00007-of-00009.safetensors",
-        "transformer.h.46.mlp.dense_h_to_4h.weight": "model-00007-of-00009.safetensors",
-        "transformer.h.46.self_attention.dense.weight": "model-00007-of-00009.safetensors",
-        "transformer.h.46.self_attention.query_key_value.weight": "model-00007-of-00009.safetensors",
-        "transformer.h.47.ln_attn.bias": "model-00007-of-00009.safetensors",
-        "transformer.h.47.ln_attn.weight": "model-00007-of-00009.safetensors",
-        "transformer.h.47.ln_mlp.bias": "model-00007-of-00009.safetensors",
-        "transformer.h.47.ln_mlp.weight": "model-00007-of-00009.safetensors",
-        "transformer.h.47.mlp.dense_4h_to_h.weight": "model-00007-of-00009.safetensors",
-        "transformer.h.47.mlp.dense_h_to_4h.weight": "model-00007-of-00009.safetensors",
-        "transformer.h.47.self_attention.dense.weight": "model-00007-of-00009.safetensors",
-        "transformer.h.47.self_attention.query_key_value.weight": "model-00007-of-00009.safetensors",
-        "transformer.h.48.ln_attn.bias": "model-00007-of-00009.safetensors",
-        "transformer.h.48.ln_attn.weight": "model-00007-of-00009.safetensors",
-        "transformer.h.48.ln_mlp.bias": "model-00007-of-00009.safetensors",
-        "transformer.h.48.ln_mlp.weight": "model-00007-of-00009.safetensors",
-        "transformer.h.48.mlp.dense_4h_to_h.weight": "model-00008-of-00009.safetensors",
-        "transformer.h.48.mlp.dense_h_to_4h.weight": "model-00008-of-00009.safetensors",
-        "transformer.h.48.self_attention.dense.weight": "model-00007-of-00009.safetensors",
-        "transformer.h.48.self_attention.query_key_value.weight": "model-00007-of-00009.safetensors",
-        "transformer.h.49.ln_attn.bias": "model-00008-of-00009.safetensors",
-        "transformer.h.49.ln_attn.weight": "model-00008-of-00009.safetensors",
-        "transformer.h.49.ln_mlp.bias": "model-00008-of-00009.safetensors",
-        "transformer.h.49.ln_mlp.weight": "model-00008-of-00009.safetensors",
-        "transformer.h.49.mlp.dense_4h_to_h.weight": "model-00008-of-00009.safetensors",
-        "transformer.h.49.mlp.dense_h_to_4h.weight": "model-00008-of-00009.safetensors",
-        "transformer.h.49.self_attention.dense.weight": "model-00008-of-00009.safetensors",
-        "transformer.h.49.self_attention.query_key_value.weight": "model-00008-of-00009.safetensors",
-        "transformer.h.5.ln_attn.bias": "model-00001-of-00009.safetensors",
-        "transformer.h.5.ln_attn.weight": "model-00001-of-00009.safetensors",
-        "transformer.h.5.ln_mlp.bias": "model-00001-of-00009.safetensors",
-        "transformer.h.5.ln_mlp.weight": "model-00001-of-00009.safetensors",
-        "transformer.h.5.mlp.dense_4h_to_h.weight": "model-00001-of-00009.safetensors",
-        "transformer.h.5.mlp.dense_h_to_4h.weight": "model-00001-of-00009.safetensors",
-        "transformer.h.5.self_attention.dense.weight": "model-00001-of-00009.safetensors",
-        "transformer.h.5.self_attention.query_key_value.weight": "model-00001-of-00009.safetensors",
-        "transformer.h.50.ln_attn.bias": "model-00008-of-00009.safetensors",
-        "transformer.h.50.ln_attn.weight": "model-00008-of-00009.safetensors",
-        "transformer.h.50.ln_mlp.bias": "model-00008-of-00009.safetensors",
-        "transformer.h.50.ln_mlp.weight": "model-00008-of-00009.safetensors",
-        "transformer.h.50.mlp.dense_4h_to_h.weight": "model-00008-of-00009.safetensors",
-        "transformer.h.50.mlp.dense_h_to_4h.weight": "model-00008-of-00009.safetensors",
-        "transformer.h.50.self_attention.dense.weight": "model-00008-of-00009.safetensors",
-        "transformer.h.50.self_attention.query_key_value.weight": "model-00008-of-00009.safetensors",
-        "transformer.h.51.ln_attn.bias": "model-00008-of-00009.safetensors",
-        "transformer.h.51.ln_attn.weight": "model-00008-of-00009.safetensors",
-        "transformer.h.51.ln_mlp.bias": "model-00008-of-00009.safetensors",
-        "transformer.h.51.ln_mlp.weight": "model-00008-of-00009.safetensors",
-        "transformer.h.51.mlp.dense_4h_to_h.weight": "model-00008-of-00009.safetensors",
-        "transformer.h.51.mlp.dense_h_to_4h.weight": "model-00008-of-00009.safetensors",
-        "transformer.h.51.self_attention.dense.weight": "model-00008-of-00009.safetensors",
-        "transformer.h.51.self_attention.query_key_value.weight": "model-00008-of-00009.safetensors",
-        "transformer.h.52.ln_attn.bias": "model-00008-of-00009.safetensors",
-        "transformer.h.52.ln_attn.weight": "model-00008-of-00009.safetensors",
-        "transformer.h.52.ln_mlp.bias": "model-00008-of-00009.safetensors",
-        "transformer.h.52.ln_mlp.weight": "model-00008-of-00009.safetensors",
-        "transformer.h.52.mlp.dense_4h_to_h.weight": "model-00008-of-00009.safetensors",
-        "transformer.h.52.mlp.dense_h_to_4h.weight": "model-00008-of-00009.safetensors",
-        "transformer.h.52.self_attention.dense.weight": "model-00008-of-00009.safetensors",
-        "transformer.h.52.self_attention.query_key_value.weight": "model-00008-of-00009.safetensors",
-        "transformer.h.53.ln_attn.bias": "model-00008-of-00009.safetensors",
-        "transformer.h.53.ln_attn.weight": "model-00008-of-00009.safetensors",
-        "transformer.h.53.ln_mlp.bias": "model-00008-of-00009.safetensors",
-        "transformer.h.53.ln_mlp.weight": "model-00008-of-00009.safetensors",
-        "transformer.h.53.mlp.dense_4h_to_h.weight": "model-00008-of-00009.safetensors",
-        "transformer.h.53.mlp.dense_h_to_4h.weight": "model-00008-of-00009.safetensors",
-        "transformer.h.53.self_attention.dense.weight": "model-00008-of-00009.safetensors",
-        "transformer.h.53.self_attention.query_key_value.weight": "model-00008-of-00009.safetensors",
-        "transformer.h.54.ln_attn.bias": "model-00008-of-00009.safetensors",
-        "transformer.h.54.ln_attn.weight": "model-00008-of-00009.safetensors",
-        "transformer.h.54.ln_mlp.bias": "model-00008-of-00009.safetensors",
-        "transformer.h.54.ln_mlp.weight": "model-00008-of-00009.safetensors",
-        "transformer.h.54.mlp.dense_4h_to_h.weight": "model-00008-of-00009.safetensors",
-        "transformer.h.54.mlp.dense_h_to_4h.weight": "model-00008-of-00009.safetensors",
-        "transformer.h.54.self_attention.dense.weight": "model-00008-of-00009.safetensors",
-        "transformer.h.54.self_attention.query_key_value.weight": "model-00008-of-00009.safetensors",
-        "transformer.h.55.ln_attn.bias": "model-00008-of-00009.safetensors",
-        "transformer.h.55.ln_attn.weight": "model-00008-of-00009.safetensors",
-        "transformer.h.55.ln_mlp.bias": "model-00008-of-00009.safetensors",
-        "transformer.h.55.ln_mlp.weight": "model-00008-of-00009.safetensors",
-        "transformer.h.55.mlp.dense_4h_to_h.weight": "model-00009-of-00009.safetensors",
-        "transformer.h.55.mlp.dense_h_to_4h.weight": "model-00009-of-00009.safetensors",
-        "transformer.h.55.self_attention.dense.weight": "model-00008-of-00009.safetensors",
-        "transformer.h.55.self_attention.query_key_value.weight": "model-00008-of-00009.safetensors",
-        "transformer.h.56.ln_attn.bias": "model-00009-of-00009.safetensors",
-        "transformer.h.56.ln_attn.weight": "model-00009-of-00009.safetensors",
-        "transformer.h.56.ln_mlp.bias": "model-00009-of-00009.safetensors",
-        "transformer.h.56.ln_mlp.weight": "model-00009-of-00009.safetensors",
-        "transformer.h.56.mlp.dense_4h_to_h.weight": "model-00009-of-00009.safetensors",
-        "transformer.h.56.mlp.dense_h_to_4h.weight": "model-00009-of-00009.safetensors",
-        "transformer.h.56.self_attention.dense.weight": "model-00009-of-00009.safetensors",
-        "transformer.h.56.self_attention.query_key_value.weight": "model-00009-of-00009.safetensors",
-        "transformer.h.57.ln_attn.bias": "model-00009-of-00009.safetensors",
-        "transformer.h.57.ln_attn.weight": "model-00009-of-00009.safetensors",
-        "transformer.h.57.ln_mlp.bias": "model-00009-of-00009.safetensors",
-        "transformer.h.57.ln_mlp.weight": "model-00009-of-00009.safetensors",
-        "transformer.h.57.mlp.dense_4h_to_h.weight": "model-00009-of-00009.safetensors",
-        "transformer.h.57.mlp.dense_h_to_4h.weight": "model-00009-of-00009.safetensors",
-        "transformer.h.57.self_attention.dense.weight": "model-00009-of-00009.safetensors",
-        "transformer.h.57.self_attention.query_key_value.weight": "model-00009-of-00009.safetensors",
-        "transformer.h.58.ln_attn.bias": "model-00009-of-00009.safetensors",
-        "transformer.h.58.ln_attn.weight": "model-00009-of-00009.safetensors",
-        "transformer.h.58.ln_mlp.bias": "model-00009-of-00009.safetensors",
-        "transformer.h.58.ln_mlp.weight": "model-00009-of-00009.safetensors",
-        "transformer.h.58.mlp.dense_4h_to_h.weight": "model-00009-of-00009.safetensors",
-        "transformer.h.58.mlp.dense_h_to_4h.weight": "model-00009-of-00009.safetensors",
-        "transformer.h.58.self_attention.dense.weight": "model-00009-of-00009.safetensors",
-        "transformer.h.58.self_attention.query_key_value.weight": "model-00009-of-00009.safetensors",
-        "transformer.h.59.ln_attn.bias": "model-00009-of-00009.safetensors",
-        "transformer.h.59.ln_attn.weight": "model-00009-of-00009.safetensors",
-        "transformer.h.59.ln_mlp.bias": "model-00009-of-00009.safetensors",
-        "transformer.h.59.ln_mlp.weight": "model-00009-of-00009.safetensors",
-        "transformer.h.59.mlp.dense_4h_to_h.weight": "model-00009-of-00009.safetensors",
-        "transformer.h.59.mlp.dense_h_to_4h.weight": "model-00009-of-00009.safetensors",
-        "transformer.h.59.self_attention.dense.weight": "model-00009-of-00009.safetensors",
-        "transformer.h.59.self_attention.query_key_value.weight": "model-00009-of-00009.safetensors",
-        "transformer.h.6.ln_attn.bias": "model-00001-of-00009.safetensors",
-        "transformer.h.6.ln_attn.weight": "model-00001-of-00009.safetensors",
-        "transformer.h.6.ln_mlp.bias": "model-00001-of-00009.safetensors",
-        "transformer.h.6.ln_mlp.weight": "model-00001-of-00009.safetensors",
-        "transformer.h.6.mlp.dense_4h_to_h.weight": "model-00002-of-00009.safetensors",
-        "transformer.h.6.mlp.dense_h_to_4h.weight": "model-00002-of-00009.safetensors",
-        "transformer.h.6.self_attention.dense.weight": "model-00001-of-00009.safetensors",
-        "transformer.h.6.self_attention.query_key_value.weight": "model-00001-of-00009.safetensors",
-        "transformer.h.7.ln_attn.bias": "model-00002-of-00009.safetensors",
-        "transformer.h.7.ln_attn.weight": "model-00002-of-00009.safetensors",
-        "transformer.h.7.ln_mlp.bias": "model-00002-of-00009.safetensors",
-        "transformer.h.7.ln_mlp.weight": "model-00002-of-00009.safetensors",
-        "transformer.h.7.mlp.dense_4h_to_h.weight": "model-00002-of-00009.safetensors",
-        "transformer.h.7.mlp.dense_h_to_4h.weight": "model-00002-of-00009.safetensors",
-        "transformer.h.7.self_attention.dense.weight": "model-00002-of-00009.safetensors",
-        "transformer.h.7.self_attention.query_key_value.weight": "model-00002-of-00009.safetensors",
-        "transformer.h.8.ln_attn.bias": "model-00002-of-00009.safetensors",
-        "transformer.h.8.ln_attn.weight": "model-00002-of-00009.safetensors",
-        "transformer.h.8.ln_mlp.bias": "model-00002-of-00009.safetensors",
-        "transformer.h.8.ln_mlp.weight": "model-00002-of-00009.safetensors",
-        "transformer.h.8.mlp.dense_4h_to_h.weight": "model-00002-of-00009.safetensors",
-        "transformer.h.8.mlp.dense_h_to_4h.weight": "model-00002-of-00009.safetensors",
-        "transformer.h.8.self_attention.dense.weight": "model-00002-of-00009.safetensors",
-        "transformer.h.8.self_attention.query_key_value.weight": "model-00002-of-00009.safetensors",
-        "transformer.h.9.ln_attn.bias": "model-00002-of-00009.safetensors",
-        "transformer.h.9.ln_attn.weight": "model-00002-of-00009.safetensors",
-        "transformer.h.9.ln_mlp.bias": "model-00002-of-00009.safetensors",
-        "transformer.h.9.ln_mlp.weight": "model-00002-of-00009.safetensors",
-        "transformer.h.9.mlp.dense_4h_to_h.weight": "model-00002-of-00009.safetensors",
-        "transformer.h.9.mlp.dense_h_to_4h.weight": "model-00002-of-00009.safetensors",
-        "transformer.h.9.self_attention.dense.weight": "model-00002-of-00009.safetensors",
-        "transformer.h.9.self_attention.query_key_value.weight": "model-00002-of-00009.safetensors",
-        "transformer.ln_f.bias": "model-00009-of-00009.safetensors",
-        "transformer.ln_f.weight": "model-00009-of-00009.safetensors",
-        "transformer.word_embeddings.weight": "model-00001-of-00009.safetensors"
-    }
-}

modeling_falcon.py → modelling_RW.py

@@ -1,20 +1,9 @@
-# coding=utf-8
-# Copyright 2023 the Falcon authors and HuggingFace Inc. team.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""PyTorch Falcon model."""
+# port of models described in RW
+# We use the bloom model as a starting point for these model.
+# Please refer to the bloom models for usage instructions.
 
 import math
+import warnings
 from typing import Optional, Tuple, Union
 
 import torch
@@ -31,60 +20,59 @@ from transformers.modeling_outputs import (
     TokenClassifierOutput,
 )
 from transformers.modeling_utils import PreTrainedModel
-from transformers.utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward, logging
-from .configuration_falcon import FalconConfig
-
+from transformers.utils import logging
+from .configuration_RW import RWConfig
 
 logger = logging.get_logger(__name__)
 
-FALCON_PRETRAINED_MODEL_ARCHIVE_LIST = [
-    "tiiuae/falcon-40b",
-    "tiiuae/falcon-40b-instruct",
-    "tiiuae/falcon-7b",
-    "tiiuae/falcon-7b-instruct",
-    "tiiuae/falcon-rw-7b",
-    "tiiuae/falcon-rw-1b",
-]
-_CHECKPOINT_FOR_DOC = "Rocketknight1/falcon-rw-1b"
-_CONFIG_FOR_DOC = "FalconConfig"
-
-
 # NOTE(Hesslow): Unfortunately we did not fuse matmul and bias during training, this means that there's one additional quantization to bfloat16 between the operations.
 # In order not to degrade the quality of our HF-port, we keep these characteristics in the final model.
-class FalconLinear(nn.Linear):
+class Linear(nn.Linear):
     def forward(self, input: torch.Tensor) -> torch.Tensor:
-        hidden_states = input @ self.weight.T
+        ret = input @ self.weight.T
         if self.bias is None:
-            return hidden_states
-        return hidden_states + self.bias
+            return ret
+        else:
+            return ret + self.bias
+
 
+from einops import rearrange
 
 # rotary pos emb helpers (torch.jit.script does not seem to support staticmethod...)
 def rotate_half(x):
     x1, x2 = x[..., : x.shape[-1] // 2], x[..., x.shape[-1] // 2 :]
-    return torch.cat((-x2, x1), dim=-1)
+    return torch.cat((-x2, x1), dim=x1.ndim - 1)  # dim=-1 triggers a bug in torch < 1.8.0
 
 
-class FalconRotaryEmbedding(nn.Module):
+class RotaryEmbedding(torch.nn.Module):
     """Implementation of RotaryEmbedding from GPT-NeoX.
-    This implementation is designed to operate on queries and keys that are compatible with `[batch_size,
-    n_heads_per_partition, seq_len, head_dim]` (e.g. MinGPTAttention format).
+    This implementation is design to operate on queries and keys that are compatible with
+    [batch_size, n_heads_per_partition, seq_len, head_dim] (e.g. MinGPTAttention format).
     """
 
-    def __init__(self, head_dim: int, base=10000):
+    def __init__(
+        self,
+        head_dim: int,
+        base=10000,
+    ):
         super().__init__()
         inv_freq = 1.0 / (base ** (torch.arange(0, head_dim, 2).float() / head_dim))
         self.register_buffer("inv_freq", inv_freq, persistent=False)
         self.head_dim = head_dim
-        self.seq_len_cached = -1
+        self.seq_len_cached = None
+        self.batch_size_cached = None
         self.cos_cached: torch.Tensor | None = None
         self.sin_cached: torch.Tensor | None = None
 
-    def cos_sin(self, seq_len: int, past_key_values_length: int, device="cpu", dtype=torch.bfloat16) -> torch.Tensor:
-        total_length = seq_len + past_key_values_length
-        if total_length > self.seq_len_cached:
-            self.seq_len_cached = total_length
-            t = torch.arange(total_length, device=device, dtype=self.inv_freq.dtype)
+    def cos_sin(
+        self,
+        seq_len: int,
+        device="cuda",
+        dtype=torch.bfloat16,
+    ) -> torch.Tensor:
+        if seq_len != self.seq_len_cached:
+            self.seq_len_cached = seq_len
+            t = torch.arange(seq_len, device=device).type_as(self.inv_freq)
             freqs = torch.einsum("i,j->ij", t, self.inv_freq)
             emb = torch.cat((freqs, freqs), dim=-1).to(device)
 
@@ -97,46 +85,36 @@ class FalconRotaryEmbedding(nn.Module):
             self.cos_cached = self.cos_cached.type(dtype)
             self.sin_cached = self.sin_cached.type(dtype)
 
-        return (
-            self.cos_cached[:, past_key_values_length : seq_len + past_key_values_length],
-            self.sin_cached[:, past_key_values_length : seq_len + past_key_values_length],
-        )
+        return self.cos_cached, self.sin_cached
 
-    def forward(self, query, key, past_key_values_length=0):
-        batch, seq_len, head_dim = query.shape
-        cos, sin = self.cos_sin(seq_len, past_key_values_length, query.device, query.dtype)
-        return (query * cos) + (rotate_half(query) * sin), (key * cos) + (rotate_half(key) * sin)
+    def forward(self, q, k):
+        batch, seq_len, head_dim = q.shape
+        cos, sin = self.cos_sin(seq_len, q.device, q.dtype)
+        return (q * cos) + (rotate_half(q) * sin), (k * cos) + (rotate_half(k) * sin)
 
 
 def _make_causal_mask(
     input_ids_shape: torch.Size, device: torch.device, past_key_values_length: int
 ) -> torch.BoolTensor:
-    """
-    Make causal mask used for self-attention. This mask does not take the existing attention mask into account - it
-    just blocks tokens from attending forwards in the sequence. The output shape will be `[batch_size, 1,
-    target_length, target_length+past_key_values_length]`.
-    """
     batch_size, target_length = input_ids_shape
+    mask = torch.empty((target_length, target_length + past_key_values_length), dtype=torch.bool, device=device)
+    # ONNX doesn't support `torch.Tensor.triu` properly, thus we use this workaround
+    seq_ids = torch.arange(target_length, device=device)
+    mask[:, past_key_values_length:] = seq_ids[:, None] < seq_ids[None, :]
+
+    if past_key_values_length > 0:
+        mask[:, :past_key_values_length] = False
 
-    mask = torch.triu(torch.ones((target_length, target_length), dtype=torch.bool, device=device), diagonal=1)
-    # If past_key_values_length is 0 this is an empty tensor and the concatenation is a no-op.
-    # This code style is an unfortunate consequence of getting your TF engineer to port models; doing it this
-    # way avoids a data-dependent conditional, which will help me when I have to port this to XLA later.
-    past_mask = torch.zeros((target_length, past_key_values_length), dtype=torch.bool, device=device)
-    mask = torch.cat([past_mask, mask], dim=-1)
     expanded_mask = mask[None, None, :, :].expand(batch_size, 1, target_length, target_length + past_key_values_length)
     return expanded_mask
 
 
-def _expand_mask(mask: torch.Tensor, past_key_values_length: int) -> torch.BoolTensor:
-    """
-    Expands attention_mask from `[batch_size, seq_length]` to `[batch_size, 1, seq_length, seq_length + past_length]`.
-    """
-    batch_size, total_length = mask.shape
-    seq_length = total_length - past_key_values_length if past_key_values_length is not None else total_length
+def _expand_mask(mask: torch.Tensor, tgt_length: int) -> torch.BoolTensor:
+    batch_size, src_length = mask.shape
+    tgt_length = tgt_length if tgt_length is not None else src_length
 
     expanded_mask = ~(mask[:, None, None, :].to(torch.bool))
-    return expanded_mask.expand(batch_size, 1, seq_length, total_length)
+    return expanded_mask.expand(batch_size, 1, tgt_length, src_length)
 
 
 def build_alibi_tensor(attention_mask: torch.Tensor, num_heads: int, dtype: torch.dtype) -> torch.Tensor:
@@ -167,32 +145,18 @@ def build_alibi_tensor(attention_mask: torch.Tensor, num_heads: int, dtype: torc
     return alibi.reshape(batch_size * num_heads, 1, seq_length).to(dtype)
 
 
-# Copied from transformers.models.bloom.modeling_bloom.dropout_add
 def dropout_add(x: torch.Tensor, residual: torch.Tensor, prob: float, training: bool) -> torch.Tensor:
-    """
-    Dropout add function
-
-    Args:
-        x (`torch.tensor`, *required*):
-            input tensor
-        residual (`torch.tensor`, *required*):
-            residual tensor
-        prob (`float`, *required*):
-            dropout probability
-        training (`bool`, *required*):
-            training mode
-    """
     out = F.dropout(x, p=prob, training=training)
     out = residual + out
     return out
 
 
-class FalconAttention(nn.Module):
-    def __init__(self, config: FalconConfig):
+class Attention(nn.Module):
+    def __init__(self, config: RWConfig):
         super().__init__()
 
         self.hidden_size = config.hidden_size
-        self.num_heads = config.num_attention_heads
+        self.num_heads = config.n_head
         self.head_dim = self.hidden_size // self.num_heads
         self.split_size = self.hidden_size
         self.hidden_dropout = config.hidden_dropout
@@ -203,62 +167,59 @@ class FalconAttention(nn.Module):
                 f" {self.num_heads})."
             )
 
-        self.maybe_rotary = FalconRotaryEmbedding(config.head_dim) if config.rotary else lambda q, k, t: (q, k)
+        self.maybe_rotary = RotaryEmbedding(config.head_dim) if config.rotary else lambda q, k: (q, k)
 
         # Layer-wise attention scaling
         self.inv_norm_factor = 1.0 / math.sqrt(self.head_dim)
         self.beta = self.inv_norm_factor
-        if config.new_decoder_architecture:
-            qkv_out_dim = (config.num_kv_heads * 2 + config.num_attention_heads) * self.head_dim
-        elif config.multi_query:
-            qkv_out_dim = self.hidden_size + 2 * self.head_dim
-        else:
-            qkv_out_dim = 3 * self.hidden_size
-        self.query_key_value = FalconLinear(self.hidden_size, qkv_out_dim, bias=config.bias)
-        self.new_decoder_architecture = config.new_decoder_architecture
-        self.multi_query = config.multi_query
-        self.dense = FalconLinear(self.hidden_size, self.hidden_size, bias=config.bias)
+
+        self.query_key_value = Linear(
+            self.hidden_size,
+            (config.n_head_kv * 2 + config.n_head) * self.head_dim,
+            bias=config.bias,
+        )
+        self.dense = Linear(self.hidden_size, self.hidden_size, bias=config.bias)
         self.attention_dropout = nn.Dropout(config.attention_dropout)
-        self.num_kv_heads = config.num_kv_heads if (self.new_decoder_architecture or not self.multi_query) else 1
+        self.num_kv = config.n_head_kv
 
     def _split_heads(self, fused_qkv: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
         """
-        Split the last dimension into (num_heads, head_dim), results share same memory storage as `fused_qkv`
+        Split the last dimension into (num_heads, head_dim), results share same memory
+        storage as `fused_qkv`
 
         Args:
             fused_qkv (`torch.tensor`, *required*): [batch_size, seq_length, num_heads * 3 * head_dim]
 
         Returns:
-            query: [batch_size, seq_length, num_heads, head_dim] key: [batch_size, seq_length, num_heads, head_dim]
+            query: [batch_size, seq_length, num_heads, head_dim]
+            key: [batch_size, seq_length, num_heads, head_dim]
             value: [batch_size, seq_length, num_heads, head_dim]
         """
-        if self.new_decoder_architecture:
-            batch, seq_len, _ = fused_qkv.shape
-            qkv = fused_qkv.view(batch, seq_len, -1, self.num_heads // self.num_kv_heads + 2, self.head_dim)
-            query = qkv[:, :, :, :-2]
-            key = qkv[:, :, :, [-2]]
-            value = qkv[:, :, :, [-1]]
-            key = torch.broadcast_to(key, query.shape)
-            value = torch.broadcast_to(value, query.shape)
-
-            query, key, value = [x.flatten(2, 3) for x in (query, key, value)]
-            return query, key, value
-        elif not self.multi_query:
-            batch_size, seq_length, three_times_hidden_size = fused_qkv.shape
-            fused_qkv = fused_qkv.view(batch_size, seq_length, self.num_heads, 3, self.head_dim)
-            return fused_qkv[..., 0, :], fused_qkv[..., 1, :], fused_qkv[..., 2, :]
-        else:
-            batch_size, seq_length, three_times_hidden_size = fused_qkv.shape
-            fused_qkv = fused_qkv.view(batch_size, seq_length, self.num_heads + 2, self.head_dim)
-            return fused_qkv[..., :-2, :], fused_qkv[..., [-2], :], fused_qkv[..., [-1], :]
+        batch, seq_len, _ = fused_qkv.shape
+        qkv = fused_qkv.view(batch, seq_len, -1, self.num_heads // self.num_kv + 2, 64)
+        q = qkv[:, :, :, :-2]
+        k = qkv[:, :, :, [-2]]
+        v = qkv[:, :, :, [-1]]
+        k = torch.broadcast_to(k, q.shape)
+        v = torch.broadcast_to(v, q.shape)
+
+        q, k, v = [
+            rearrange(
+                x,
+                "batch seq_len group num_heads head_dim ->\
+                batch seq_len (group num_heads) head_dim",
+                head_dim=self.head_dim,
+            )
+            for x in [q, k, v]
+        ]
+        return q, k, v
 
-    # Copied from transformers.models.bloom.modeling_bloom.BloomAttention._merge_heads
     def _merge_heads(self, x: torch.Tensor) -> torch.Tensor:
         """
         Merge heads together over the last dimenstion
 
         Args:
-            x (`torch.tensor`, *required*): [batch_size * num_heads, seq_length, head_dim]
+            x: (`torch.tensor`, *required*): [batch_size * num_heads, seq_length, head_dim]
 
         Returns:
             torch.tensor: [batch_size, seq_length, num_heads * head_dim]
@@ -281,7 +242,7 @@ class FalconAttention(nn.Module):
     def forward(
         self,
         hidden_states: torch.Tensor,
-        alibi: Optional[torch.Tensor],
+        alibi: torch.Tensor,
         attention_mask: torch.Tensor,
         layer_past: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
         head_mask: Optional[torch.Tensor] = None,
@@ -289,120 +250,106 @@ class FalconAttention(nn.Module):
         output_attentions: bool = False,
     ):
         fused_qkv = self.query_key_value(hidden_states)  # [batch_size, seq_length, 3 x hidden_size]
-        num_kv_heads = self.num_heads if self.new_decoder_architecture else self.num_kv_heads
+
         # 3 x [batch_size, seq_length, num_heads, head_dim]
         (query_layer, key_layer, value_layer) = self._split_heads(fused_qkv)
 
-        batch_size, query_length, _, _ = query_layer.shape
+        batch_size, q_length, _, _ = query_layer.shape
 
-        query_layer = query_layer.transpose(1, 2).reshape(batch_size * self.num_heads, query_length, self.head_dim)
+        query_layer = query_layer.transpose(1, 2).reshape(batch_size * self.num_heads, q_length, self.head_dim)
         key_layer = key_layer.transpose(1, 2).reshape(
-            batch_size * num_kv_heads,
-            query_length,
+            batch_size * self.num_heads,
+            q_length,
             self.head_dim,
         )
-        value_layer = value_layer.transpose(1, 2).reshape(batch_size * num_kv_heads, query_length, self.head_dim)
+        value_layer = value_layer.transpose(1, 2).reshape(batch_size * self.num_heads, q_length, self.head_dim)
 
-        past_kv_length = 0 if layer_past is None else layer_past[0].shape[1]
-        query_layer, key_layer = self.maybe_rotary(query_layer, key_layer, past_kv_length)
+        query_layer, key_layer = self.maybe_rotary(query_layer, key_layer)
 
         if layer_past is not None:
             past_key, past_value = layer_past
             # concatenate along seq_length dimension:
-            #  - key: [batch_size * self.num_heads, kv_length, head_dim]
+            #  - key: [batch_size * self.num_heads, head_dim, kv_length]
             #  - value: [batch_size * self.num_heads, kv_length, head_dim]
             key_layer = torch.cat((past_key, key_layer), dim=1)
             value_layer = torch.cat((past_value, value_layer), dim=1)
 
         _, kv_length, _ = key_layer.shape
-        if use_cache:
+
+        if use_cache is True:
             present = (key_layer, value_layer)
         else:
             present = None
 
-        attention_mask_float = (attention_mask * 1.0).masked_fill(attention_mask, float("-1e9")).to(query_layer.dtype)
-
-        query_layer_ = query_layer.reshape(batch_size, self.num_heads, -1, self.head_dim)
-        key_layer_ = key_layer.reshape(batch_size, num_kv_heads, -1, self.head_dim)
-        value_layer_ = value_layer.reshape(batch_size, num_kv_heads, -1, self.head_dim)
-
         if alibi is None:
-            if output_attentions:
-                # F.scaled_dot_product_attention doesn't return the attention weights, so we have
-                # to do it by hand if we want them
-                attention_scores = query_layer_ @ key_layer_.transpose(-1, -2)
-                attention_scores /= math.sqrt(self.head_dim)
+            query_layer_ = query_layer.reshape(batch_size, self.num_heads, -1, self.head_dim)
+            key_layer_ = key_layer.reshape(batch_size, self.num_heads, -1, self.head_dim)
+            value_layer_ = value_layer.reshape(batch_size, self.num_heads, -1, self.head_dim)
 
-                attention_scores = F.softmax(
-                    attention_scores + attention_mask_float, dim=-1, dtype=hidden_states.dtype
-                )
-                attn_output = attention_scores @ value_layer_
-            else:
-                attn_output = F.scaled_dot_product_attention(
-                    query_layer_, key_layer_, value_layer_, attention_mask_float, 0.0, is_causal=False
-                )
-                attention_scores = None
+            attn_output = F.scaled_dot_product_attention(
+                query_layer_, key_layer_, value_layer_, None, 0.0, is_causal=True
+            )
 
-            attn_output = attn_output.view(batch_size, self.num_heads, query_length, self.head_dim)
-            attn_output = attn_output.permute(0, 2, 1, 3)
-            attn_output = attn_output.reshape(batch_size, query_length, self.num_heads * self.head_dim)
+            x = attn_output.view(batch_size, self.num_heads, q_length, self.head_dim)
+            x = x.permute(0, 2, 1, 3)
+            attn_output = x.reshape(batch_size, q_length, self.num_heads * self.head_dim)
 
             output_tensor = self.dense(attn_output)
 
-            if output_attentions:
-                return output_tensor, present, attention_scores
-            else:
-                return output_tensor, present
-
+            outputs = (output_tensor, present)
+            assert not output_attentions  # not supported.
+            return outputs
         else:
-            matmul_result = query_layer_ @ key_layer_.transpose(-1, -2)
+            attention_mask_float = (attention_mask * 1.0).masked_fill(attention_mask, -1e9).to(torch.bfloat16)
+            matmul_result = query_layer @ key_layer.transpose(-1, -2)
 
             # change view to [batch_size, num_heads, q_length, kv_length]
-            attention_scores = matmul_result.view(batch_size, self.num_heads, query_length, kv_length)
+            attention_scores = matmul_result.view(batch_size, self.num_heads, q_length, kv_length)
 
             # cast attention scores to fp32, compute scaled softmax and cast back to initial dtype - [batch_size, num_heads, q_length, kv_length]
             input_dtype = attention_scores.dtype
             # `float16` has a minimum value of -65504.0, whereas `bfloat16` and `float32` have a minimum value of `-3.4e+38`
             if input_dtype == torch.float16 or input_dtype == torch.bfloat16:
                 attention_scores = attention_scores.to(torch.float32)
-            # Matt (HF) note: We could possibly use F.scaled_dot_product_attention here too, by
-            # adding (alibi * self.inv_norm_factor) to attention_mask_float. I think this would be mathematically
-            # equivalent and more performant, but there might be a numerical difference. If you're reading this
-            # and you'd like to experiment and maybe file a PR, feel free!
-            attention_logits = attention_scores + alibi.view(batch_size, self.num_heads, 1, -1)
-            attention_logits *= self.inv_norm_factor
-            attention_probs = F.softmax(attention_logits + attention_mask_float, dim=-1, dtype=hidden_states.dtype)
+            # attn_weights = torch.masked_fill(attention_scores, attention_mask, torch.finfo(attention_scores.dtype).min)
+            attention_probs = F.softmax(
+                (attention_scores + alibi.view(batch_size, self.num_heads, 1, -1)) * self.inv_norm_factor
+                + attention_mask_float,
+                dim=-1,
+                dtype=hidden_states.dtype,
+            )
             # [batch_size, num_heads, q_length, kv_length]
             attention_probs = self.attention_dropout(attention_probs)
 
             if head_mask is not None:
                 attention_probs = attention_probs * head_mask
 
-            # change view [batch_size, num_heads, q_length, kv_length]
-            attention_probs_reshaped = attention_probs.view(batch_size, self.num_heads, query_length, kv_length)
+            # change view [batch_size x num_heads, q_length, kv_length]
+            attention_probs_reshaped = attention_probs.view(batch_size * self.num_heads, q_length, kv_length)
 
             # matmul: [batch_size * num_heads, q_length, head_dim]
-            context_layer = (attention_probs_reshaped @ value_layer_).flatten(0, 1)
+            context_layer = attention_probs_reshaped @ value_layer
 
             # change view [batch_size, num_heads, q_length, head_dim]
             context_layer = self._merge_heads(context_layer)
 
             output_tensor = self.dense(context_layer)
 
+            outputs = (output_tensor, present)
             if output_attentions:
-                return output_tensor, present, attention_probs
-            else:
-                return output_tensor, present
+                outputs += (attention_probs,)
+
+            return outputs
 
 
-class FalconMLP(nn.Module):
-    def __init__(self, config: FalconConfig):
+class MLP(nn.Module):
+    def __init__(self, config: RWConfig):
         super().__init__()
         hidden_size = config.hidden_size
 
-        self.dense_h_to_4h = FalconLinear(hidden_size, 4 * hidden_size, bias=config.bias)
+        self.dense_h_to_4h = Linear(hidden_size, 4 * hidden_size, bias=config.bias)
         self.act = nn.GELU()
-        self.dense_4h_to_h = FalconLinear(4 * hidden_size, hidden_size, bias=config.bias)
+        self.dense_4h_to_h = Linear(4 * hidden_size, hidden_size, bias=config.bias)
         self.hidden_dropout = config.hidden_dropout
 
     def forward(self, x: torch.Tensor) -> torch.Tensor:
@@ -411,47 +358,43 @@ class FalconMLP(nn.Module):
         return x
 
 
-class FalconDecoderLayer(nn.Module):
-    def __init__(self, config: FalconConfig):
+class DecoderLayer(nn.Module):
+    def __init__(self, config: RWConfig):
         super().__init__()
         hidden_size = config.hidden_size
-        self.num_heads = config.num_attention_heads
-        self.self_attention = FalconAttention(config)
-        self.mlp = FalconMLP(config)
+
+        self.ln_attn = LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
+        self.ln_mlp = LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
+
+        self.num_heads = config.n_head
+        self.self_attention = Attention(config)
+
+        self.mlp = MLP(config)
+
+        self.apply_residual_connection_post_layernorm = config.apply_residual_connection_post_layernorm
         self.hidden_dropout = config.hidden_dropout
-        self.config = config
 
-        if config.new_decoder_architecture:
-            # The layer norm before self-attention
-            self.ln_attn = LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
-            # The layer norm before the MLP
-            self.ln_mlp = LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
-        else:
-            self.input_layernorm = LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
-            if not config.parallel_attn:
-                self.post_attention_layernorm = LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
+        self.config = config
 
     def forward(
         self,
         hidden_states: torch.Tensor,
-        alibi: Optional[torch.Tensor],
+        alibi: torch.Tensor,
         attention_mask: torch.Tensor,
         layer_past: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
         head_mask: Optional[torch.Tensor] = None,
         use_cache: bool = False,
         output_attentions: bool = False,
     ):
-        residual = hidden_states
 
-        if self.config.new_decoder_architecture:
-            attention_layernorm_out = self.ln_attn(hidden_states)
-            mlp_layernorm_out = self.ln_mlp(hidden_states)
-        else:
-            attention_layernorm_out = self.input_layernorm(hidden_states)
+        ln_attn = self.ln_attn(hidden_states)
+        ln_mlp = self.ln_mlp(hidden_states)
+
+        residual = hidden_states
 
         # Self attention.
         attn_outputs = self.self_attention(
-            attention_layernorm_out,
+            ln_attn,
             layer_past=layer_past,
             attention_mask=attention_mask,
             alibi=alibi,
@@ -462,24 +405,14 @@ class FalconDecoderLayer(nn.Module):
 
         attention_output = attn_outputs[0]
 
-        if not self.config.new_decoder_architecture:
-            if self.config.parallel_attn:
-                mlp_layernorm_out = attention_layernorm_out
-            else:
-                residual = dropout_add(
-                    attention_output, residual, self.config.attention_dropout, training=self.training
-                )
-                mlp_layernorm_out = self.post_attention_layernorm(residual)
-
         outputs = attn_outputs[1:]
 
         # MLP.
-        mlp_output = self.mlp(mlp_layernorm_out)
+        mlp_output = self.mlp(ln_mlp)
 
-        if self.config.new_decoder_architecture or self.config.parallel_attn:
-            mlp_output += attention_output
-
-        output = dropout_add(mlp_output, residual, self.config.hidden_dropout, training=self.training)
+        output = dropout_add(
+            mlp_output + attention_output, residual, self.config.hidden_dropout, training=self.training
+        )
 
         if use_cache:
             outputs = (output,) + outputs
@@ -489,93 +422,24 @@ class FalconDecoderLayer(nn.Module):
         return outputs  # hidden_states, present, attentions
 
 
-FALCON_START_DOCSTRING = r"""
-
-    This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
-    library implements for all its model (such as downloading or saving, resizing the input embeddings etc.)
-
-    This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
-    Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
-    and behavior.
-
-    Parameters:
-        config ([`FalconConfig`]): Model configuration class with all the parameters of the model.
-            Initializing with a config file does not load the weights associated with the model, only the
-            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
-"""
-
-FALCON_INPUTS_DOCSTRING = r"""
-    Args:
-        input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`):
-            `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values[0][0].shape[2]`
-            (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary.
-
-            If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as
-            `input_ids`.
-
-            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
-            [`PreTrainedTokenizer.__call__`] for details.
-
-            [What are input IDs?](../glossary#input-ids)
-        past_key_values (`Tuple[Tuple[torch.Tensor]]` of length `config.num_hidden_layers`):
-            Contains precomputed hidden-states (key and values in the attention blocks) as computed by the model (see
-            `past_key_values` output below). Can be used to speed up sequential decoding. The `input_ids` which have
-            their past given to this model should not be passed as `input_ids` as they have already been computed.
-
-            Each element of `past_key_values` is a tuple (past_key, past_value):
-            - past_key: [batch_size * num_heads, head_dim, kv_length]
-            - past_value: [batch_size * num_heads, kv_length, head_dim]
-        attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*):
-            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
-
-            - 1 for tokens that are **not masked**,
-            - 0 for tokens that are **masked**.
-
-            [What are attention masks?](../glossary#attention-mask)
-        head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
-            Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`:
-
-            - 1 indicates the head is **not masked**,
-            - 0 indicates the head is **masked**.
-
-        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
-            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
-            is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
-            model's internal embedding lookup matrix.
-
-            If `past_key_values` is used, optionally only the last `inputs_embeds` have to be input (see
-            `past_key_values`).
-        use_cache (`bool`, *optional*):
-            If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
-            `past_key_values`).
-        output_attentions (`bool`, *optional*):
-            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
-            tensors for more detail.
-        output_hidden_states (`bool`, *optional*):
-            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
-            more detail.
-        return_dict (`bool`, *optional*):
-            Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple.
-"""
-
-
-class FalconPreTrainedModel(PreTrainedModel):
+class RWPreTrainedModel(PreTrainedModel):
+    _keys_to_ignore_on_load_missing = [r"h.*.self_attention.scale_mask_softmax.causal_mask", r"lm_head.weight"]
     """
     An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
     models.
     """
 
-    config_class = FalconConfig
+    config_class = RWConfig
     base_model_prefix = "transformer"
     supports_gradient_checkpointing = True
-    _no_split_modules = ["FalconDecoderLayer"]
+    _no_split_modules = ["DecoderLayer"]
 
     def __init__(self, *inputs, **kwargs):
         super().__init__(*inputs, **kwargs)
 
     def _init_weights(self, module: nn.Module):
         """Initialize the weights."""
-        if isinstance(module, nn.Linear) or isinstance(module, FalconLinear):
+        if isinstance(module, nn.Linear) or isinstance(module, Linear):
             # Slightly different from the TF version which uses truncated_normal for initialization
             # cf https://github.com/pytorch/pytorch/pull/5617
             module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
@@ -589,28 +453,26 @@ class FalconPreTrainedModel(PreTrainedModel):
             module.bias.data.zero_()
             module.weight.data.fill_(1.0)
 
-    # Copied from transformers.models.bloom.modeling_bloom.BloomPreTrainedModel._set_gradient_checkpointing with BloomModel->FalconModel
     def _set_gradient_checkpointing(self, module: nn.Module, value: bool = False):
-        if isinstance(module, FalconModel):
+        if isinstance(module, RWModel):
             module.gradient_checkpointing = value
 
     @staticmethod
-    def _convert_cache_to_standard_format(
+    def _convert_to_standard_cache(
         past_key_value: Tuple[Tuple[torch.Tensor, torch.Tensor]], batch_size: int
     ) -> Tuple[Tuple[torch.Tensor, torch.Tensor]]:
         """
         Standardizes the format of the cache so as to match most implementations, i.e. to tuple(tuple([batch_size,
         num_heads, ...]))
         """
-        batch_size_times_num_heads, kv_length, head_dim = past_key_value[0][0].shape
-        # [batch_size * self.num_heads, kv_length, head_dim] -> [batch_size, num_heads, kv_length, head_dim]
-        # Note that don't want to use self.num_attention_heads because the number of heads may vary depending
-        # on whether we use multi_query attention.
+        batch_size_times_num_heads, head_dim, seq_length = past_key_value[0][0].shape
         num_heads = batch_size_times_num_heads // batch_size
+        # key: [batch_size * num_heads, head_dim, seq_length] -> [batch_size, num_heads, head_dim, seq_length]
+        # value: [batch_size * num_heads, seq_length, head_dim] -> [batch_size, num_heads, seq_length, head_dim]
         return tuple(
             (
-                layer_past[0].view(batch_size, num_heads, kv_length, head_dim),
-                layer_past[1].view(batch_size, num_heads, kv_length, head_dim),
+                layer_past[0].view(batch_size, num_heads, head_dim, seq_length),
+                layer_past[1].view(batch_size, num_heads, seq_length, head_dim),
             )
             for layer_past in past_key_value
         )
@@ -619,35 +481,32 @@ class FalconPreTrainedModel(PreTrainedModel):
     def _convert_to_rw_cache(
         past_key_value: Tuple[Tuple[torch.Tensor, torch.Tensor]]
     ) -> Tuple[Tuple[torch.Tensor, torch.Tensor]]:
-        batch_size, num_heads, kv_length, head_dim = past_key_value[0][0].shape
+        batch_size, num_heads, head_dim, seq_length = past_key_value[0][0].shape
         batch_size_times_num_heads = batch_size * num_heads
-        # [batch_size, num_heads, kv_length, head_dim] -> [batch_size * num_heads, kv_length, head_dim]
+        # key:  [batch_size, num_heads, head_dim, seq_length] -> [batch_size * num_heads, head_dim, seq_length]
+        # value: [batch_size, num_heads, seq_length, head_dim] -> [batch_size * num_heads, seq_length, head_dim]
         return tuple(
             (
-                layer_past[0].view(batch_size_times_num_heads, kv_length, head_dim),
-                layer_past[1].view(batch_size_times_num_heads, kv_length, head_dim),
+                layer_past[0].view(batch_size_times_num_heads, head_dim, seq_length),
+                layer_past[1].view(batch_size_times_num_heads, seq_length, head_dim),
             )
             for layer_past in past_key_value
         )
 
 
-@add_start_docstrings(
-    "The bare Falcon Model transformer outputting raw hidden-states without any specific head on top.",
-    FALCON_START_DOCSTRING,
-)
-class FalconModel(FalconPreTrainedModel):
-    def __init__(self, config: FalconConfig):
+class RWModel(RWPreTrainedModel):
+    def __init__(self, config: RWConfig):
         super().__init__(config)
 
         self.embed_dim = config.hidden_size
-        self.num_heads = config.num_attention_heads
-        self.use_alibi = config.alibi
+        self.num_heads = config.n_head
+        self.alibi = config.alibi
 
         # Embedding + LN Embedding
         self.word_embeddings = nn.Embedding(config.vocab_size, self.embed_dim)
 
         # Transformer blocks
-        self.h = nn.ModuleList([FalconDecoderLayer(config) for _ in range(config.num_hidden_layers)])
+        self.h = nn.ModuleList([DecoderLayer(config) for _ in range(config.num_hidden_layers)])
 
         # Final Layer Norm
         self.ln_f = LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
@@ -660,31 +519,22 @@ class FalconModel(FalconPreTrainedModel):
     def get_input_embeddings(self):
         return self.word_embeddings
 
-    @staticmethod
     def _prepare_attn_mask(
-        attention_mask: torch.Tensor, input_shape: Tuple[int, int], past_key_values_length: int
+        self, attention_mask: torch.Tensor, input_shape: Tuple[int, int], past_key_values_length: int
     ) -> torch.BoolTensor:
-        # Create a causal mask
-        # The attention mask we receive as input should cover the whole extended sequence, including any past
-        # cache, so its shape should be [batch_size, seq_length + past_key_values_length]
-        # The output shape will be [batch_size, 1, seq_length, seq_length + past_key_values_length]
-        if input_shape[1] + past_key_values_length != attention_mask.shape[1]:
-            raise ValueError(
-                "Attention mask shape should be (batch_size, seq_length + past_key_values_length)"
-                f" but is {attention_mask.shape} with input_ids shape {input_shape} and past length"
-                f" {past_key_values_length}."
-            )
+        # create causal mask
+        # [batch_size, seq_length] -> [batch_size, 1, tgt_length, src_length]
         combined_attention_mask = None
         device = attention_mask.device
-        _, seq_length = input_shape
+        _, src_length = input_shape
 
-        if seq_length > 1:
+        if src_length > 1:
             combined_attention_mask = _make_causal_mask(
                 input_shape, device=device, past_key_values_length=past_key_values_length
             )
 
-        # [batch_size, seq_length + past_key_values_length] -> [batch_size, 1, seq_length, seq_length + past_key_values_length]
-        expanded_attn_mask = _expand_mask(attention_mask, past_key_values_length=past_key_values_length)
+        # [batch_size, seq_length] -> [batch_size, 1, tgt_length, src_length]
+        expanded_attn_mask = _expand_mask(attention_mask, tgt_length=src_length)
         combined_attention_mask = (
             expanded_attn_mask if combined_attention_mask is None else expanded_attn_mask | combined_attention_mask
         )
@@ -694,12 +544,6 @@ class FalconModel(FalconPreTrainedModel):
     def set_input_embeddings(self, new_embeddings: torch.Tensor):
         self.word_embeddings = new_embeddings
 
-    @add_start_docstrings_to_model_forward(FALCON_INPUTS_DOCSTRING)
-    @add_code_sample_docstrings(
-        checkpoint=_CHECKPOINT_FOR_DOC,
-        output_type=BaseModelOutputWithPastAndCrossAttentions,
-        config_class=_CONFIG_FOR_DOC,
-    )
     def forward(
         self,
         input_ids: Optional[torch.LongTensor] = None,
@@ -711,7 +555,18 @@ class FalconModel(FalconPreTrainedModel):
         output_attentions: Optional[bool] = None,
         output_hidden_states: Optional[bool] = None,
         return_dict: Optional[bool] = None,
+        **deprecated_arguments,
     ) -> Union[Tuple[torch.Tensor, ...], BaseModelOutputWithPastAndCrossAttentions]:
+        if deprecated_arguments.pop("position_ids", False) is not False:
+            # `position_ids` could have been `torch.Tensor` or `None` so defaulting pop to `False` allows to detect if users were passing explicitly `None`
+            warnings.warn(
+                "`position_ids` have no functionality in BLOOM and will be removed in v5.0.0. You can safely ignore"
+                " passing `position_ids`.",
+                FutureWarning,
+            )
+        if len(deprecated_arguments) > 0:
+            raise ValueError(f"Got unexpected arguments: {deprecated_arguments}")
+
         output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
         output_hidden_states = (
             output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
@@ -730,14 +585,12 @@ class FalconModel(FalconPreTrainedModel):
 
         if past_key_values is None:
             past_key_values = tuple([None] * len(self.h))
-        else:
-            past_key_values = self._convert_to_rw_cache(past_key_values)
 
         # Prepare head mask if needed
         # 1.0 in head_mask indicate we keep the head
         # attention_probs has shape batch_size x num_heads x N x N
         # head_mask has shape n_layer x batch x num_heads x N x N
-        head_mask = self.get_head_mask(head_mask, self.config.num_hidden_layers)
+        head_mask = self.get_head_mask(head_mask, self.config.n_layer)
 
         if inputs_embeds is None:
             inputs_embeds = self.word_embeddings(input_ids)
@@ -749,15 +602,17 @@ class FalconModel(FalconPreTrainedModel):
         all_hidden_states = () if output_hidden_states else None
 
         # Compute alibi tensor: check build_alibi_tensor documentation
+        seq_length_with_past = seq_length
         past_key_values_length = 0
         if past_key_values[0] is not None:
-            past_key_values_length = past_key_values[0][0].shape[1]  # 1 because RW-cache, not standard format
+            past_key_values_length = past_key_values[0][0].shape[2]
+            seq_length_with_past = seq_length_with_past + past_key_values_length
         if attention_mask is None:
-            attention_mask = torch.ones((batch_size, seq_length + past_key_values_length), device=hidden_states.device)
+            attention_mask = torch.ones((batch_size, seq_length_with_past), device=hidden_states.device)
         else:
             attention_mask = attention_mask.to(hidden_states.device)
 
-        if self.use_alibi:
+        if self.alibi:
             alibi = build_alibi_tensor(attention_mask, self.num_heads, dtype=hidden_states.dtype)
         else:
             alibi = None
@@ -769,10 +624,12 @@ class FalconModel(FalconPreTrainedModel):
         )
 
         for i, (block, layer_past) in enumerate(zip(self.h, past_key_values)):
+
             if output_hidden_states:
                 all_hidden_states = all_hidden_states + (hidden_states,)
 
             if self.gradient_checkpointing and self.training:
+
                 if use_cache:
                     logger.warning(
                         "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..."
@@ -817,9 +674,6 @@ class FalconModel(FalconPreTrainedModel):
         if output_hidden_states:
             all_hidden_states = all_hidden_states + (hidden_states,)
 
-        if presents is not None:
-            presents = self._convert_cache_to_standard_format(presents, batch_size)
-
         if not return_dict:
             return tuple(v for v in [hidden_states, presents, all_hidden_states, all_self_attentions] if v is not None)
 
@@ -831,16 +685,12 @@ class FalconModel(FalconPreTrainedModel):
         )
 
 
-@add_start_docstrings(
-    "The Falcon Model transformer with a language modeling head on top (linear layer with weights tied to the input embeddings).",
-    FALCON_START_DOCSTRING,
-)
-class FalconForCausalLM(FalconPreTrainedModel):
-    _tied_weights_keys = ["lm_head.weight"]
+class RWForCausalLM(RWPreTrainedModel):
+    _keys_to_ignore_on_load_missing = [r"h.*.self_attention.scale_mask_softmax.causal_mask", r"lm_head.weight"]
 
-    def __init__(self, config: FalconConfig):
+    def __init__(self, config: RWConfig):
         super().__init__(config)
-        self.transformer = FalconModel(config)
+        self.transformer = RWModel(config)
         self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
 
         # Initialize weights and apply final processing
@@ -855,26 +705,25 @@ class FalconForCausalLM(FalconPreTrainedModel):
     def prepare_inputs_for_generation(
         self,
         input_ids: torch.LongTensor,
-        past_key_values: Optional[torch.Tensor] = None,
+        past: Optional[torch.Tensor] = None,
         attention_mask: Optional[torch.Tensor] = None,
         **kwargs,
     ) -> dict:
-        if past_key_values is not None:
-            input_ids = input_ids[:, -1:]
+        # only last token for input_ids if past is not None
+        if past:
+            input_ids = input_ids[:, -1].unsqueeze(-1)
+
+            # the cache may be in the stardard format (e.g. in contrastive search), convert to our's format if needed
+            if past[0][0].shape[0] == input_ids.shape[0]:
+                past = self._convert_to_rw_cache(past)
 
         return {
             "input_ids": input_ids,
-            "past_key_values": past_key_values,
+            "past_key_values": past,
             "use_cache": kwargs.get("use_cache"),
             "attention_mask": attention_mask,
         }
 
-    @add_start_docstrings_to_model_forward(FALCON_INPUTS_DOCSTRING)
-    @add_code_sample_docstrings(
-        checkpoint=_CHECKPOINT_FOR_DOC,
-        output_type=CausalLMOutputWithCrossAttentions,
-        config_class=_CONFIG_FOR_DOC,
-    )
     def forward(
         self,
         input_ids: Optional[torch.LongTensor] = None,
@@ -887,6 +736,7 @@ class FalconForCausalLM(FalconPreTrainedModel):
         output_attentions: Optional[bool] = None,
         output_hidden_states: Optional[bool] = None,
         return_dict: Optional[bool] = None,
+        **deprecated_arguments,
     ) -> Union[Tuple[torch.Tensor], CausalLMOutputWithCrossAttentions]:
         r"""
         labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -894,6 +744,15 @@ class FalconForCausalLM(FalconPreTrainedModel):
             `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100`
             are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
         """
+        if deprecated_arguments.pop("position_ids", False) is not False:
+            # `position_ids` could have been `torch.Tensor` or `None` so defaulting pop to `False` allows to detect if users were passing explicitly `None`
+            warnings.warn(
+                "`position_ids` have no functionality in BLOOM and will be removed in v5.0.0. You can safely ignore"
+                " passing `position_ids`.",
+                FutureWarning,
+            )
+        if len(deprecated_arguments) > 0:
+            raise ValueError(f"Got unexpected arguments: {deprecated_arguments}")
 
         return_dict = return_dict if return_dict is not None else self.config.use_return_dict
 
@@ -946,6 +805,7 @@ class FalconForCausalLM(FalconPreTrainedModel):
 
         Output shares the same memory storage as `past`.
         """
+        standardized_past = self._convert_to_standard_cache(past, batch_size=len(beam_idx))
 
         # Get a copy of `beam_idx` on all the devices where we need those indices.
         device_to_beam_idx = {
@@ -956,42 +816,23 @@ class FalconForCausalLM(FalconPreTrainedModel):
                 layer_past[0].index_select(0, device_to_beam_idx[layer_past[0].device]),
                 layer_past[1].index_select(0, device_to_beam_idx[layer_past[0].device]),
             )
-            for layer_past in past
+            for layer_past in standardized_past
         )
-        return reordered_past
+        return self._convert_to_rw_cache(reordered_past)
 
 
-@add_start_docstrings(
-    """
-    The Falcon Model transformer with a sequence classification head on top (linear layer).
-
-    [`FalconForSequenceClassification`] uses the last token in order to do the classification, as other causal models
-    (e.g. GPT-1) do.
-
-    Since it does classification on the last token, it requires to know the position of the last token. If a
-    `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If
-    no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
-    padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in
-    each row of the batch).
-    """,
-    FALCON_START_DOCSTRING,
-)
-class FalconForSequenceClassification(FalconPreTrainedModel):
-    def __init__(self, config: FalconConfig):
+class RWForSequenceClassification(RWPreTrainedModel):
+    _keys_to_ignore_on_load_missing = [r"h.*.self_attention.scale_mask_softmax.causal_mask", r"lm_head.weight"]
+
+    def __init__(self, config: RWConfig):
         super().__init__(config)
         self.num_labels = config.num_labels
-        self.transformer = FalconModel(config)
+        self.transformer = RWModel(config)
         self.score = nn.Linear(config.hidden_size, config.num_labels, bias=False)
 
         # Initialize weights and apply final processing
         self.post_init()
 
-    @add_start_docstrings_to_model_forward(FALCON_INPUTS_DOCSTRING)
-    @add_code_sample_docstrings(
-        checkpoint=_CHECKPOINT_FOR_DOC,
-        output_type=SequenceClassifierOutputWithPast,
-        config_class=_CONFIG_FOR_DOC,
-    )
     def forward(
         self,
         input_ids: Optional[torch.LongTensor] = None,
@@ -1004,6 +845,7 @@ class FalconForSequenceClassification(FalconPreTrainedModel):
         output_attentions: Optional[bool] = None,
         output_hidden_states: Optional[bool] = None,
         return_dict: Optional[bool] = None,
+        **deprecated_arguments,
     ) -> Union[Tuple[torch.Tensor], SequenceClassifierOutputWithPast]:
         r"""
         labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
@@ -1011,6 +853,15 @@ class FalconForSequenceClassification(FalconPreTrainedModel):
             config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
             `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
         """
+        if deprecated_arguments.pop("position_ids", False) is not False:
+            # `position_ids` could have been `torch.Tensor` or `None` so defaulting pop to `False` allows to detect if users were passing explicitly `None`
+            warnings.warn(
+                "`position_ids` have no functionality in BLOOM and will be removed in v5.0.0. You can safely ignore"
+                " passing `position_ids`.",
+                FutureWarning,
+            )
+        if len(deprecated_arguments) > 0:
+            raise ValueError(f"Got unexpected arguments: {deprecated_arguments}")
 
         return_dict = return_dict if return_dict is not None else self.config.use_return_dict
 
@@ -1085,22 +936,17 @@ class FalconForSequenceClassification(FalconPreTrainedModel):
         )
 
 
-@add_start_docstrings(
-    """
-    Falcon Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for
-    Named-Entity-Recognition (NER) tasks.
-    """,
-    FALCON_START_DOCSTRING,
-)
-class FalconForTokenClassification(FalconPreTrainedModel):
-    def __init__(self, config: FalconConfig):
+class RWForTokenClassification(RWPreTrainedModel):
+    _keys_to_ignore_on_load_missing = [r"h.*.self_attention.scale_mask_softmax.causal_mask", r"lm_head.weight"]
+
+    def __init__(self, config: RWConfig):
         super().__init__(config)
         self.num_labels = config.num_labels
 
-        self.transformer = FalconModel(config)
-        if getattr(config, "classifier_dropout", None) is not None:
+        self.transformer = RWModel(config)
+        if hasattr(config, "classifier_dropout") and config.classifier_dropout is not None:
             classifier_dropout = config.classifier_dropout
-        elif getattr(config, "hidden_dropout", None) is not None:
+        elif hasattr(config, "hidden_dropout") and config.hidden_dropout is not None:
             classifier_dropout = config.hidden_dropout
         else:
             classifier_dropout = 0.1
@@ -1110,12 +956,6 @@ class FalconForTokenClassification(FalconPreTrainedModel):
         # Initialize weights and apply final processing
         self.post_init()
 
-    @add_start_docstrings_to_model_forward(FALCON_INPUTS_DOCSTRING)
-    @add_code_sample_docstrings(
-        checkpoint=_CHECKPOINT_FOR_DOC,
-        output_type=TokenClassifierOutput,
-        config_class=_CONFIG_FOR_DOC,
-    )
     def forward(
         self,
         input_ids: Optional[torch.LongTensor] = None,
@@ -1128,6 +968,7 @@ class FalconForTokenClassification(FalconPreTrainedModel):
         output_attentions: Optional[bool] = None,
         output_hidden_states: Optional[bool] = None,
         return_dict: Optional[bool] = None,
+        **deprecated_arguments,
     ) -> Union[Tuple[torch.Tensor], TokenClassifierOutput]:
         r"""
         labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
@@ -1135,6 +976,15 @@ class FalconForTokenClassification(FalconPreTrainedModel):
             config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
             `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
         """
+        if deprecated_arguments.pop("position_ids", False) is not False:
+            # `position_ids` could have been `torch.Tensor` or `None` so defaulting pop to `False` allows to detect if users were passing explicitly `None`
+            warnings.warn(
+                "`position_ids` have no functionality in BLOOM and will be removed in v5.0.0. You can safely ignore"
+                " passing `position_ids`.",
+                FutureWarning,
+            )
+        if len(deprecated_arguments) > 0:
+            raise ValueError(f"Got unexpected arguments: {deprecated_arguments}")
 
         return_dict = return_dict if return_dict is not None else self.config.use_return_dict
 
@@ -1158,9 +1008,7 @@ class FalconForTokenClassification(FalconPreTrainedModel):
         if labels is not None:
             batch_size, seq_length = labels.shape
             loss_fct = CrossEntropyLoss()
-            loss = loss_fct(
-                logits.view(batch_size * seq_length, self.num_labels), labels.view(batch_size * seq_length)
-            )
+            loss = loss_fct(logits.view(batch_size * seq_length, self.num_labels), labels.view(batch_size * seq_length))
 
         if not return_dict:
             output = (logits,) + transformer_outputs[2:]
@@ -1174,27 +1022,22 @@ class FalconForTokenClassification(FalconPreTrainedModel):
         )
 
 
-@add_start_docstrings(
-    """
-    The Falcon Model transformer with a span classification head on top for extractive question-answering tasks like
-    SQuAD (a linear layers on top of the hidden-states output to compute `span start logits` and `span end logits`).
-    """,
-    FALCON_START_DOCSTRING,
-)
-class FalconForQuestionAnswering(FalconPreTrainedModel):
+class RWForQuestionAnswering(RWPreTrainedModel):
+    _keys_to_ignore_on_load_missing = [r"h.*.self_attention.scale_mask_softmax.causal_mask", r"lm_head.weight"]
+
     def __init__(self, config):
         super().__init__(config)
-        self.transformer = FalconModel(config)
+        self.transformer = RWModel(config)
         self.qa_outputs = nn.Linear(config.hidden_size, 2)
 
         # Initialize weights and apply final processing
         self.post_init()
 
-    @add_start_docstrings_to_model_forward(FALCON_INPUTS_DOCSTRING)
     def forward(
         self,
         input_ids: Optional[torch.LongTensor] = None,
         attention_mask: Optional[torch.FloatTensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
         head_mask: Optional[torch.FloatTensor] = None,
         inputs_embeds: Optional[torch.FloatTensor] = None,
         start_positions: Optional[torch.LongTensor] = None,
@@ -1218,6 +1061,7 @@ class FalconForQuestionAnswering(FalconPreTrainedModel):
         outputs = self.transformer(
             input_ids,
             attention_mask=attention_mask,
+            position_ids=position_ids,
             head_mask=head_mask,
             inputs_embeds=inputs_embeds,
             output_attentions=output_attentions,

tokenizer_config.json

@@ -1,11 +1,7 @@
 {
   "add_prefix_space": false,
   "eos_token": "<|endoftext|>",
-  "model_input_names": [
-    "input_ids",
-    "attention_mask"
-  ],
   "model_max_length": 2048,
   "special_tokens_map_file": null,
   "tokenizer_class": "PreTrainedTokenizerFast"
-}
+}