|
--- |
|
base_model: |
|
- meta-llama/Meta-Llama-3-8B-Instruct |
|
- meta-llama/Meta-Llama-3-8B |
|
library_name: transformers |
|
tags: |
|
- mergekit |
|
- merge |
|
license: other |
|
license_name: llama3 |
|
license_link: LICENSE |
|
extra_gated_prompt: >- |
|
### META LLAMA 3 COMMUNITY LICENSE AGREEMENT |
|
--- |
|
# Llama-3-DARE-v1-8B |
|
|
|
This is a merge of pre-trained language models created using [mergekit](https://github.com/cg123/mergekit). |
|
|
|
## Merge Details |
|
### Merge Method |
|
|
|
This model was merged using the [DARE](https://arxiv.org/abs/2311.03099) [TIES](https://arxiv.org/abs/2306.01708) merge method using [meta-llama/Meta-Llama-3-8B](https://huggingface.co/meta-llama/Meta-Llama-3-8B) as a base. |
|
|
|
### Models Merged |
|
|
|
The following models were included in the merge: |
|
* [meta-llama/Meta-Llama-3-8B-Instruct](https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct) |
|
|
|
### Configuration |
|
|
|
The following YAML configuration was used to produce this model: |
|
|
|
```yaml |
|
base_model: |
|
model: |
|
path: meta-llama/Meta-Llama-3-8B |
|
dtype: bfloat16 |
|
merge_method: dare_ties |
|
parameters: |
|
int8_mask: 1.0 |
|
normalize: 0.0 |
|
slices: |
|
- sources: |
|
- layer_range: [0, 32] |
|
model: |
|
model: |
|
path: meta-llama/Meta-Llama-3-8B-Instruct |
|
parameters: |
|
density: |
|
- filter: embed_token |
|
value: 0.07928115257656705 |
|
- filter: model.norm |
|
value: 0.8644655087483669 |
|
- filter: lm_head |
|
value: 0.8644655087483669 |
|
- filter: layers.0. |
|
value: 0.7023215975496551 |
|
- filter: layers.1. |
|
value: 0.5930151025883963 |
|
- filter: layers.2. |
|
value: 0.9235600142754441 |
|
- filter: layers.3. |
|
value: 0.30002273183016587 |
|
- filter: layers.4. |
|
value: 0.8718725802605614 |
|
- filter: layers.5. |
|
value: 0.8431819361408258 |
|
- filter: layers.6. |
|
value: 0.26586459452250655 |
|
- filter: layers.7. |
|
value: 0.20565052604836487 |
|
- filter: layers.8. |
|
value: 0.7865609337773642 |
|
- filter: layers.9. |
|
value: 0.8707795208055977 |
|
- filter: layers.10. |
|
value: 0.19128908483148235 |
|
- filter: layers.11. |
|
value: 0.925076317937204 |
|
- filter: layers.12. |
|
value: 0.3764120891817022 |
|
- filter: layers.13. |
|
value: 0.1537759874314265 |
|
- filter: layers.14. |
|
value: 0.8827103791905803 |
|
- filter: layers.15. |
|
value: 0.12994574083319627 |
|
- filter: layers.16. |
|
value: 0.9421382084146839 |
|
- filter: layers.17. |
|
value: 0.19784189207350633 |
|
- filter: layers.18. |
|
value: 0.9174803701757963 |
|
- filter: layers.19. |
|
value: 0.018511680579063827 |
|
- filter: layers.20. |
|
value: 0.6089701838828931 |
|
- filter: layers.21. |
|
value: 0.10667940525089978 |
|
- filter: layers.22. |
|
value: 0.7569387463687963 |
|
- filter: layers.23. |
|
value: 0.8617573261932969 |
|
- filter: layers.24. |
|
value: 0.8851543429501336 |
|
- filter: layers.25. |
|
value: 0.788944964233173 |
|
- filter: layers.26. |
|
value: 0.022117233408225547 |
|
- filter: layers.27. |
|
value: 0.9534983183659907 |
|
- filter: layers.28. |
|
value: 0.792604065605754 |
|
- filter: layers.29. |
|
value: 0.861671120755631 |
|
- filter: layers.30. |
|
value: 0.2948434371778307 |
|
- filter: layers.31. |
|
value: 0.18891036670059388 |
|
weight: |
|
- filter: embed_token |
|
value: 0.244303585022914 |
|
- filter: model.norm. |
|
value: 0.7921741064842713 |
|
- filter: lm_head |
|
value: 0.7921741064842713 |
|
- filter: layers.0. |
|
value: 0.7781818034133037 |
|
- filter: layers.1. |
|
value: 0.8847835098465024 |
|
- filter: layers.2. |
|
value: 0.6397880690508684 |
|
- filter: layers.3. |
|
value: 0.7937466750856308 |
|
- filter: layers.4. |
|
value: 0.8172928460203015 |
|
- filter: layers.5. |
|
value: 0.8653872213596285 |
|
- filter: layers.6. |
|
value: 0.9693001060347038 |
|
- filter: layers.7. |
|
value: 0.8929433929262818 |
|
- filter: layers.8. |
|
value: 0.8148535632999063 |
|
- filter: layers.9. |
|
value: 0.9540888550674099 |
|
- filter: layers.10. |
|
value: 0.9588139020445848 |
|
- filter: layers.11. |
|
value: 0.9210433265234956 |
|
- filter: layers.12. |
|
value: 0.7406799222922607 |
|
- filter: layers.13. |
|
value: 0.6955899334998573 |
|
- filter: layers.14. |
|
value: 0.15982240434953007 |
|
- filter: layers.15. |
|
value: 0.911878214829457 |
|
- filter: layers.16. |
|
value: 0.03946607654759535 |
|
- filter: layers.17. |
|
value: 0.8322149510147696 |
|
- filter: layers.18. |
|
value: 0.3733540416200388 |
|
- filter: layers.19. |
|
value: 0.6932851946473836 |
|
- filter: layers.20. |
|
value: 0.8918513138412437 |
|
- filter: layers.21. |
|
value: 0.08682354503629197 |
|
- filter: layers.22. |
|
value: 0.017627513785178843 |
|
- filter: layers.23. |
|
value: 0.5246168031857519 |
|
- filter: layers.24. |
|
value: 0.9561735656673052 |
|
- filter: layers.25. |
|
value: 0.9036966249098499 |
|
- filter: layers.26. |
|
value: 0.3270824190389672 |
|
- filter: layers.27. |
|
value: 0.8585505386205572 |
|
- filter: layers.28. |
|
value: 0.768055374051272 |
|
- filter: layers.29. |
|
value: 0.8343611910799547 |
|
- filter: layers.30. |
|
value: 0.9794887267657514 |
|
- filter: layers.31. |
|
value: 0.9143801295721963 |
|
- layer_range: [0, 32] |
|
model: |
|
model: |
|
path: meta-llama/Meta-Llama-3-8B |
|
parameters: |
|
density: |
|
- filter: embed_token |
|
value: 0.8288618306009948 |
|
- filter: model.norm |
|
value: 0.23292234562975161 |
|
- filter: lm_head |
|
value: 0.23292234562975161 |
|
- filter: layers.0. |
|
value: 0.372454056194375 |
|
- filter: layers.1. |
|
value: 0.14253218919354635 |
|
- filter: layers.2. |
|
value: 0.3078424546105084 |
|
- filter: layers.3. |
|
value: 0.25930581838885686 |
|
- filter: layers.4. |
|
value: 0.978321706776098 |
|
- filter: layers.5. |
|
value: 0.05609446797559111 |
|
- filter: layers.6. |
|
value: 0.24314602951396613 |
|
- filter: layers.7. |
|
value: 0.911885464400789 |
|
- filter: layers.8. |
|
value: 0.7414851452758723 |
|
- filter: layers.9. |
|
value: 0.18093642039158353 |
|
- filter: layers.10. |
|
value: 0.9084501788509814 |
|
- filter: layers.11. |
|
value: 0.2484987778735853 |
|
- filter: layers.12. |
|
value: 0.02864842329393736 |
|
- filter: layers.13. |
|
value: 0.035996105784139906 |
|
- filter: layers.14. |
|
value: 0.022814159286914156 |
|
- filter: layers.15. |
|
value: 0.8982289455883636 |
|
- filter: layers.16. |
|
value: 0.6749279016774873 |
|
- filter: layers.17. |
|
value: 0.2231666521672218 |
|
- filter: layers.18. |
|
value: 0.08042893992871515 |
|
- filter: layers.19. |
|
value: 0.9245191619395147 |
|
- filter: layers.20. |
|
value: 0.10245727381170405 |
|
- filter: layers.21. |
|
value: 0.6351565505035355 |
|
- filter: layers.22. |
|
value: 0.13631302757594085 |
|
- filter: layers.23. |
|
value: 0.02093704907904486 |
|
- filter: layers.24. |
|
value: 0.046387358073609594 |
|
- filter: layers.25. |
|
value: 0.39078092530283026 |
|
- filter: layers.26. |
|
value: 0.6422945171240722 |
|
- filter: layers.27. |
|
value: 0.9356630912083387 |
|
- filter: layers.28. |
|
value: 0.20795911007902343 |
|
- filter: layers.29. |
|
value: 0.10607547240686495 |
|
- filter: layers.30. |
|
value: 0.0142610579064266 |
|
- filter: layers.31. |
|
value: 0.9041406411956623 |
|
weight: |
|
- filter: embed_token |
|
value: 0.11199251330860797 |
|
- filter: model.norm. |
|
value: 0.14400396011333005 |
|
- filter: lm_head |
|
value: 0.14400396011333005 |
|
- filter: layers.0. |
|
value: 0.04471455188049395 |
|
- filter: layers.1. |
|
value: 0.9523895147273159 |
|
- filter: layers.2. |
|
value: 0.8434787249757318 |
|
- filter: layers.3. |
|
value: 0.8016501716390354 |
|
- filter: layers.4. |
|
value: 0.19555927362126765 |
|
- filter: layers.5. |
|
value: 0.11657419158719552 |
|
- filter: layers.6. |
|
value: 0.08984925597011956 |
|
- filter: layers.7. |
|
value: 0.2629227995683227 |
|
- filter: layers.8. |
|
value: 0.9596221742529512 |
|
- filter: layers.9. |
|
value: 0.754944626119186 |
|
- filter: layers.10. |
|
value: 0.6975471541191236 |
|
- filter: layers.11. |
|
value: 0.7057432892778266 |
|
- filter: layers.12. |
|
value: 0.8718457125948287 |
|
- filter: layers.13. |
|
value: 0.08609329339067016 |
|
- filter: layers.14. |
|
value: 0.062238388979337526 |
|
- filter: layers.15. |
|
value: 0.06582279783729371 |
|
- filter: layers.16. |
|
value: 0.16911181845635778 |
|
- filter: layers.17. |
|
value: 0.2734099561761782 |
|
- filter: layers.18. |
|
value: 0.21849327205885294 |
|
- filter: layers.19. |
|
value: 0.128433065340601 |
|
- filter: layers.20. |
|
value: 0.6468521489194113 |
|
- filter: layers.21. |
|
value: 0.16152478595976968 |
|
- filter: layers.22. |
|
value: 0.18505624743492918 |
|
- filter: layers.23. |
|
value: 0.5230576418780342 |
|
- filter: layers.24. |
|
value: 0.6970657976288235 |
|
- filter: layers.25. |
|
value: 0.9119218240518024 |
|
- filter: layers.26. |
|
value: 0.250155176705362 |
|
- filter: layers.27. |
|
value: 0.9157060215586246 |
|
- filter: layers.28. |
|
value: 0.9021879089954816 |
|
- filter: layers.29. |
|
value: 0.1541974948279516 |
|
- filter: layers.30. |
|
value: 0.8388637604851198 |
|
- filter: layers.31. |
|
value: 0.06372200149189318 |
|
``` |
|
|
