Hugging Face's logo

bs-la
/
bloom-560m_my_sft_100000samples_-1vocab_original-frozen

Text Generation
Transformers
PyTorch
bloom
text-generation-inference
Inference Endpoints
Model card Files Community
1
khalidalt commited on
Commit
fc3e573
·
1 Parent(s): f4233a2

Push to HF hub

Browse files
Files changed (16) hide show
  1. .gitattributes +1 -0
  2. .gitignore +3 -0
  3. all_results.json +14 -0
  4. config.json +38 -0
  5. eval_results.json +9 -0
  6. pytorch_diff.bin +3 -0
  7. pytorch_model.bin +3 -0
  8. special_tokens_map.json +1 -0
  9. tokenizer.json +3 -0
  10. tokenizer_config.json +1 -0
  11. train_results.json +8 -0
  12. trainer_state.json +135 -0
  13. training_args.bin +3 -0
  14. wikiann-my-results.txt +8 -0
  15. word_embeddings.pt +3 -0
  16. word_embeddings_layernorm.pt +3 -0
.gitattributes CHANGED
@@ -30,3 +30,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
30
  *.zip filter=lfs diff=lfs merge=lfs -text
31
  *.zst filter=lfs diff=lfs merge=lfs -text
32
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
30
  *.zip filter=lfs diff=lfs merge=lfs -text
31
  *.zst filter=lfs diff=lfs merge=lfs -text
32
  *tfevents* filter=lfs diff=lfs merge=lfs -text
33
+ tokenizer.json filter=lfs diff=lfs merge=lfs -text
.gitignore ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ checkpoint-*/
2
+ */pilot_*/
3
+ pilot_*/
all_results.json ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 0.4,
3
+ "eval_loss": 0.7879278063774109,
4
+ "eval_runtime": 2660.4642,
5
+ "eval_samples": 5000,
6
+ "eval_samples_per_second": 9.427,
7
+ "eval_steps_per_second": 4.714,
8
+ "perplexity": 2.198835289644634,
9
+ "train_loss": 0.8401528125,
10
+ "train_runtime": 72520.074,
11
+ "train_samples": 100000,
12
+ "train_samples_per_second": 2.758,
13
+ "train_steps_per_second": 0.345
14
+ }
config.json ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "bigscience/bloom-560m",
3
+ "adapters": {
4
+ "adapters": {},
5
+ "config_map": {},
6
+ "fusion_config_map": {},
7
+ "fusions": {}
8
+ },
9
+ "apply_residual_connection_post_layernorm": false,
10
+ "architectures": [
11
+ "BloomForCausalLM"
12
+ ],
13
+ "attention_dropout": 0.0,
14
+ "attention_softmax_in_fp32": true,
15
+ "bias_dropout_fusion": true,
16
+ "bos_token_id": 1,
17
+ "eos_token_id": 2,
18
+ "hidden_dropout": 0.0,
19
+ "hidden_size": 1024,
20
+ "initializer_range": 0.02,
21
+ "layer_norm_epsilon": 1e-05,
22
+ "masked_softmax_fusion": true,
23
+ "model_type": "bloom",
24
+ "n_head": 16,
25
+ "n_inner": null,
26
+ "n_layer": 24,
27
+ "offset_alibi": 100,
28
+ "pad_token_id": 3,
29
+ "pretraining_tp": 1,
30
+ "skip_bias_add": true,
31
+ "skip_bias_add_qkv": false,
32
+ "slow_but_exact": false,
33
+ "torch_dtype": "float32",
34
+ "transformers_version": "4.20.0.dev0",
35
+ "unk_token_id": 0,
36
+ "use_cache": true,
37
+ "vocab_size": 250880
38
+ }
eval_results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 0.4,
3
+ "eval_loss": 0.7879278063774109,
4
+ "eval_runtime": 2660.4642,
5
+ "eval_samples": 5000,
6
+ "eval_samples_per_second": 9.427,
7
+ "eval_steps_per_second": 4.714,
8
+ "perplexity": 2.198835289644634
9
+ }
pytorch_diff.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c9099ff5427b77a6c77bd943300a6109f72de9ce131c500984d1b54ca6aef834
3
+ size 22518653
pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:855bd07f188449f1b814b82d9dd4eb12122aa9ec51f9d1d510ee8716b9ba0576
3
+ size 2236955191
special_tokens_map.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"bos_token": "<s>", "eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>"}
tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8f6efc66e73f1fd69da4f436e48befb519fdff3fe18910850c1d41bd862293a5
3
+ size 14500443
tokenizer_config.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"unk_token": "<unk>", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "<pad>", "add_prefix_space": false, "name_or_path": "bigscience/bloom-560m", "special_tokens_map_file": null, "padding_side": "left", "tokenizer_class": "BloomTokenizer"}
train_results.json ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 0.4,
3
+ "train_loss": 0.8401528125,
4
+ "train_runtime": 72520.074,
5
+ "train_samples": 100000,
6
+ "train_samples_per_second": 2.758,
7
+ "train_steps_per_second": 0.345
8
+ }
trainer_state.json ADDED
@@ -0,0 +1,135 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": 0.7879278063774109,
3
+ "best_model_checkpoint": "/home/khalid/Documents/github_rep/bigscience/data/processed/35/bloom-560m_my_sft_100000samples_-1vocab_original-frozen/checkpoint-25000",
4
+ "epoch": 0.39631741854686253,
5
+ "global_step": 25000,
6
+ "is_hyper_param_search": false,
7
+ "is_local_process_zero": true,
8
+ "is_world_process_zero": true,
9
+ "log_history": [
10
+ {
11
+ "epoch": 0.04,
12
+ "l1_reg_loss": 0.0,
13
+ "learning_rate": 9e-05,
14
+ "loss": 1.0457,
15
+ "step": 2500
16
+ },
17
+ {
18
+ "epoch": 0.08,
19
+ "l1_reg_loss": 0.0,
20
+ "learning_rate": 8e-05,
21
+ "loss": 0.8994,
22
+ "step": 5000
23
+ },
24
+ {
25
+ "epoch": 0.08,
26
+ "eval_loss": 0.8788017630577087,
27
+ "eval_runtime": 2668.4849,
28
+ "eval_samples_per_second": 9.399,
29
+ "eval_steps_per_second": 4.7,
30
+ "step": 5000
31
+ },
32
+ {
33
+ "epoch": 0.12,
34
+ "l1_reg_loss": 0.0,
35
+ "learning_rate": 7e-05,
36
+ "loss": 0.8556,
37
+ "step": 7500
38
+ },
39
+ {
40
+ "epoch": 0.16,
41
+ "l1_reg_loss": 0.0,
42
+ "learning_rate": 6e-05,
43
+ "loss": 0.8299,
44
+ "step": 10000
45
+ },
46
+ {
47
+ "epoch": 0.16,
48
+ "eval_loss": 0.8276682496070862,
49
+ "eval_runtime": 2661.8009,
50
+ "eval_samples_per_second": 9.423,
51
+ "eval_steps_per_second": 4.711,
52
+ "step": 10000
53
+ },
54
+ {
55
+ "epoch": 0.2,
56
+ "l1_reg_loss": 0.0,
57
+ "learning_rate": 5e-05,
58
+ "loss": 0.815,
59
+ "step": 12500
60
+ },
61
+ {
62
+ "epoch": 0.24,
63
+ "l1_reg_loss": 0.0,
64
+ "learning_rate": 4e-05,
65
+ "loss": 0.8019,
66
+ "step": 15000
67
+ },
68
+ {
69
+ "epoch": 0.24,
70
+ "eval_loss": 0.804674506187439,
71
+ "eval_runtime": 2658.9167,
72
+ "eval_samples_per_second": 9.433,
73
+ "eval_steps_per_second": 4.717,
74
+ "step": 15000
75
+ },
76
+ {
77
+ "epoch": 0.28,
78
+ "l1_reg_loss": 0.0,
79
+ "learning_rate": 3e-05,
80
+ "loss": 0.7965,
81
+ "step": 17500
82
+ },
83
+ {
84
+ "epoch": 0.32,
85
+ "l1_reg_loss": 0.0,
86
+ "learning_rate": 2e-05,
87
+ "loss": 0.7875,
88
+ "step": 20000
89
+ },
90
+ {
91
+ "epoch": 0.32,
92
+ "eval_loss": 0.7925558686256409,
93
+ "eval_runtime": 2660.1365,
94
+ "eval_samples_per_second": 9.428,
95
+ "eval_steps_per_second": 4.714,
96
+ "step": 20000
97
+ },
98
+ {
99
+ "epoch": 0.36,
100
+ "l1_reg_loss": 0.0,
101
+ "learning_rate": 1e-05,
102
+ "loss": 0.7866,
103
+ "step": 22500
104
+ },
105
+ {
106
+ "epoch": 0.4,
107
+ "l1_reg_loss": 0.0,
108
+ "learning_rate": 0.0,
109
+ "loss": 0.7835,
110
+ "step": 25000
111
+ },
112
+ {
113
+ "epoch": 0.4,
114
+ "eval_loss": 0.7879278063774109,
115
+ "eval_runtime": 2660.4205,
116
+ "eval_samples_per_second": 9.427,
117
+ "eval_steps_per_second": 4.714,
118
+ "step": 25000
119
+ },
120
+ {
121
+ "epoch": 0.4,
122
+ "step": 25000,
123
+ "total_flos": 3.714827943936e+17,
124
+ "train_loss": 0.8401528125,
125
+ "train_runtime": 72520.074,
126
+ "train_samples_per_second": 2.758,
127
+ "train_steps_per_second": 0.345
128
+ }
129
+ ],
130
+ "max_steps": 25000,
131
+ "num_train_epochs": 1,
132
+ "total_flos": 3.714827943936e+17,
133
+ "trial_name": null,
134
+ "trial_params": null
135
+ }
training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b2d97da5990671ba193c059ce37424f266fecd079b4a5aab5bb7d1d16b054cdb
3
+ size 3375
wikiann-my-results.txt ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ ==================================================
2
+ Results
3
+ ==================================================
4
+ Model: /home/khalid/Documents/github_rep/bigscience/data/processed/32/bloom-560m_my_sft_100000samples_-1vocab_original-frozen/
5
+ [0.09599999999999999, 0.0855263157894737, 0.13919413919413917, 0.05393258426966292, 0.07772020725388601, 0.06896551724137931, 0.03636363636363636, 0.04294478527607362, 0.07162534435261708, 0.0781758957654723]
6
+ 7.50
7
+ 2.78
8
+ ==================================================
word_embeddings.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:174ed618237771e5906be0e8d70c568de63633f3bb5e8a1e303bbdbaeaedc1ca
3
+ size 1027605867
word_embeddings_layernorm.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:10917f86841a4f322406bd72ba2e4ae8e4780aaf462c98a76eca01e0c5fbc893
3
+ size 9703