Hugging Face's logo

bs-la
/
bloom-560m_az_sft_10000samples_-1vocab_original-frozen

Text Generation
Transformers
PyTorch
bloom
text-generation-inference
Inference Endpoints
Model card Files Community
1
yongzx commited on
Commit
f425ef9
·
1 Parent(s): d12344a

Push to HF hub

Browse files
Files changed (16) hide show
  1. .gitattributes +1 -0
  2. .gitignore +3 -0
  3. all_results.json +14 -0
  4. config.json +39 -0
  5. eval_results.json +9 -0
  6. pytorch_diff.bin +3 -0
  7. pytorch_model.bin +3 -0
  8. special_tokens_map.json +1 -0
  9. tokenizer.json +3 -0
  10. tokenizer_config.json +1 -0
  11. train_results.json +8 -0
  12. trainer_state.json +135 -0
  13. training_args.bin +3 -0
  14. wikiann-az-results.txt +8 -0
  15. word_embeddings.pt +3 -0
  16. word_embeddings_layernorm.pt +3 -0
.gitattributes CHANGED
@@ -30,3 +30,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
30
  *.zip filter=lfs diff=lfs merge=lfs -text
31
  *.zst filter=lfs diff=lfs merge=lfs -text
32
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
30
  *.zip filter=lfs diff=lfs merge=lfs -text
31
  *.zst filter=lfs diff=lfs merge=lfs -text
32
  *tfevents* filter=lfs diff=lfs merge=lfs -text
33
+ tokenizer.json filter=lfs diff=lfs merge=lfs -text
.gitignore ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ checkpoint-*/
2
+ */pilot_*/
3
+ pilot_*/
all_results.json ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 19.86,
3
+ "eval_loss": 2.3477182388305664,
4
+ "eval_runtime": 514.3522,
5
+ "eval_samples": 4906,
6
+ "eval_samples_per_second": 9.538,
7
+ "eval_steps_per_second": 4.769,
8
+ "perplexity": 10.461671434354535,
9
+ "train_loss": 2.42385498046875,
10
+ "train_runtime": 61408.2472,
11
+ "train_samples": 10000,
12
+ "train_samples_per_second": 3.257,
13
+ "train_steps_per_second": 0.407
14
+ }
config.json ADDED
@@ -0,0 +1,39 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "bigscience/bloom-350m",
3
+ "adapters": {
4
+ "adapters": {},
5
+ "config_map": {},
6
+ "fusion_config_map": {},
7
+ "fusions": {}
8
+ },
9
+ "apply_residual_connection_post_layernorm": false,
10
+ "architectures": [
11
+ "BloomForCausalLM"
12
+ ],
13
+ "attention_dropout": 0.0,
14
+ "attention_softmax_in_fp32": true,
15
+ "bias_dropout_fusion": true,
16
+ "bos_token_id": 1,
17
+ "eos_token_id": 2,
18
+ "hidden_dropout": 0.0,
19
+ "hidden_size": 1024,
20
+ "initializer_range": 0.02,
21
+ "layer_norm_epsilon": 1e-05,
22
+ "masked_softmax_fusion": true,
23
+ "model_type": "bloom",
24
+ "n_head": 16,
25
+ "n_inner": null,
26
+ "n_layer": 24,
27
+ "offset_alibi": 100,
28
+ "pad_token_id": 3,
29
+ "pretraining_tp": 1,
30
+ "seq_length": 2048,
31
+ "skip_bias_add": true,
32
+ "skip_bias_add_qkv": false,
33
+ "slow_but_exact": false,
34
+ "torch_dtype": "float32",
35
+ "transformers_version": "4.20.0.dev0",
36
+ "unk_token_id": 0,
37
+ "use_cache": true,
38
+ "vocab_size": 250880
39
+ }
eval_results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 19.86,
3
+ "eval_loss": 2.3477182388305664,
4
+ "eval_runtime": 514.3522,
5
+ "eval_samples": 4906,
6
+ "eval_samples_per_second": 9.538,
7
+ "eval_steps_per_second": 4.769,
8
+ "perplexity": 10.461671434354535
9
+ }
pytorch_diff.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0091b1a5642a01858d1aa1bc7b003758b297a937b04c17d14c9625fe10e3714c
3
+ size 22506685
pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b18bfd59140ba8bcdf1bc69cd403d6fc06514a00bba834982ab9f804932a0c23
3
+ size 2236955191
special_tokens_map.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"bos_token": "<s>", "eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>"}
tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8f6efc66e73f1fd69da4f436e48befb519fdff3fe18910850c1d41bd862293a5
3
+ size 14500443
tokenizer_config.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"unk_token": "<unk>", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "<pad>", "add_prefix_space": false, "name_or_path": "bigscience/bloom-350m", "special_tokens_map_file": null, "padding_side": "left", "tokenizer_class": "BloomTokenizer"}
train_results.json ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 19.86,
3
+ "train_loss": 2.42385498046875,
4
+ "train_runtime": 61408.2472,
5
+ "train_samples": 10000,
6
+ "train_samples_per_second": 3.257,
7
+ "train_steps_per_second": 0.407
8
+ }
trainer_state.json ADDED
@@ -0,0 +1,135 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": 2.3477182388305664,
3
+ "best_model_checkpoint": "/users/zyong2/data/zyong2/bigscience/data/processed/024/bloom-350m_az_sft_10000samples_-1vocab_original-frozen/checkpoint-25000",
4
+ "epoch": 19.856519150625125,
5
+ "global_step": 25000,
6
+ "is_hyper_param_search": false,
7
+ "is_local_process_zero": true,
8
+ "is_world_process_zero": true,
9
+ "log_history": [
10
+ {
11
+ "epoch": 1.99,
12
+ "l1_reg_loss": 0.0,
13
+ "learning_rate": 9e-05,
14
+ "loss": 3.1946,
15
+ "step": 2500
16
+ },
17
+ {
18
+ "epoch": 3.97,
19
+ "l1_reg_loss": 0.0001,
20
+ "learning_rate": 8e-05,
21
+ "loss": 2.6396,
22
+ "step": 5000
23
+ },
24
+ {
25
+ "epoch": 3.97,
26
+ "eval_loss": 2.5872137546539307,
27
+ "eval_runtime": 514.4581,
28
+ "eval_samples_per_second": 9.536,
29
+ "eval_steps_per_second": 4.768,
30
+ "step": 5000
31
+ },
32
+ {
33
+ "epoch": 5.96,
34
+ "l1_reg_loss": 0.0001,
35
+ "learning_rate": 7e-05,
36
+ "loss": 2.4815,
37
+ "step": 7500
38
+ },
39
+ {
40
+ "epoch": 7.94,
41
+ "l1_reg_loss": 0.0001,
42
+ "learning_rate": 6e-05,
43
+ "loss": 2.3898,
44
+ "step": 10000
45
+ },
46
+ {
47
+ "epoch": 7.94,
48
+ "eval_loss": 2.439215898513794,
49
+ "eval_runtime": 514.6109,
50
+ "eval_samples_per_second": 9.533,
51
+ "eval_steps_per_second": 4.767,
52
+ "step": 10000
53
+ },
54
+ {
55
+ "epoch": 9.93,
56
+ "l1_reg_loss": 0.0001,
57
+ "learning_rate": 5e-05,
58
+ "loss": 2.3298,
59
+ "step": 12500
60
+ },
61
+ {
62
+ "epoch": 11.91,
63
+ "l1_reg_loss": 0.0001,
64
+ "learning_rate": 4e-05,
65
+ "loss": 2.287,
66
+ "step": 15000
67
+ },
68
+ {
69
+ "epoch": 11.91,
70
+ "eval_loss": 2.3796885013580322,
71
+ "eval_runtime": 514.3961,
72
+ "eval_samples_per_second": 9.537,
73
+ "eval_steps_per_second": 4.769,
74
+ "step": 15000
75
+ },
76
+ {
77
+ "epoch": 13.9,
78
+ "l1_reg_loss": 0.0001,
79
+ "learning_rate": 3e-05,
80
+ "loss": 2.254,
81
+ "step": 17500
82
+ },
83
+ {
84
+ "epoch": 15.89,
85
+ "l1_reg_loss": 0.0001,
86
+ "learning_rate": 2e-05,
87
+ "loss": 2.2339,
88
+ "step": 20000
89
+ },
90
+ {
91
+ "epoch": 15.89,
92
+ "eval_loss": 2.354508399963379,
93
+ "eval_runtime": 514.4981,
94
+ "eval_samples_per_second": 9.536,
95
+ "eval_steps_per_second": 4.768,
96
+ "step": 20000
97
+ },
98
+ {
99
+ "epoch": 17.87,
100
+ "l1_reg_loss": 0.0001,
101
+ "learning_rate": 1e-05,
102
+ "loss": 2.2184,
103
+ "step": 22500
104
+ },
105
+ {
106
+ "epoch": 19.86,
107
+ "l1_reg_loss": 0.0001,
108
+ "learning_rate": 0.0,
109
+ "loss": 2.21,
110
+ "step": 25000
111
+ },
112
+ {
113
+ "epoch": 19.86,
114
+ "eval_loss": 2.3477182388305664,
115
+ "eval_runtime": 514.3805,
116
+ "eval_samples_per_second": 9.538,
117
+ "eval_steps_per_second": 4.769,
118
+ "step": 25000
119
+ },
120
+ {
121
+ "epoch": 19.86,
122
+ "step": 25000,
123
+ "total_flos": 3.7165924872093696e+17,
124
+ "train_loss": 2.42385498046875,
125
+ "train_runtime": 61408.2472,
126
+ "train_samples_per_second": 3.257,
127
+ "train_steps_per_second": 0.407
128
+ }
129
+ ],
130
+ "max_steps": 25000,
131
+ "num_train_epochs": 20,
132
+ "total_flos": 3.7165924872093696e+17,
133
+ "trial_name": null,
134
+ "trial_params": null
135
+ }
training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b10603cd710f3bcfa48f2004adff9c85427b634f98a7c029323c0d19e2a90d87
3
+ size 3375
wikiann-az-results.txt ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ ==================================================
2
+ Results
3
+ ==================================================
4
+ Model: /users/zyong2/data/zyong2/bigscience/data/processed/024/bloom-350m_az_sft_10000samples_-1vocab_original-frozen
5
+ [0.4049122807017544, 0.3977110157367668, 0.37656099903938517, 0.397480755773268, 0.40575640575640576, 0.40650406504065045, 0.4002813928948294, 0.41371994342291374, 0.37447108603667134, 0.397887323943662]
6
+ 39.75
7
+ 1.20
8
+ ==================================================
word_embeddings.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:174ed618237771e5906be0e8d70c568de63633f3bb5e8a1e303bbdbaeaedc1ca
3
+ size 1027605867
word_embeddings_layernorm.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:10917f86841a4f322406bd72ba2e4ae8e4780aaf462c98a76eca01e0c5fbc893
3
+ size 9703