tianyuz commited on
Commit
d93a6e4
1 Parent(s): d8c946c

* 2023/03/20 Update the model weight and config files such that it can be loaded via Huggingface's official GPT-NeoX implementation.

Browse files
Files changed (3) hide show
  1. README.md +5 -7
  2. config.json +11 -29
  3. pytorch_model.bin +2 -2
README.md CHANGED
@@ -22,17 +22,15 @@ inference: false
22
 
23
  This repository provides a small-sized Japanese GPT-NeoX model. The model was trained using code based on [EleutherAI/gpt-neox](https://github.com/EleutherAI/gpt-neox).
24
 
25
- # How to use the model
 
26
 
27
- *NOTE:*
28
- * Use `T5Tokenizer` to load its corresponding tokenizer.
29
- * The files for modeling and configuration are not in the Transformers library yet. In order to load the model, use files from [this PR in EleutherAI/gpt-neox](https://github.com/EleutherAI/gpt-neox/pull/480).
30
 
31
  ~~~~
32
- from transformers import T5Tokenizer
33
- from modeling_gpt_neox import GPTNeoXForCausalLM
34
 
35
- tokenizer = T5Tokenizer.from_pretrained("rinna/japanese-gpt-neox-small")
36
  model = GPTNeoXForCausalLM.from_pretrained("rinna/japanese-gpt-neox-small")
37
  ~~~~
38
 
 
22
 
23
  This repository provides a small-sized Japanese GPT-NeoX model. The model was trained using code based on [EleutherAI/gpt-neox](https://github.com/EleutherAI/gpt-neox).
24
 
25
+ # Update log
26
+ * 2023/03/20 Update the model weight and config files such that it can be loaded via Huggingface's official GPT-NeoX implementation.
27
 
28
+ # How to use the model
 
 
29
 
30
  ~~~~
31
+ from transformers import AutoTokenizer, AutoModelForCausalLM
 
32
 
33
+ tokenizer = AutoTokenizer.from_pretrained("rinna/japanese-gpt-neox-small", use_fast=False)
34
  model = GPTNeoXForCausalLM.from_pretrained("rinna/japanese-gpt-neox-small")
35
  ~~~~
36
 
config.json CHANGED
@@ -1,42 +1,24 @@
1
  {
2
- "activation_function": "gelu",
3
  "architectures": [
4
  "GPTNeoXForCausalLM"
5
  ],
6
- "attn_pdrop": 0.0,
7
  "bos_token_id": 2,
8
- "embd_pdrop": 0.0,
9
  "eos_token_id": 3,
10
- "gpt_j_residual": false,
11
- "gradient_checkpointing": false,
12
  "initializer_range": 0.02,
13
- "layer_norm_epsilon": 1e-05,
14
- "lm_head_bias": false,
15
- "model_type": "gpt-neox",
16
- "n_embd": 768,
17
- "n_head": 12,
18
- "n_inner": null,
19
- "n_layer": 12,
20
- "n_positions": 2048,
21
- "resid_pdrop": 0.0,
22
- "rotary": true,
23
- "rotary_dim": null,
24
- "scale_attn_weights": true,
25
- "summary_activation": null,
26
- "summary_first_dropout": 0.1,
27
- "summary_proj_to_labels": true,
28
- "summary_type": "cls_index",
29
- "summary_use_proj": true,
30
- "task_specific_params": {
31
- "text-generation": {
32
- "do_sample": true,
33
- "max_length": 50,
34
- "temperature": 1.0
35
- }
36
- },
37
  "tie_word_embeddings": false,
38
  "tokenizer_class": "T5Tokenizer",
39
  "torch_dtype": "float32",
40
  "use_cache": true,
 
41
  "vocab_size": 44416
42
  }
 
1
  {
 
2
  "architectures": [
3
  "GPTNeoXForCausalLM"
4
  ],
 
5
  "bos_token_id": 2,
 
6
  "eos_token_id": 3,
7
+ "hidden_act": "gelu",
8
+ "hidden_size": 768,
9
  "initializer_range": 0.02,
10
+ "intermediate_size": 3072,
11
+ "layer_norm_eps": 1e-05,
12
+ "max_position_embeddings": 2048,
13
+ "model_type": "gpt_neox",
14
+ "num_attention_heads": 12,
15
+ "num_hidden_layers": 12,
16
+ "rotary_emb_base": 10000,
17
+ "rotary_pct": 1.0,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
18
  "tie_word_embeddings": false,
19
  "tokenizer_class": "T5Tokenizer",
20
  "torch_dtype": "float32",
21
  "use_cache": true,
22
+ "use_parallel_residual": false,
23
  "vocab_size": 44416
24
  }
pytorch_model.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:b59cd532d22f3dad2a3afea49ba4eb66c084ca13e3e703bf3090a558ac9a1ced
3
- size 663506729
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:58e3f25be62e0e5e860b36a4e9b1ef0bd9299378a488c1dafb48526222076d5b
3
+ size 663506641