birgermoell commited on
Commit
d0624ed
1 Parent(s): b7a3382

Trained model upload

Browse files
README.md ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ language: no
3
+ widget:
4
+ - text: "Det er flott"
5
+ ---
6
+
7
+ # GPT2-svenska-wikipedia
8
+ A norwegian GPT2 style model trained using Flax CLM pipeline on the Norwegian
9
+ part of the wiki40b dataset.
10
+
11
+ https://huggingface.co/datasets/wiki40b
12
+
13
+ ## Data cleaning and preprocessing
14
+ The data was cleaned and preprocessed using the following script. Make sure to install depencies for beam_runner to make the dataset work.
15
+
16
+ ```python
17
+ from datasets import load_dataset
18
+ def load_and_clean_wiki():
19
+ dataset = load_dataset('wiki40b', 'no', beam_runner='DirectRunner', split="train")
20
+ #dataset = load_dataset('wiki40b', 'sv', beam_runner='DirectRunner')
21
+ dataset = dataset.remove_columns(['wikidata_id', 'version_id'])
22
+ filtered_dataset = dataset.map(filter_wikipedia)
23
+ # filtered_dataset[:3]
24
+ # print(filtered_dataset[:3])
25
+ return filtered_dataset
26
+
27
+ def filter_wikipedia(batch):
28
+ batch["text"] = " ".join(batch["text"].split("\n_START_SECTION_\n"))
29
+ batch["text"] = " ".join(batch["text"].split("\n_START_ARTICLE_\n"))
30
+ batch["text"] = " ".join(batch["text"].split("\n_START_ARTICLE_\n"))
31
+ batch["text"] = " ".join(batch["text"].split("\n_START_PARAGRAPH_\n"))
32
+ batch["text"] = " ".join(batch["text"].split("_NEWLINE_"))
33
+ batch["text"] = " ".join(batch["text"].split("\xa0"))
34
+ return batch
35
+ ```
36
+
37
+ ## Training script
38
+ The following training script was used to train the model.
39
+ ```bash
40
+ ./run_clm_flax.py --output_dir="${MODEL_DIR}" --model_type="gpt2" --config_name="${MODEL_DIR}" --tokenizer_name="${MODEL_DIR}" --dataset_name="wiki40b" --dataset_config_name="no" --do_train --do_eval --block_size="512" --per_device_train_batch_size="64" --per_device_eval_batch_size="64" --learning_rate="5e-3" --warmup_steps="1000" --adam_beta1="0.9" --adam_beta2="0.98" --weight_decay="0.01" --overwrite_output_dir --num_train_epochs="20" --logging_steps="500" --save_steps="1000" --eval_steps="2500" --push_to_hub
41
+ ```
42
+
added_tokens.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"<|endoftext|>": 50265}
config.json ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": ".",
3
+ "activation_function": "gelu_new",
4
+ "architectures": [
5
+ "GPT2LMHeadModel"
6
+ ],
7
+ "attn_pdrop": 0.0,
8
+ "bos_token_id": 50256,
9
+ "embd_pdrop": 0.0,
10
+ "eos_token_id": 50256,
11
+ "gradient_checkpointing": false,
12
+ "initializer_range": 0.02,
13
+ "layer_norm_epsilon": 1e-05,
14
+ "model_type": "gpt2",
15
+ "n_ctx": 1024,
16
+ "n_embd": 768,
17
+ "n_head": 12,
18
+ "n_inner": null,
19
+ "n_layer": 12,
20
+ "n_positions": 1024,
21
+ "resid_pdrop": 0.0,
22
+ "scale_attn_weights": true,
23
+ "summary_activation": null,
24
+ "summary_first_dropout": 0.1,
25
+ "summary_proj_to_labels": true,
26
+ "summary_type": "cls_index",
27
+ "summary_use_proj": true,
28
+ "task_specific_params": {
29
+ "text-generation": {
30
+ "do_sample": true,
31
+ "max_length": 50
32
+ }
33
+ },
34
+ "transformers_version": "4.8.2",
35
+ "use_cache": true,
36
+ "vocab_size": 50257
37
+ }
evaluate.py ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ from transformers import GPT2Tokenizer, GPT2Model, FlaxGPT2LMHeadModel, GPT2LMHeadModel, pipeline, set_seed
2
+
3
+ tokenizer = GPT2Tokenizer.from_pretrained("flax-community/swe-gpt-wiki")
4
+ model = GPT2LMHeadModel.from_pretrained("flax-community/swe-gpt-wiki")
5
+
6
+
7
+ generator = pipeline('text-generation', model=model, tokenizer=tokenizer)
8
+ set_seed(42)
9
+ result = generator("Det er flott", max_length=150, num_return_sequences=5)
10
+ print(result)
events.out.tfevents.1626214676.t1v-n-905ccd3e-w-0.93628.3.v2 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:96eea81b96a95a8ddf5c516912c6686f9afd98236db97c8c2d3d62e9273e1414
3
+ size 735617
flax_model.msgpack ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:112a250b6061edbb1afec25e1f9c2dc844be6be67471b95d2e48846c6bdc828d
3
+ size 497764120
make_config.py ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ from transformers import GPT2Config
2
+
3
+ model_dir = "./swe-gpt-wiki" # ${MODEL_DIR}
4
+
5
+ config = GPT2Config.from_pretrained("gpt2", resid_pdrop=0.0, embd_pdrop=0.0, attn_pdrop=0.0)
6
+ config.save_pretrained(model_dir)
merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0ad3e5639a357aabb61433cc806a336e4c10c4430c1b00bc93a5ec6e1f86d9f0
3
+ size 510401385
save_model.py ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ from transformers import AutoTokenizer, GPT2LMHeadModel
2
+ '''
3
+ This is a script to convert the Jax model and the tokenizer to Pytorch model
4
+ '''
5
+ model = GPT2LMHeadModel.from_pretrained(".", from_flax=True)
6
+ model.save_pretrained(".")
7
+ tokenizer = AutoTokenizer.from_pretrained(".")
8
+ tokenizer.save_pretrained(".")
special_tokens_map.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"bos_token": "<|endoftext|>", "eos_token": "<|endoftext|>", "unk_token": "<|endoftext|>"}
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"unk_token": "<|endoftext|>", "bos_token": "<|endoftext|>", "eos_token": "<|endoftext|>", "add_prefix_space": false, "special_tokens_map_file": null, "name_or_path": ".", "tokenizer_class": "GPT2Tokenizer"}
vocab.json ADDED
The diff for this file is too large to render. See raw diff