Trained model upload

README.md

@@ -0,0 +1,42 @@
+---
+language: no
+widget:
+- text: "Det er flott"
+---
+
+# GPT2-svenska-wikipedia
+A norwegian GPT2 style model trained using Flax CLM pipeline on the Norwegian
+part of the wiki40b dataset.
+
+https://huggingface.co/datasets/wiki40b
+
+## Data cleaning and preprocessing
+The data was cleaned and preprocessed using the following script. Make sure to install depencies for beam_runner to make the dataset work.
+
+```python
+from datasets import load_dataset
+def load_and_clean_wiki():
+    dataset = load_dataset('wiki40b', 'no', beam_runner='DirectRunner', split="train")
+    #dataset = load_dataset('wiki40b', 'sv', beam_runner='DirectRunner')
+    dataset = dataset.remove_columns(['wikidata_id', 'version_id'])
+    filtered_dataset = dataset.map(filter_wikipedia)
+    # filtered_dataset[:3]
+    # print(filtered_dataset[:3])
+    return filtered_dataset
+
+def filter_wikipedia(batch):
+    batch["text"] = " ".join(batch["text"].split("\n_START_SECTION_\n"))
+    batch["text"] = " ".join(batch["text"].split("\n_START_ARTICLE_\n"))
+    batch["text"] = " ".join(batch["text"].split("\n_START_ARTICLE_\n"))
+    batch["text"] = " ".join(batch["text"].split("\n_START_PARAGRAPH_\n"))
+    batch["text"] = " ".join(batch["text"].split("_NEWLINE_"))
+    batch["text"] = " ".join(batch["text"].split("\xa0"))
+    return batch
+```
+
+## Training script
+The following training script was used to train the model.
+```bash
+./run_clm_flax.py     --output_dir="${MODEL_DIR}"     --model_type="gpt2"     --config_name="${MODEL_DIR}"     --tokenizer_name="${MODEL_DIR}"     --dataset_name="wiki40b"     --dataset_config_name="no"     --do_train --do_eval     --block_size="512"     --per_device_train_batch_size="64"     --per_device_eval_batch_size="64"     --learning_rate="5e-3" --warmup_steps="1000"     --adam_beta1="0.9" --adam_beta2="0.98" --weight_decay="0.01"     --overwrite_output_dir     --num_train_epochs="20"     --logging_steps="500"     --save_steps="1000"     --eval_steps="2500"     --push_to_hub
+```
+

added_tokens.json

@@ -0,0 +1 @@
+{"<|endoftext|>": 50265}

config.json

@@ -0,0 +1,37 @@
+{
+  "_name_or_path": ".",
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.0,
+  "bos_token_id": 50256,
+  "embd_pdrop": 0.0,
+  "eos_token_id": 50256,
+  "gradient_checkpointing": false,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_ctx": 1024,
+  "n_embd": 768,
+  "n_head": 12,
+  "n_inner": null,
+  "n_layer": 12,
+  "n_positions": 1024,
+  "resid_pdrop": 0.0,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "task_specific_params": {
+    "text-generation": {
+      "do_sample": true,
+      "max_length": 50
+    }
+  },
+  "transformers_version": "4.8.2",
+  "use_cache": true,
+  "vocab_size": 50257
+}

evaluate.py

@@ -0,0 +1,10 @@
+from transformers import GPT2Tokenizer, GPT2Model, FlaxGPT2LMHeadModel, GPT2LMHeadModel, pipeline, set_seed
+ 
+tokenizer = GPT2Tokenizer.from_pretrained("flax-community/swe-gpt-wiki")
+model = GPT2LMHeadModel.from_pretrained("flax-community/swe-gpt-wiki")
+
+
+generator = pipeline('text-generation', model=model, tokenizer=tokenizer)
+set_seed(42)
+result = generator("Det er flott", max_length=150, num_return_sequences=5)
+print(result)

events.out.tfevents.1626214676.t1v-n-905ccd3e-w-0.93628.3.v2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:96eea81b96a95a8ddf5c516912c6686f9afd98236db97c8c2d3d62e9273e1414
+size 735617

flax_model.msgpack

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:112a250b6061edbb1afec25e1f9c2dc844be6be67471b95d2e48846c6bdc828d
+size 497764120

make_config.py

@@ -0,0 +1,6 @@
+from transformers import GPT2Config
+
+model_dir = "./swe-gpt-wiki" # ${MODEL_DIR}
+
+config = GPT2Config.from_pretrained("gpt2", resid_pdrop=0.0, embd_pdrop=0.0, attn_pdrop=0.0)
+config.save_pretrained(model_dir)

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ad3e5639a357aabb61433cc806a336e4c10c4430c1b00bc93a5ec6e1f86d9f0
+size 510401385

save_model.py

@@ -0,0 +1,8 @@
+from transformers import AutoTokenizer, GPT2LMHeadModel
+'''
+This is a script to convert the Jax model and the tokenizer to Pytorch model
+'''
+model = GPT2LMHeadModel.from_pretrained(".", from_flax=True)
+model.save_pretrained(".")
+tokenizer = AutoTokenizer.from_pretrained(".")
+tokenizer.save_pretrained(".")

special_tokens_map.json

@@ -0,0 +1 @@
+{"bos_token": "<|endoftext|>", "eos_token": "<|endoftext|>", "unk_token": "<|endoftext|>"}

tokenizer_config.json

@@ -0,0 +1 @@
+{"unk_token": "<|endoftext|>", "bos_token": "<|endoftext|>", "eos_token": "<|endoftext|>", "add_prefix_space": false, "special_tokens_map_file": null, "name_or_path": ".", "tokenizer_class": "GPT2Tokenizer"}