first release

README.md

@@ -1,3 +1,34 @@
 ---
-license: cc-by-sa-4.0
+tags:
+- generated_from_trainer
+language:
+- sl
+- en
+licence: cc-by-sa-4.0
 ---
+
+# SloBERTa-SlEng
+
+SloBERTa-SlEng is a masked language model, based on the [SloBERTa](https://huggingface.co/EMBEDDIA/sloberta) Slovene model.
+
+SloBERTa-SlEng replaces the tokenizer, vocabulary and the embeddings layer of the SloBERTa model. 
+The tokenizer and vocabulary used are bilingual, Slovene-English, based on conversational, non-standard, and slang language the model was trained on.
+They are the same as in the [SlEng-bert](https://huggingface.co/cjvt/sleng-bert) model.
+The new embedding weights were initialized from the SloBERTa embeddings.
+
+The new SloBERTa-SlEng model is SloBERTa model, which was further pre-trained for two epochs on the conversational English and Slovene corpora,
+the same as the [SlEng-bert](https://huggingface.co/cjvt/sleng-bert) model.
+
+## Training corpora
+
+The model was trained on English and Slovene tweets, Slovene corpora [MaCoCu](http://hdl.handle.net/11356/1517) and [Frenk](http://hdl.handle.net/11356/1201),
+and a small subset of English [Oscar](https://huggingface.co/datasets/oscar) corpus. We tried to keep the sizes of English and Slovene corpora as equal as possible.
+Training corpora had in total about 2.7 billion words.
+
+
+### Framework versions
+
+- Transformers 4.22.0.dev0
+- Pytorch 1.13.0a0+d321be6
+- Datasets 2.4.0
+- Tokenizers 0.12.1

all_results.json

@@ -0,0 +1,15 @@
+{
+    "epoch": 2.0,
+    "eval_accuracy": 0.5992899920812731,
+    "eval_loss": 2.087890625,
+    "eval_runtime": 4563.1687,
+    "eval_samples": 636089,
+    "eval_samples_per_second": 139.396,
+    "eval_steps_per_second": 2.178,
+    "perplexity": 8.067879020505075,
+    "train_loss": 0.10199262492233857,
+    "train_runtime": 71458.7973,
+    "train_samples": 125699346,
+    "train_samples_per_second": 3518.093,
+    "train_steps_per_second": 27.485
+}

config.json

@@ -0,0 +1,28 @@
+{
+  "_name_or_path": "./slen-from-sloberta",
+  "architectures": [
+    "CamembertForMaskedLM"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 0,
+  "classifier_dropout": null,
+  "eos_token_id": 2,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 514,
+  "model_type": "camembert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 1,
+  "position_embedding_type": "absolute",
+  "torch_dtype": "float16",
+  "transformers_version": "4.22.0.dev0",
+  "type_vocab_size": 1,
+  "use_cache": true,
+  "vocab_size": 40005
+}

eval_results.json

@@ -0,0 +1,10 @@
+{
+    "epoch": 2.0,
+    "eval_accuracy": 0.5992899920812731,
+    "eval_loss": 2.087890625,
+    "eval_runtime": 4563.1687,
+    "eval_samples": 636089,
+    "eval_samples_per_second": 139.396,
+    "eval_steps_per_second": 2.178,
+    "perplexity": 8.067879020505075
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c86db419a34f6c0e2d88e8933a800452fc3dfbeadc4862eb7551ae57e150ec8d
+size 233648127

special_tokens_map.json

@@ -0,0 +1,19 @@
+{
+  "additional_special_tokens": [
+    "<s>NOTUSED",
+    "</s>NOTUSED"
+  ],
+  "bos_token": "<s>",
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "unk_token": "<unk>"
+}

tokenizer_config.json

@@ -0,0 +1,24 @@
+{
+  "additional_special_tokens": [
+    "<s>NOTUSED",
+    "</s>NOTUSED"
+  ],
+  "bos_token": "<s>",
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": {
+    "__type": "AddedToken",
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "name_or_path": "./slen-from-sloberta",
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "sp_model_kwargs": {},
+  "special_tokens_map_file": null,
+  "tokenizer_class": "CamembertTokenizer",
+  "unk_token": "<unk>"
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 2.0,
+    "train_loss": 0.10199262492233857,
+    "train_runtime": 71458.7973,
+    "train_samples": 125699346,
+    "train_samples_per_second": 3518.093,
+    "train_steps_per_second": 27.485
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4c8f8e4af5ab3b5506b2586f94a4784233af0c0a55ccfb6fc4f881fc3920bc6d
+size 4347