first release

README.md

@@ -1,3 +1,22 @@
 ---
-license: cc-by-sa-4.0
+language:
+- sl
+- en
+licence: cc-by-sa-4.0
 ---
+
+# SlEng-bert
+
+SlEng-bert is a bilingual, Slovene-English masked language model.
+
+SlEng-bert was trained from scratch on Slovene and English, conversational, non-standard, and slang language.
+The model has 12 transformer layers, and is roughly equal in size to BERT and RoBERTa base models. The pre-training task used was masked language modeling, with no other tasks (like NSP).
+
+The tokenizer and corpora used to train SlEng-bert were also used for training the [SloBERTa-SlEng](https://huggingface.co/cjvt/sloberta-sleng) model.
+The difference between the two is: SlEng-bert was trained from scratch for 40 epochs; SloBERTa-SlEng is SloBERTa further pre-trained for 2 epochs on new corpora.
+
+## Training corpora
+
+The model was trained on English and Slovene tweets, Slovene corpora [MaCoCu](http://hdl.handle.net/11356/1517) and [Frenk](http://hdl.handle.net/11356/1201),
+and a small subset of English [Oscar](https://huggingface.co/datasets/oscar) corpus. We tried to keep the sizes of English and Slovene corpora as equal as possible.
+Training corpora had in total about 2.7 billion words.

config.json

@@ -0,0 +1,25 @@
+{
+  "architectures": [
+    "CamembertForMaskedLM"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 0,
+  "eos_token_id": 2,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 514,
+  "model_type": "camembert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 1,
+  "position_embedding_type": "absolute",
+  "transformers_version": "4.6.1",
+  "type_vocab_size": 1,
+  "use_cache": true,
+  "vocab_size": 40005
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a7b1b3e94569c1da0d4409daa6b38daf048bc1feb149845b97eead1a8fc777e
+size 467443364

tokenizer_config.json

@@ -0,0 +1,24 @@
+{
+  "additional_special_tokens": [
+    "<s>NOTUSED",
+    "</s>NOTUSED"
+  ],
+  "bos_token": "<s>",
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": {
+    "__type": "AddedToken",
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "name_or_path": "./slen-from-sloberta",
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "sp_model_kwargs": {},
+  "special_tokens_map_file": null,
+  "tokenizer_class": "CamembertTokenizer",
+  "unk_token": "<unk>"
+}