Upload 9 files

1_Pooling/config.json

@@ -0,0 +1,7 @@
+{
+  "word_embedding_dimension": 768,
+  "pooling_mode_cls_token": true,
+  "pooling_mode_mean_tokens": false,
+  "pooling_mode_max_tokens": false,
+  "pooling_mode_mean_sqrt_len_tokens": false
+}

README.md

@@ -1,3 +1,62 @@
 ---
+language:
+- ru
+
+pipeline_tag: sentence-similarity
+
+tags:
+- russian
+- pretraining
+- embeddings
+- feature-extraction
+- sentence-similarity
+- sentence-transformers
+- transformers
+
+datasets:
+- IlyaGusev/gazeta
+- zloelias/lenta-ru
+
 license: mit
+base_model: cointegrated/LaBSE-en-ru
+
 ---
+
+Модель BERT для расчетов эмбедингов предложений на русском языке. Модель основана на [cointegrated/LaBSE-en-ru](https://huggingface.co/cointegrated/LaBSE-en-ru) - имеет аналогичные размеры контекста (512), ембединга (768) и быстродействие.
+
+
+## Использование:
+```Python
+from sentence_transformers import SentenceTransformer, util
+
+model = SentenceTransformer('sergeyzh/LaBSE-ru-turbo')
+
+sentences = ["привет мир", "hello world", "здравствуй вселенная"]
+embeddings = model.encode(sentences)
+print(util.dot_score(embeddings, embeddings))
+```
+
+## Метрики
+Оценки модели на бенчмарке [encodechka](https://github.com/avidale/encodechka):
+
+| Model                              | CPU       | GPU      | size     |   Mean S  | Mean S+W   |   dim  |
+|:-----------------------------------|----------:|---------:|---------:|----------:|-----------:|-------:|
+| **sergeyzh/LaBSE-ru-turbo**        | **133.40**|**15.30** |**490**   |  **0.789**| **0.703**  | **768**|
+| BAAI/bge-m3                        |   523.40  |  22.50   | 2166     |    0.787  |   0.696    |  1024  |
+| intfloat/multilingual-e5-large     |   506.80  |  30.80   | 2136     |    0.780  |   0.686    |  1024  |
+| intfloat/multilingual-e5-base      |   130.61  |  14.39   | 1061     |    0.761  |   0.669    |   768  |
+| sergeyzh/rubert-tiny-turbo         |     5.51  |   3.25   |  111     |    0.749  |   0.667    |   312  |
+| intfloat/multilingual-e5-small     |    40.86  |  12.09   |  449     |    0.742  |   0.645    |   384  |
+| cointegrated/LaBSE-en-ru           |   133.40  |  15.30   |  490     |    0.739  |   0.667    |   768  |
+
+| Model                              | STS      | PI       | NLI      | SA       | TI       | IA       | IC       | ICX      | NE1      | NE2      |
+|:-----------------------------------|:---------|:---------|:---------|:---------|:---------|:---------|:---------|:---------|:---------|:---------|
+| **sergeyzh/LaBSE-ru-turbo**        |**0.864** |**0.748** |**0.490** |**0.814** |**0.974** |**0.806** |**0.815** |**0.802** |**0.320** |**0.401** |
+| BAAI/bge-m3                        |  0.864   |  0.749   |  0.510   |  0.819   |  0.973   |  0.792   |  0.809   |  0.783   |  0.240   |  0.422   |
+| intfloat/multilingual-e5-large     |  0.862   |  0.727   |  0.473   |  0.810   |  0.979   |  0.798   |  0.819   |  0.773   |  0.224   |  0.374   |
+| intfloat/multilingual-e5-base      |  0.835   |  0.704   |  0.459   |  0.796   |  0.964   |  0.783   |  0.802   |  0.738   |  0.235   |  0.376   |
+| sergeyzh/rubert-tiny-turbo         |  0.828   |  0.722   |  0.476   |  0.787   |  0.955   |  0.757   |  0.780   |  0.685   |  0.305   |  0.373   |
+| intfloat/multilingual-e5-small     |  0.822   |  0.714   |  0.457   |  0.758   |  0.957   |  0.761   |  0.779   |  0.691   |  0.234   |  0.275   |
+| cointegrated/LaBSE-en-ru           |  0.794   |  0.659   |  0.431   |  0.761   |  0.946   |  0.766   |  0.789   |  0.769   |  0.340   |  0.414   |
+
+

config.json

@@ -0,0 +1,32 @@
+{
+  "_name_or_path": "sergeyzh/LaBSE-ru-turbo",
+  "architectures": [
+    "BertModel"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "directionality": "bidi",
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "pooler_fc_size": 768,
+  "pooler_num_attention_heads": 12,
+  "pooler_num_fc_layers": 3,
+  "pooler_size_per_head": 128,
+  "pooler_type": "first_token_transform",
+  "position_embedding_type": "absolute",
+  "torch_dtype": "float32",
+  "transformers_version": "4.38.1",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 55083
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:600d3b4cc3176f6010ddeff239dc8f0aac94f0eb63fb7d99d8f902c94d1fe597
+size 513402728

modules.json

@@ -0,0 +1,20 @@
+[
+  {
+    "idx": 0,
+    "name": "0",
+    "path": "",
+    "type": "sentence_transformers.models.Transformer"
+  },
+  {
+    "idx": 1,
+    "name": "1",
+    "path": "1_Pooling",
+    "type": "sentence_transformers.models.Pooling"
+  },
+  {
+    "idx": 2,
+    "name": "2",
+    "path": "2_Normalize",
+    "type": "sentence_transformers.models.Normalize"
+  }
+]

sentence_bert_config.json

@@ -0,0 +1,4 @@
+{
+  "max_seq_length": 512,
+  "do_lower_case": false
+}

special_tokens_map.json

@@ -0,0 +1,37 @@
+{
+  "cls_token": {
+    "content": "[CLS]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "[MASK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "[PAD]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "[SEP]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "[UNK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,57 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "do_basic_tokenize": true,
+  "do_lower_case": false,
+  "mask_token": "[MASK]",
+  "model_max_length": 512,
+  "never_split": null,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "BertTokenizer",
+  "unk_token": "[UNK]"
+}