add model

1_Pooling/config.json

@@ -0,0 +1,7 @@
+{
+  "word_embedding_dimension": 768,
+  "pooling_mode_cls_token": false,
+  "pooling_mode_mean_tokens": true,
+  "pooling_mode_max_tokens": false,
+  "pooling_mode_mean_sqrt_len_tokens": false
+}

README.md

@@ -0,0 +1,126 @@
+---
+pipeline_tag: sentence-similarity
+tags:
+- sentence-transformers
+- feature-extraction
+- sentence-similarity
+- transformers
+
+---
+
+# {MODEL_NAME}
+
+This is a [sentence-transformers](https://www.SBERT.net) model: It maps sentences & paragraphs to a 768 dimensional dense vector space and can be used for tasks like clustering or semantic search.
+
+<!--- Describe your model here -->
+
+## Usage (Sentence-Transformers)
+
+Using this model becomes easy when you have [sentence-transformers](https://www.SBERT.net) installed:
+
+```
+pip install -U sentence-transformers
+```
+
+Then you can use the model like this:
+
+```python
+from sentence_transformers import SentenceTransformer
+sentences = ["This is an example sentence", "Each sentence is converted"]
+
+model = SentenceTransformer('{MODEL_NAME}')
+embeddings = model.encode(sentences)
+print(embeddings)
+```
+
+
+
+## Usage (HuggingFace Transformers)
+Without [sentence-transformers](https://www.SBERT.net), you can use the model like this: First, you pass your input through the transformer model, then you have to apply the right pooling-operation on-top of the contextualized word embeddings.
+
+```python
+from transformers import AutoTokenizer, AutoModel
+import torch
+
+
+#Mean Pooling - Take attention mask into account for correct averaging
+def mean_pooling(model_output, attention_mask):
+    token_embeddings = model_output[0] #First element of model_output contains all token embeddings
+    input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()
+    return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9)
+
+
+# Sentences we want sentence embeddings for
+sentences = ['This is an example sentence', 'Each sentence is converted']
+
+# Load model from HuggingFace Hub
+tokenizer = AutoTokenizer.from_pretrained('{MODEL_NAME}')
+model = AutoModel.from_pretrained('{MODEL_NAME}')
+
+# Tokenize sentences
+encoded_input = tokenizer(sentences, padding=True, truncation=True, return_tensors='pt')
+
+# Compute token embeddings
+with torch.no_grad():
+    model_output = model(**encoded_input)
+
+# Perform pooling. In this case, mean pooling.
+sentence_embeddings = mean_pooling(model_output, encoded_input['attention_mask'])
+
+print("Sentence embeddings:")
+print(sentence_embeddings)
+```
+
+
+
+## Evaluation Results
+
+<!--- Describe how your model was evaluated -->
+
+For an automated evaluation of this model, see the *Sentence Embeddings Benchmark*: [https://seb.sbert.net](https://seb.sbert.net?model_name={MODEL_NAME})
+
+
+## Training
+The model was trained with the parameters:
+
+**DataLoader**:
+
+`torch.utils.data.dataloader.DataLoader` of length 181 with parameters:
+```
+{'batch_size': 128, 'sampler': 'torch.utils.data.sampler.RandomSampler', 'batch_sampler': 'torch.utils.data.sampler.BatchSampler'}
+```
+
+**Loss**:
+
+`sentence_transformers.losses.CosineSimilarityLoss.CosineSimilarityLoss` 
+
+Parameters of the fit()-Method:
+```
+{
+    "epochs": 4,
+    "evaluation_steps": 18,
+    "evaluator": "sentence_transformers.evaluation.EmbeddingSimilarityEvaluator.EmbeddingSimilarityEvaluator",
+    "max_grad_norm": 1,
+    "optimizer_class": "<class 'torch.optim.adamw.AdamW'>",
+    "optimizer_params": {
+        "lr": 2e-05
+    },
+    "scheduler": "WarmupLinear",
+    "steps_per_epoch": null,
+    "warmup_steps": 19,
+    "weight_decay": 0.01
+}
+```
+
+
+## Full Model Architecture
+```
+SentenceTransformer(
+  (0): Transformer({'max_seq_length': 64, 'do_lower_case': False}) with Transformer model: RobertaModel 
+  (1): Pooling({'word_embedding_dimension': 768, 'pooling_mode_cls_token': False, 'pooling_mode_mean_tokens': True, 'pooling_mode_max_tokens': False, 'pooling_mode_mean_sqrt_len_tokens': False})
+)
+```
+
+## Citing & Authors
+
+<!--- Describe where people can find more information -->

config.json

@@ -0,0 +1,29 @@
+{
+  "_name_or_path": "2-nli",
+  "architectures": [
+    "RobertaModel"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 0,
+  "classifier_dropout": null,
+  "eos_token_id": 2,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 514,
+  "model_type": "roberta",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 1,
+  "position_embedding_type": "absolute",
+  "tokenizer_class": "BertTokenizer",
+  "torch_dtype": "float32",
+  "transformers_version": "4.23.1",
+  "type_vocab_size": 1,
+  "use_cache": true,
+  "vocab_size": 32000
+}

config_sentence_transformers.json

@@ -0,0 +1,7 @@
+{
+  "__version__": {
+    "sentence_transformers": "2.2.2",
+    "transformers": "4.23.1",
+    "pytorch": "1.12.1+cu113"
+  }
+}

eval/similarity_evaluation_sts-dev_results.csv

@@ -0,0 +1,45 @@
+epoch,steps,cosine_pearson,cosine_spearman,euclidean_pearson,euclidean_spearman,manhattan_pearson,manhattan_spearman,dot_pearson,dot_spearman
+0,18,0.8347399212668769,0.8332797246121304,0.8228220683419281,0.8291475729499039,0.8224948288387292,0.8287429386237023,0.8198962087912096,0.8163464398739215
+0,36,0.8500997690703382,0.8475338790190667,0.8403686912061845,0.8451934720007476,0.8398559197004054,0.8446179517588199,0.8339076666110501,0.8288875098714102
+0,54,0.8576566398218138,0.8559641912288868,0.8486803608109401,0.8545001315966161,0.8481685979138911,0.8539090961657757,0.8403883347366339,0.8359288034188052
+0,72,0.8614263068172443,0.8596708119554897,0.8522681422852729,0.8585348199914217,0.8518689131452688,0.8581417371871679,0.8443385338635461,0.8399265118973109
+0,90,0.8622625389359875,0.860560169554113,0.8515291638856235,0.8588410823504314,0.851154761421723,0.8584830810501113,0.8440354285689844,0.840074830974381
+0,108,0.8647591293014995,0.86290906294464,0.8560882421084428,0.861504033704325,0.8558415408343907,0.8612399247248212,0.8452730548031558,0.8399604496823538
+0,126,0.8647270542731139,0.8630877865211849,0.8559168895556856,0.8612565485032916,0.8556219266579475,0.8608921886332993,0.8465999944298044,0.8413775931812334
+0,144,0.8658541798993468,0.8647096944783821,0.8578204514207111,0.8635178154592491,0.8574504745711753,0.8630453783068518,0.8473106339647456,0.8427280891617673
+0,162,0.8637890757901965,0.8629986680397617,0.8554230391120468,0.8613618352662179,0.8550639612012569,0.860917712340066,0.8482091945416701,0.8443134820686646
+0,180,0.8643662671791474,0.8633070281770931,0.8573613883053599,0.8625068745519917,0.8571068452313275,0.8622077884636112,0.8457946424308453,0.8412400547088651
+0,-1,0.8644664815384532,0.8633331032989949,0.8570022557900225,0.8625069612010325,0.8567472239295306,0.8622086019264869,0.8456090322632973,0.841232090538985
+1,18,0.8649467739741313,0.8643213018348747,0.8564863944965605,0.8630343102687182,0.8562837455613967,0.8627033882166811,0.8467196175918461,0.843000781980554
+1,36,0.8670320162388881,0.8661895444920018,0.8579846258691203,0.8646417561655916,0.8576322402930996,0.8642853656233519,0.8474400390738416,0.8437195020019772
+1,54,0.8672721928425715,0.8667410704691062,0.8577094122727167,0.8644846259346682,0.8574009857849926,0.8641509587085607,0.8499204238901013,0.8468365766982453
+1,72,0.8671227238281696,0.8662132402092656,0.8572520648295227,0.8638892928947055,0.8569520968366384,0.8635997329348132,0.8492351620786516,0.8458578831062495
+1,90,0.8678416534519859,0.8674494740983647,0.85949224116522,0.8650699878224272,0.8593205538381226,0.8648033840377954,0.8510941932729723,0.8472607770039374
+1,108,0.8674370770132906,0.8665039126647166,0.8579711375947334,0.8643730722827421,0.8578152877916907,0.864209259737278,0.8494317607157732,0.845908987771433
+1,126,0.8671927289696291,0.8663455387038639,0.8585410974841627,0.8650263582078347,0.8584032823259056,0.8649601325330181,0.8492849288686632,0.8460015511508617
+1,144,0.86783215553488,0.8672121324271329,0.8595240386539258,0.865713729499918,0.8595299059338269,0.8657695154216907,0.8500786944379743,0.8467179069836721
+1,162,0.8682126859731041,0.8678705709348622,0.8605869057693138,0.8658463925491341,0.8606341319511247,0.8659476192027802,0.8512538794304347,0.8473896670488367
+1,180,0.8668683193901168,0.8660543013420865,0.8582272796621346,0.864642344963953,0.8582189316196551,0.8647265599352282,0.8496334187843326,0.8463044026185661
+1,-1,0.8666838978892663,0.8659092223980983,0.8580741441572912,0.8645576658744774,0.8580646322974534,0.8645966595836864,0.8495103081610313,0.8462394354995008
+2,18,0.8678189377632195,0.8672234990497559,0.8598144185747968,0.8661260957023846,0.8597836432129506,0.8660327251646098,0.8505501153182834,0.8470558250261008
+2,36,0.8679237117855623,0.8675644285587523,0.8592115526459045,0.8653466491709809,0.8591844778733689,0.865253624672131,0.8519981115949455,0.8489493981952804
+2,54,0.8683278871671422,0.8680075854151419,0.8589016493671165,0.8656490872521079,0.8588297939238064,0.8655685434230549,0.8512740718633409,0.848539388817333
+2,72,0.8682208923620229,0.8679733979061138,0.8595386770982167,0.8660592919775737,0.8594880388807085,0.8659973086473929,0.8501689368964893,0.847150930936474
+2,90,0.867903899102004,0.867781796016561,0.8596243761034937,0.865898066630332,0.8595724278106595,0.8659380431041441,0.8491428916231438,0.845865392498769
+2,108,0.8680728917227094,0.8677120846776806,0.8592655905287316,0.8658850009673803,0.8591539057256747,0.8657729952807556,0.8473125601552337,0.8439435472255371
+2,126,0.8686002923537537,0.8675904473750036,0.8587589726695357,0.8652918029477185,0.8586911620526064,0.8652853693748853,0.8493271485339755,0.8457298361192633
+2,144,0.8699787315338802,0.8691034396259002,0.860496912037321,0.866389432336292,0.8604837949420179,0.8664523461229845,0.8515662686345393,0.8476207407641948
+2,162,0.8694853063734511,0.8688807584688368,0.860078753549108,0.8659412611355093,0.8600458817901875,0.8659224880748791,0.8509570278616316,0.8471775792754245
+2,180,0.8689125018486616,0.86806145581276,0.8592369718966293,0.865639197014906,0.859200572022798,0.8655605030077858,0.849392974165772,0.8456411797382215
+2,-1,0.8689147422129135,0.8680784430084467,0.8593326924115381,0.8656928611888111,0.859295167531994,0.8656098800489083,0.8494639785915404,0.8456843944927636
+3,18,0.8692606086600012,0.8686308677052375,0.8595307121575516,0.8658615513487748,0.8595039907866738,0.8658081712867041,0.850538801194304,0.8468919369858394
+3,36,0.8691477594297009,0.8684244236881645,0.8589370369593333,0.8655052811233253,0.8589191137722088,0.8654551429845935,0.8501842614330577,0.846720505906113
+3,54,0.8694177200880064,0.868831819977,0.8599421492535604,0.866302085634479,0.8598848271606944,0.8663053169404198,0.85063930183484,0.847168952611844
+3,72,0.8693645458199576,0.8689118912042776,0.8600922189477644,0.8664368331178692,0.8600185945160068,0.8664088018287545,0.8500505491670486,0.8465801083670688
+3,90,0.869530629553876,0.8691253245219047,0.8599990015693046,0.8663922758594101,0.8599292799095679,0.8663036564720893,0.8510847974453335,0.8476369751684418
+3,108,0.869247607092199,0.8686768245000296,0.8594596441517444,0.8661218843397859,0.8594028568411224,0.8660431502249939,0.8502917841416529,0.8469578198941338
+3,126,0.8692530615240412,0.8685640576902675,0.8595136777465838,0.8661009639427938,0.8594629770286901,0.8660582507125328,0.8495377139672118,0.8459985913304943
+3,144,0.8693411320539325,0.868634075100808,0.8600217827361845,0.8664030447412382,0.8599669046173116,0.86636762724216,0.8496594348486364,0.8459853567430311
+3,162,0.8693231782053177,0.8686547453534398,0.8599801536113283,0.8663383456056776,0.859924015960567,0.8662926968533435,0.8498052270020244,0.8461966585011806
+3,180,0.8693290739692177,0.8686755584565303,0.859967757750282,0.8663366169125918,0.8599165663277595,0.8663018851949956,0.8498248030180029,0.8462364679175464
+3,-1,0.8693288957325637,0.868676355411704,0.859967824542672,0.8663357776899981,0.8599166281194313,0.8663017108828438,0.8498242771225573,0.846235379447515

modules.json

@@ -0,0 +1,14 @@
+[
+  {
+    "idx": 0,
+    "name": "0",
+    "path": "",
+    "type": "sentence_transformers.models.Transformer"
+  },
+  {
+    "idx": 1,
+    "name": "1",
+    "path": "1_Pooling",
+    "type": "sentence_transformers.models.Pooling"
+  }
+]

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:97ccbeff4ac7ba7c23a683fb834d48ab1ab2faf10b7f9c8bdd125ef5849cafc3
+size 442541937

sentence_bert_config.json

@@ -0,0 +1,4 @@
+{
+  "max_seq_length": 64,
+  "do_lower_case": false
+}

similarity_evaluation_sts-dev_results.csv

@@ -0,0 +1,2 @@
+epoch,steps,cosine_pearson,cosine_spearman,euclidean_pearson,euclidean_spearman,manhattan_pearson,manhattan_spearman,dot_pearson,dot_spearman
+-1,-1,0.869530629553876,0.8691253245219047,0.8599990015693046,0.8663922758594101,0.8599292799095679,0.8663036564720893,0.8510847974453335,0.8476369751684418

special_tokens_map.json

@@ -0,0 +1,9 @@
+{
+  "bos_token": "[CLS]",
+  "cls_token": "[CLS]",
+  "eos_token": "[SEP]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

tokenizer_config.json

@@ -0,0 +1,18 @@
+{
+  "bos_token": "[CLS]",
+  "cls_token": "[CLS]",
+  "do_basic_tokenize": true,
+  "do_lower_case": false,
+  "eos_token": "[SEP]",
+  "mask_token": "[MASK]",
+  "model_max_length": 512,
+  "name_or_path": "2-nli",
+  "never_split": null,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "special_tokens_map_file": "/home/cleaning/.cache/huggingface/hub/models--klue--roberta-base/snapshots/67dd433d36ebc66a42c9aaa85abcf8d2620e41d9/special_tokens_map.json",
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "BertTokenizer",
+  "unk_token": "[UNK]"
+}