{ "STSBenchmark": { "train": { "pearson": [ 0.700217766857111, 0.0 ], "spearman": [ 0.694596243514894, 0.0 ], "nsamples": 5749 }, "dev": { "pearson": [ 0.7188725565702827, 7.642460243707613e-239 ], "spearman": [ 0.7430861317995934, 1.3229042265692801e-263 ], "nsamples": 1500 }, "test": { "pearson": [ 0.6697149917661123, 3.5510311886663796e-180 ], "spearman": [ 0.7028355722100523, 6.396511507534624e-206 ], "nsamples": 1379 }, "all": { "pearson": { "all": 0.7003515412165919, "mean": 0.6962684383978353, "wmean": 0.698585738312752 }, "spearman": { "all": 0.708944371429387, "mean": 0.7135059825081799, "wmean": 0.7043432146203266 } } }, "SICKRelatedness": { "train": { "pearson": [ 0.7900787687078513, 0.0 ], "spearman": [ 0.7162798220982042, 0.0 ], "nsamples": 4500 }, "dev": { "pearson": [ 0.7941456305052318, 8.704175570070149e-110 ], "spearman": [ 0.7323591663810709, 3.70700656086251e-85 ], "nsamples": 500 }, "test": { "pearson": [ 0.7827901659487767, 0.0 ], "spearman": [ 0.7066436772359563, 0.0 ], "nsamples": 4927 }, "all": { "pearson": { "all": 0.7866207722959027, "mean": 0.7890048550539532, "wmean": 0.7866661047715896 }, "spearman": { "all": 0.7122486034563766, "mean": 0.7184275552384105, "wmean": 0.7123070595722788 } } }, "MR": { "devacc": 80.75, "acc": 80.52, "ndev": 10662, "ntest": 10662 }, "CR": { "devacc": 87.02, "acc": 86.41, "ndev": 3775, "ntest": 3775 }, "SUBJ": { "devacc": 95.02, "acc": 94.49, "ndev": 10000, "ntest": 10000 }, "MPQA": { "devacc": 88.67, "acc": 88.53, "ndev": 10606, "ntest": 10606 }, "SST2": { "devacc": 82.8, "acc": 85.89, "ndev": 872, "ntest": 1821 }, "TREC": { "devacc": 78.63, "acc": 88.6, "ndev": 5452, "ntest": 500 }, "MRPC": { "devacc": 70.17, "acc": 62.55, "f1": 68.4, "ndev": 4076, "ntest": 1725 }, "STS12": { "MSRpar": { "pearson": [ 0.3279591431665635, 2.8934198828902823e-20 ], "spearman": [ 0.37151182527291027, 5.848988168730171e-26 ], "nsamples": 750 }, "MSRvid": { "pearson": [ 0.8600914351071427, 7.615361092408985e-221 ], "spearman": [ 0.8710479889820992, 3.841468226701095e-233 ], "nsamples": 750 }, "SMTeuroparl": { "pearson": [ 0.49106075103459146, 3.1227262696721795e-29 ], "spearman": [ 0.5976234583132141, 8.832799021137483e-46 ], "nsamples": 459 }, "surprise.OnWN": { "pearson": [ 0.6903453132676263, 2.993644573897565e-107 ], "spearman": [ 0.6522049690471122, 4.576466079694001e-92 ], "nsamples": 750 }, "surprise.SMTnews": { "pearson": [ 0.5747570138816134, 1.8705693938405458e-36 ], "spearman": [ 0.6161895683302498, 4.402482065232814e-43 ], "nsamples": 399 }, "all": { "pearson": { "all": 0.6532360047609772, "mean": 0.5888427312915075, "wmean": 0.5995887554438998 }, "spearman": { "all": 0.6520942930121344, "mean": 0.6217155619891171, "wmean": 0.6245953644163854 } } }, "STS13": { "FNWN": { "pearson": [ 0.5455691661756268, 4.746052411411139e-16 ], "spearman": [ 0.5552285672079286, 1.1207604310221007e-16 ], "nsamples": 189 }, "headlines": { "pearson": [ 0.7376364893975698, 1.0257488464328715e-129 ], "spearman": [ 0.7584758247313932, 3.1099800660504727e-141 ], "nsamples": 750 }, "OnWN": { "pearson": [ 0.8062599488981587, 1.4758922804276925e-129 ], "spearman": [ 0.791620979725446, 1.0741113505197788e-121 ], "nsamples": 561 }, "all": { "pearson": { "all": 0.7161952323138701, "mean": 0.6964885348237851, "wmean": 0.7391011805248252 }, "spearman": { "all": 0.7489808796663362, "mean": 0.701775123888256, "wmean": 0.7452629582512124 } } }, "STS14": { "deft-forum": { "pearson": [ 0.4539905158613596, 2.8973755391032382e-24 ], "spearman": [ 0.4539149626420418, 2.954477747813041e-24 ], "nsamples": 450 }, "deft-news": { "pearson": [ 0.6909565157190697, 6.731301106701071e-44 ], "spearman": [ 0.654918374815774, 3.9304042018460903e-38 ], "nsamples": 300 }, "headlines": { "pearson": [ 0.7163249206179311, 5.064372905060631e-119 ], "spearman": [ 0.6890244755927888, 1.1000588945875164e-106 ], "nsamples": 750 }, "images": { "pearson": [ 0.8341976013585205, 1.581737521468821e-195 ], "spearman": [ 0.8009087778380325, 9.127029827171149e-169 ], "nsamples": 750 }, "OnWN": { "pearson": [ 0.8416372752913062, 2.488984870540694e-202 ], "spearman": [ 0.835891574935076, 4.7883053518051054e-197 ], "nsamples": 750 }, "tweet-news": { "pearson": [ 0.5893351545919092, 2.4679056644216343e-71 ], "spearman": [ 0.5523429250762462, 4.020802928899476e-61 ], "nsamples": 750 }, "all": { "pearson": { "all": 0.6458742493245161, "mean": 0.6877403305733494, "wmean": 0.7060543735328222 }, "spearman": { "all": 0.6214541243526541, "mean": 0.6645001818166599, "wmean": 0.6824968161907357 } } }, "STS15": { "answers-forums": { "pearson": [ 0.5843871869461419, 1.0151032193286744e-35 ], "spearman": [ 0.6084659043667563, 2.3820444508668996e-39 ], "nsamples": 375 }, "answers-students": { "pearson": [ 0.6152132265657764, 2.6617778013870566e-79 ], "spearman": [ 0.6302561398409405, 2.773969605891675e-84 ], "nsamples": 750 }, "belief": { "pearson": [ 0.6392710473707196, 1.799179528686185e-44 ], "spearman": [ 0.686875976718544, 1.2015551469859737e-53 ], "nsamples": 375 }, "headlines": { "pearson": [ 0.7734487150038147, 2.927357789667028e-150 ], "spearman": [ 0.7896018531226366, 8.23498473346381e-161 ], "nsamples": 750 }, "images": { "pearson": [ 0.864344989852659, 1.7210495814828955e-225 ], "spearman": [ 0.8824596984841369, 3.2828816211446655e-247 ], "nsamples": 750 }, "all": { "pearson": { "all": 0.7203379074291151, "mean": 0.6953330331478222, "wmean": 0.7162090121451702 }, "spearman": { "all": 0.7497940183329284, "mean": 0.7195319145066028, "wmean": 0.7374971579975911 } } }, "STS16": { "answer-answer": { "pearson": [ 0.6368696070010881, 2.678858858103159e-30 ], "spearman": [ 0.6921912704278468, 1.4681538672067537e-37 ], "nsamples": 254 }, "headlines": { "pearson": [ 0.7289776103151235, 1.5277023512508603e-42 ], "spearman": [ 0.7621038994295888, 1.551008215103711e-48 ], "nsamples": 249 }, "plagiarism": { "pearson": [ 0.7702063765670754, 2.003814813962993e-46 ], "spearman": [ 0.7911510680474892, 1.3837761303567582e-50 ], "nsamples": 230 }, "postediting": { "pearson": [ 0.719837823485291, 3.0286294416646686e-40 ], "spearman": [ 0.7566461459669797, 1.4825051882045699e-46 ], "nsamples": 244 }, "question-question": { "pearson": [ 0.7625392869659763, 5.114455019237254e-41 ], "spearman": [ 0.7800407128371691, 5.033994633121257e-44 ], "nsamples": 209 }, "all": { "pearson": { "all": 0.7140924962133864, "mean": 0.7236861408669109, "wmean": 0.721280701233954 }, "spearman": { "all": 0.7490972681198437, "mean": 0.7564266193418148, "wmean": 0.7548021651740933 } } }, "eval_senteval-STS12": 0.6520942930121344, "eval_senteval-STS13": 0.7489808796663362, "eval_senteval-STS14": 0.6214541243526541, "eval_senteval-STS15": 0.7497940183329284, "eval_senteval-STS16": 0.7490972681198437, "eval_senteval-STSBenchmark": 0.708944371429387, "eval_senteval-SICKRelatedness": 0.7122486034563766, "eval_senteval-avg_sts_7": 0.7060876511956659, "eval_senteval-MR": 80.75, "eval_senteval-CR": 87.02, "eval_senteval-SUBJ": 95.02, "eval_senteval-MPQA": 88.67, "eval_senteval-SST2": 82.8, "eval_senteval-TREC": 78.63, "eval_senteval-MRPC": 70.17, "eval_senteval-avg_transfer": 83.2942857142857 }