{ "STSBenchmark": { "train": { "pearson": [ 0.7450494648920531, 0.0 ], "spearman": [ 0.718852932944271, 0.0 ], "nsamples": 5749 }, "dev": { "pearson": [ 0.7506528902322174, 6.334785308459189e-272 ], "spearman": [ 0.7572938795946844, 1.7723543437102127e-279 ], "nsamples": 1500 }, "test": { "pearson": [ 0.7350173167948336, 1.2934482649472062e-234 ], "spearman": [ 0.7263841319575137, 1.6287041037900951e-226 ], "nsamples": 1379 }, "all": { "pearson": { "all": 0.7417281269891771, "mean": 0.7435732239730347, "wmean": 0.7444202119694965 }, "spearman": { "all": 0.731134165691079, "mean": 0.7341769814988229, "wmean": 0.7267396904100664 } } }, "SICKRelatedness": { "train": { "pearson": [ 0.7406122998418166, 0.0 ], "spearman": [ 0.6737860533258943, 0.0 ], "nsamples": 4500 }, "dev": { "pearson": [ 0.7491355419681752, 3.992619644206497e-91 ], "spearman": [ 0.6974671648657597, 4.380991542566403e-74 ], "nsamples": 500 }, "test": { "pearson": [ 0.7341897490069431, 0.0 ], "spearman": [ 0.6637347101284393, 0.0 ], "nsamples": 4927 }, "all": { "pearson": { "all": 0.7377595629576476, "mean": 0.7413125302723117, "wmean": 0.7378539350890975 }, "spearman": { "all": 0.6699463146611633, "mean": 0.6783293094400311, "wmean": 0.6699901016623577 } } }, "MR": { "devacc": 82.31, "acc": 82.05, "ndev": 10662, "ntest": 10662 }, "CR": { "devacc": 88.2, "acc": 86.99, "ndev": 3775, "ntest": 3775 }, "SUBJ": { "devacc": 96.19, "acc": 96.02, "ndev": 10000, "ntest": 10000 }, "MPQA": { "devacc": 89.61, "acc": 89.74, "ndev": 10606, "ntest": 10606 }, "SST2": { "devacc": 85.55, "acc": 87.59, "ndev": 872, "ntest": 1821 }, "TREC": { "devacc": 84.13, "acc": 89.4, "ndev": 5452, "ntest": 500 }, "MRPC": { "devacc": 71.1, "acc": 67.25, "f1": 75.21, "ndev": 4076, "ntest": 1725 }, "STS12": { "MSRpar": { "pearson": [ 0.4636894610396082, 2.9932393013005614e-41 ], "spearman": [ 0.4671701071614276, 6.326220240462369e-42 ], "nsamples": 750 }, "MSRvid": { "pearson": [ 0.7835433426304015, 9.533313112923909e-157 ], "spearman": [ 0.7899914141899944, 4.464625137082651e-161 ], "nsamples": 750 }, "SMTeuroparl": { "pearson": [ 0.5157704693177547, 1.4644419340036422e-32 ], "spearman": [ 0.6100654108812495, 3.8917771567505466e-48 ], "nsamples": 459 }, "surprise.OnWN": { "pearson": [ 0.7266864885115535, 4.286261687692017e-124 ], "spearman": [ 0.6862656117967088, 1.6302568011197229e-105 ], "nsamples": 750 }, "surprise.SMTnews": { "pearson": [ 0.6252078952724913, 1.167855897437133e-44 ], "spearman": [ 0.6194525975795115, 1.2002375978187872e-43 ], "nsamples": 399 }, "all": { "pearson": { "all": 0.5887285356802923, "mean": 0.6229795313543618, "wmean": 0.6327657866044871 }, "spearman": { "all": 0.5800298176889247, "mean": 0.6345890283217783, "wmean": 0.6385945816891302 } } }, "STS13": { "FNWN": { "pearson": [ 0.5805790417532318, 2.014986575263064e-18 ], "spearman": [ 0.6051140995186025, 2.914805969394101e-20 ], "nsamples": 189 }, "headlines": { "pearson": [ 0.7735405820911545, 2.5642513354078795e-150 ], "spearman": [ 0.7770887857628307, 1.4673875549570448e-152 ], "nsamples": 750 }, "OnWN": { "pearson": [ 0.7407689575128353, 1.1759474604362382e-98 ], "spearman": [ 0.7416989622827319, 4.993673629116722e-99 ], "nsamples": 561 }, "all": { "pearson": { "all": 0.7068855742244011, "mean": 0.6982961937857405, "wmean": 0.7369708404162849 }, "spearman": { "all": 0.7277593282424332, "mean": 0.7079672825213884, "wmean": 0.742184181314501 } } }, "STS14": { "deft-forum": { "pearson": [ 0.5292420408489044, 7.573764882162778e-34 ], "spearman": [ 0.5209832244330959, 1.1249019540967971e-32 ], "nsamples": 450 }, "deft-news": { "pearson": [ 0.7607426764562661, 6.946896036723345e-58 ], "spearman": [ 0.7184224573696152, 6.779631006204726e-49 ], "nsamples": 300 }, "headlines": { "pearson": [ 0.7397522885205156, 7.80335796344721e-131 ], "spearman": [ 0.6979654635630582, 1.4287756444623493e-110 ], "nsamples": 750 }, "images": { "pearson": [ 0.8050635122757817, 8.070931278794031e-172 ], "spearman": [ 0.77209684438404, 2.040397006963273e-149 ], "nsamples": 750 }, "OnWN": { "pearson": [ 0.7978143316572779, 1.5410591755272875e-166 ], "spearman": [ 0.8098966806346459, 1.8229474486709954e-175 ], "nsamples": 750 }, "tweet-news": { "pearson": [ 0.7540876736646366, 1.0327629108496458e-138 ], "spearman": [ 0.680847896525736, 2.9778143917773655e-103 ], "nsamples": 750 }, "all": { "pearson": { "all": 0.6942302397102361, "mean": 0.7311170872372305, "wmean": 0.7437120202420122 }, "spearman": { "all": 0.6560249452737311, "mean": 0.7000354278183653, "wmean": 0.7121531605430367 } } }, "STS15": { "answers-forums": { "pearson": [ 0.7594027668040867, 1.2758558949751847e-71 ], "spearman": [ 0.7655617212556672, 1.927443510544834e-73 ], "nsamples": 375 }, "answers-students": { "pearson": [ 0.7143566765163898, 4.389575860211754e-118 ], "spearman": [ 0.7174403252516115, 1.4772753160034449e-119 ], "nsamples": 750 }, "belief": { "pearson": [ 0.7978689086664256, 5.322951694721659e-84 ], "spearman": [ 0.8046906371216603, 1.7721663221432892e-86 ], "nsamples": 375 }, "headlines": { "pearson": [ 0.7935110897123238, 1.6659037976269556e-163 ], "spearman": [ 0.7934622789273995, 1.8015506979194712e-163 ], "nsamples": 750 }, "images": { "pearson": [ 0.8476525308721238, 4.293125384021689e-208 ], "spearman": [ 0.852462935166319, 6.939054754973825e-213 ], "nsamples": 750 }, "all": { "pearson": { "all": 0.7734727465952778, "mean": 0.7825583945142699, "wmean": 0.7835390337090233 }, "spearman": { "all": 0.781469498622039, "mean": 0.7867235795445315, "wmean": 0.7871229296334984 } } }, "STS16": { "answer-answer": { "pearson": [ 0.7185430532553294, 1.2557109311561009e-41 ], "spearman": [ 0.7162994847494388, 2.905422009452839e-41 ], "nsamples": 254 }, "headlines": { "pearson": [ 0.7699259135956438, 4.2719687854897803e-50 ], "spearman": [ 0.7740145549371511, 6.171530207273899e-51 ], "nsamples": 249 }, "plagiarism": { "pearson": [ 0.8172949259931438, 1.6720551154970233e-56 ], "spearman": [ 0.8224018754027003, 9.034865185832892e-58 ], "nsamples": 230 }, "postediting": { "pearson": [ 0.8514470041515851, 8.81597018547873e-70 ], "spearman": [ 0.8636307176150665, 6.122058887887909e-74 ], "nsamples": 244 }, "question-question": { "pearson": [ 0.764255607380706, 2.6639262896118643e-41 ], "spearman": [ 0.76423713237739, 2.6827754621204016e-41 ], "nsamples": 209 }, "all": { "pearson": { "all": 0.7712276391528412, "mean": 0.7842933008752817, "wmean": 0.7838801112530746 }, "spearman": { "all": 0.7815478011237279, "mean": 0.7881167530163494, "wmean": 0.7877517541427318 } } }, "eval_senteval-STS12": 0.5800298176889247, "eval_senteval-STS13": 0.7277593282424332, "eval_senteval-STS14": 0.6560249452737311, "eval_senteval-STS15": 0.781469498622039, "eval_senteval-STS16": 0.7815478011237279, "eval_senteval-STSBenchmark": 0.731134165691079, "eval_senteval-SICKRelatedness": 0.6699463146611633, "eval_senteval-avg_sts_7": 0.7039874101861568, "eval_senteval-MR": 82.31, "eval_senteval-CR": 88.2, "eval_senteval-SUBJ": 96.19, "eval_senteval-MPQA": 89.61, "eval_senteval-SST2": 85.55, "eval_senteval-TREC": 84.13, "eval_senteval-MRPC": 71.1, "eval_senteval-avg_transfer": 85.29857142857144 }