| { | |
| "STSBenchmark": { | |
| "train": { | |
| "pearson": [ | |
| 0.7450494648920531, | |
| 0.0 | |
| ], | |
| "spearman": [ | |
| 0.718852932944271, | |
| 0.0 | |
| ], | |
| "nsamples": 5749 | |
| }, | |
| "dev": { | |
| "pearson": [ | |
| 0.7506528902322174, | |
| 6.334785308459189e-272 | |
| ], | |
| "spearman": [ | |
| 0.7572938795946844, | |
| 1.7723543437102127e-279 | |
| ], | |
| "nsamples": 1500 | |
| }, | |
| "test": { | |
| "pearson": [ | |
| 0.7350173167948336, | |
| 1.2934482649472062e-234 | |
| ], | |
| "spearman": [ | |
| 0.7263841319575137, | |
| 1.6287041037900951e-226 | |
| ], | |
| "nsamples": 1379 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.7417281269891771, | |
| "mean": 0.7435732239730347, | |
| "wmean": 0.7444202119694965 | |
| }, | |
| "spearman": { | |
| "all": 0.731134165691079, | |
| "mean": 0.7341769814988229, | |
| "wmean": 0.7267396904100664 | |
| } | |
| } | |
| }, | |
| "SICKRelatedness": { | |
| "train": { | |
| "pearson": [ | |
| 0.7406122998418166, | |
| 0.0 | |
| ], | |
| "spearman": [ | |
| 0.6737860533258943, | |
| 0.0 | |
| ], | |
| "nsamples": 4500 | |
| }, | |
| "dev": { | |
| "pearson": [ | |
| 0.7491355419681752, | |
| 3.992619644206497e-91 | |
| ], | |
| "spearman": [ | |
| 0.6974671648657597, | |
| 4.380991542566403e-74 | |
| ], | |
| "nsamples": 500 | |
| }, | |
| "test": { | |
| "pearson": [ | |
| 0.7341897490069431, | |
| 0.0 | |
| ], | |
| "spearman": [ | |
| 0.6637347101284393, | |
| 0.0 | |
| ], | |
| "nsamples": 4927 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.7377595629576476, | |
| "mean": 0.7413125302723117, | |
| "wmean": 0.7378539350890975 | |
| }, | |
| "spearman": { | |
| "all": 0.6699463146611633, | |
| "mean": 0.6783293094400311, | |
| "wmean": 0.6699901016623577 | |
| } | |
| } | |
| }, | |
| "MR": { | |
| "devacc": 82.31, | |
| "acc": 82.05, | |
| "ndev": 10662, | |
| "ntest": 10662 | |
| }, | |
| "CR": { | |
| "devacc": 88.2, | |
| "acc": 86.99, | |
| "ndev": 3775, | |
| "ntest": 3775 | |
| }, | |
| "SUBJ": { | |
| "devacc": 96.19, | |
| "acc": 96.02, | |
| "ndev": 10000, | |
| "ntest": 10000 | |
| }, | |
| "MPQA": { | |
| "devacc": 89.61, | |
| "acc": 89.74, | |
| "ndev": 10606, | |
| "ntest": 10606 | |
| }, | |
| "SST2": { | |
| "devacc": 85.55, | |
| "acc": 87.59, | |
| "ndev": 872, | |
| "ntest": 1821 | |
| }, | |
| "TREC": { | |
| "devacc": 84.13, | |
| "acc": 89.4, | |
| "ndev": 5452, | |
| "ntest": 500 | |
| }, | |
| "MRPC": { | |
| "devacc": 71.1, | |
| "acc": 67.25, | |
| "f1": 75.21, | |
| "ndev": 4076, | |
| "ntest": 1725 | |
| }, | |
| "STS12": { | |
| "MSRpar": { | |
| "pearson": [ | |
| 0.4636894610396082, | |
| 2.9932393013005614e-41 | |
| ], | |
| "spearman": [ | |
| 0.4671701071614276, | |
| 6.326220240462369e-42 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "MSRvid": { | |
| "pearson": [ | |
| 0.7835433426304015, | |
| 9.533313112923909e-157 | |
| ], | |
| "spearman": [ | |
| 0.7899914141899944, | |
| 4.464625137082651e-161 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "SMTeuroparl": { | |
| "pearson": [ | |
| 0.5157704693177547, | |
| 1.4644419340036422e-32 | |
| ], | |
| "spearman": [ | |
| 0.6100654108812495, | |
| 3.8917771567505466e-48 | |
| ], | |
| "nsamples": 459 | |
| }, | |
| "surprise.OnWN": { | |
| "pearson": [ | |
| 0.7266864885115535, | |
| 4.286261687692017e-124 | |
| ], | |
| "spearman": [ | |
| 0.6862656117967088, | |
| 1.6302568011197229e-105 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "surprise.SMTnews": { | |
| "pearson": [ | |
| 0.6252078952724913, | |
| 1.167855897437133e-44 | |
| ], | |
| "spearman": [ | |
| 0.6194525975795115, | |
| 1.2002375978187872e-43 | |
| ], | |
| "nsamples": 399 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.5887285356802923, | |
| "mean": 0.6229795313543618, | |
| "wmean": 0.6327657866044871 | |
| }, | |
| "spearman": { | |
| "all": 0.5800298176889247, | |
| "mean": 0.6345890283217783, | |
| "wmean": 0.6385945816891302 | |
| } | |
| } | |
| }, | |
| "STS13": { | |
| "FNWN": { | |
| "pearson": [ | |
| 0.5805790417532318, | |
| 2.014986575263064e-18 | |
| ], | |
| "spearman": [ | |
| 0.6051140995186025, | |
| 2.914805969394101e-20 | |
| ], | |
| "nsamples": 189 | |
| }, | |
| "headlines": { | |
| "pearson": [ | |
| 0.7735405820911545, | |
| 2.5642513354078795e-150 | |
| ], | |
| "spearman": [ | |
| 0.7770887857628307, | |
| 1.4673875549570448e-152 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "OnWN": { | |
| "pearson": [ | |
| 0.7407689575128353, | |
| 1.1759474604362382e-98 | |
| ], | |
| "spearman": [ | |
| 0.7416989622827319, | |
| 4.993673629116722e-99 | |
| ], | |
| "nsamples": 561 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.7068855742244011, | |
| "mean": 0.6982961937857405, | |
| "wmean": 0.7369708404162849 | |
| }, | |
| "spearman": { | |
| "all": 0.7277593282424332, | |
| "mean": 0.7079672825213884, | |
| "wmean": 0.742184181314501 | |
| } | |
| } | |
| }, | |
| "STS14": { | |
| "deft-forum": { | |
| "pearson": [ | |
| 0.5292420408489044, | |
| 7.573764882162778e-34 | |
| ], | |
| "spearman": [ | |
| 0.5209832244330959, | |
| 1.1249019540967971e-32 | |
| ], | |
| "nsamples": 450 | |
| }, | |
| "deft-news": { | |
| "pearson": [ | |
| 0.7607426764562661, | |
| 6.946896036723345e-58 | |
| ], | |
| "spearman": [ | |
| 0.7184224573696152, | |
| 6.779631006204726e-49 | |
| ], | |
| "nsamples": 300 | |
| }, | |
| "headlines": { | |
| "pearson": [ | |
| 0.7397522885205156, | |
| 7.80335796344721e-131 | |
| ], | |
| "spearman": [ | |
| 0.6979654635630582, | |
| 1.4287756444623493e-110 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "images": { | |
| "pearson": [ | |
| 0.8050635122757817, | |
| 8.070931278794031e-172 | |
| ], | |
| "spearman": [ | |
| 0.77209684438404, | |
| 2.040397006963273e-149 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "OnWN": { | |
| "pearson": [ | |
| 0.7978143316572779, | |
| 1.5410591755272875e-166 | |
| ], | |
| "spearman": [ | |
| 0.8098966806346459, | |
| 1.8229474486709954e-175 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "tweet-news": { | |
| "pearson": [ | |
| 0.7540876736646366, | |
| 1.0327629108496458e-138 | |
| ], | |
| "spearman": [ | |
| 0.680847896525736, | |
| 2.9778143917773655e-103 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.6942302397102361, | |
| "mean": 0.7311170872372305, | |
| "wmean": 0.7437120202420122 | |
| }, | |
| "spearman": { | |
| "all": 0.6560249452737311, | |
| "mean": 0.7000354278183653, | |
| "wmean": 0.7121531605430367 | |
| } | |
| } | |
| }, | |
| "STS15": { | |
| "answers-forums": { | |
| "pearson": [ | |
| 0.7594027668040867, | |
| 1.2758558949751847e-71 | |
| ], | |
| "spearman": [ | |
| 0.7655617212556672, | |
| 1.927443510544834e-73 | |
| ], | |
| "nsamples": 375 | |
| }, | |
| "answers-students": { | |
| "pearson": [ | |
| 0.7143566765163898, | |
| 4.389575860211754e-118 | |
| ], | |
| "spearman": [ | |
| 0.7174403252516115, | |
| 1.4772753160034449e-119 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "belief": { | |
| "pearson": [ | |
| 0.7978689086664256, | |
| 5.322951694721659e-84 | |
| ], | |
| "spearman": [ | |
| 0.8046906371216603, | |
| 1.7721663221432892e-86 | |
| ], | |
| "nsamples": 375 | |
| }, | |
| "headlines": { | |
| "pearson": [ | |
| 0.7935110897123238, | |
| 1.6659037976269556e-163 | |
| ], | |
| "spearman": [ | |
| 0.7934622789273995, | |
| 1.8015506979194712e-163 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "images": { | |
| "pearson": [ | |
| 0.8476525308721238, | |
| 4.293125384021689e-208 | |
| ], | |
| "spearman": [ | |
| 0.852462935166319, | |
| 6.939054754973825e-213 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.7734727465952778, | |
| "mean": 0.7825583945142699, | |
| "wmean": 0.7835390337090233 | |
| }, | |
| "spearman": { | |
| "all": 0.781469498622039, | |
| "mean": 0.7867235795445315, | |
| "wmean": 0.7871229296334984 | |
| } | |
| } | |
| }, | |
| "STS16": { | |
| "answer-answer": { | |
| "pearson": [ | |
| 0.7185430532553294, | |
| 1.2557109311561009e-41 | |
| ], | |
| "spearman": [ | |
| 0.7162994847494388, | |
| 2.905422009452839e-41 | |
| ], | |
| "nsamples": 254 | |
| }, | |
| "headlines": { | |
| "pearson": [ | |
| 0.7699259135956438, | |
| 4.2719687854897803e-50 | |
| ], | |
| "spearman": [ | |
| 0.7740145549371511, | |
| 6.171530207273899e-51 | |
| ], | |
| "nsamples": 249 | |
| }, | |
| "plagiarism": { | |
| "pearson": [ | |
| 0.8172949259931438, | |
| 1.6720551154970233e-56 | |
| ], | |
| "spearman": [ | |
| 0.8224018754027003, | |
| 9.034865185832892e-58 | |
| ], | |
| "nsamples": 230 | |
| }, | |
| "postediting": { | |
| "pearson": [ | |
| 0.8514470041515851, | |
| 8.81597018547873e-70 | |
| ], | |
| "spearman": [ | |
| 0.8636307176150665, | |
| 6.122058887887909e-74 | |
| ], | |
| "nsamples": 244 | |
| }, | |
| "question-question": { | |
| "pearson": [ | |
| 0.764255607380706, | |
| 2.6639262896118643e-41 | |
| ], | |
| "spearman": [ | |
| 0.76423713237739, | |
| 2.6827754621204016e-41 | |
| ], | |
| "nsamples": 209 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.7712276391528412, | |
| "mean": 0.7842933008752817, | |
| "wmean": 0.7838801112530746 | |
| }, | |
| "spearman": { | |
| "all": 0.7815478011237279, | |
| "mean": 0.7881167530163494, | |
| "wmean": 0.7877517541427318 | |
| } | |
| } | |
| }, | |
| "eval_senteval-STS12": 0.5800298176889247, | |
| "eval_senteval-STS13": 0.7277593282424332, | |
| "eval_senteval-STS14": 0.6560249452737311, | |
| "eval_senteval-STS15": 0.781469498622039, | |
| "eval_senteval-STS16": 0.7815478011237279, | |
| "eval_senteval-STSBenchmark": 0.731134165691079, | |
| "eval_senteval-SICKRelatedness": 0.6699463146611633, | |
| "eval_senteval-avg_sts_7": 0.7039874101861568, | |
| "eval_senteval-MR": 82.31, | |
| "eval_senteval-CR": 88.2, | |
| "eval_senteval-SUBJ": 96.19, | |
| "eval_senteval-MPQA": 89.61, | |
| "eval_senteval-SST2": 85.55, | |
| "eval_senteval-TREC": 84.13, | |
| "eval_senteval-MRPC": 71.1, | |
| "eval_senteval-avg_transfer": 85.29857142857144 | |
| } | |