| { | |
| "STSBenchmark": { | |
| "train": { | |
| "pearson": [ | |
| 0.7868313677903059, | |
| 0.0 | |
| ], | |
| "spearman": [ | |
| 0.7533061602723176, | |
| 0.0 | |
| ], | |
| "nsamples": 5749 | |
| }, | |
| "dev": { | |
| "pearson": [ | |
| 0.8010234738701788, | |
| 0.0 | |
| ], | |
| "spearman": [ | |
| 0.8010173901569266, | |
| 0.0 | |
| ], | |
| "nsamples": 1500 | |
| }, | |
| "test": { | |
| "pearson": [ | |
| 0.7766524771374954, | |
| 1.1662374252806932e-278 | |
| ], | |
| "spearman": [ | |
| 0.7638460069862814, | |
| 3.8220116833205965e-264 | |
| ], | |
| "nsamples": 1379 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.7882517084559206, | |
| "mean": 0.7881691062659933, | |
| "wmean": 0.7876718254757005 | |
| }, | |
| "spearman": { | |
| "all": 0.7668425138024548, | |
| "mean": 0.7727231858051752, | |
| "wmean": 0.7632854478761041 | |
| } | |
| } | |
| }, | |
| "SICKRelatedness": { | |
| "train": { | |
| "pearson": [ | |
| 0.7853064744370001, | |
| 0.0 | |
| ], | |
| "spearman": [ | |
| 0.7045889188784015, | |
| 0.0 | |
| ], | |
| "nsamples": 4500 | |
| }, | |
| "dev": { | |
| "pearson": [ | |
| 0.7952406911697713, | |
| 2.6811654216760936e-110 | |
| ], | |
| "spearman": [ | |
| 0.7309801454276812, | |
| 1.0951577254305346e-84 | |
| ], | |
| "nsamples": 500 | |
| }, | |
| "test": { | |
| "pearson": [ | |
| 0.7794843403240526, | |
| 0.0 | |
| ], | |
| "spearman": [ | |
| 0.6977349592492605, | |
| 0.0 | |
| ], | |
| "nsamples": 4927 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.7828650051750712, | |
| "mean": 0.7866771686436079, | |
| "wmean": 0.7829171779317007 | |
| }, | |
| "spearman": { | |
| "all": 0.7025121676824595, | |
| "mean": 0.7111013411851145, | |
| "wmean": 0.702516404944873 | |
| } | |
| } | |
| }, | |
| "MR": { | |
| "devacc": 81.09, | |
| "acc": 80.59, | |
| "ndev": 10662, | |
| "ntest": 10662 | |
| }, | |
| "CR": { | |
| "devacc": 87.01, | |
| "acc": 86.6, | |
| "ndev": 3775, | |
| "ntest": 3775 | |
| }, | |
| "SUBJ": { | |
| "devacc": 95.34, | |
| "acc": 95.02, | |
| "ndev": 10000, | |
| "ntest": 10000 | |
| }, | |
| "MPQA": { | |
| "devacc": 88.89, | |
| "acc": 88.99, | |
| "ndev": 10606, | |
| "ntest": 10606 | |
| }, | |
| "SST2": { | |
| "devacc": 84.06, | |
| "acc": 84.95, | |
| "ndev": 872, | |
| "ntest": 1821 | |
| }, | |
| "TREC": { | |
| "devacc": 82.7, | |
| "acc": 87.2, | |
| "ndev": 5452, | |
| "ntest": 500 | |
| }, | |
| "MRPC": { | |
| "devacc": 70.8, | |
| "acc": 68.81, | |
| "f1": 76.97, | |
| "ndev": 4076, | |
| "ntest": 1725 | |
| }, | |
| "STS12": { | |
| "MSRpar": { | |
| "pearson": [ | |
| 0.4506656429961876, | |
| 8.587667122234213e-39 | |
| ], | |
| "spearman": [ | |
| 0.4537783738578545, | |
| 2.270328303306969e-39 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "MSRvid": { | |
| "pearson": [ | |
| 0.8735316429807825, | |
| 4.363488536726283e-236 | |
| ], | |
| "spearman": [ | |
| 0.8745116790315722, | |
| 2.8887254873277706e-237 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "SMTeuroparl": { | |
| "pearson": [ | |
| 0.5276646768590786, | |
| 2.9080130640693665e-34 | |
| ], | |
| "spearman": [ | |
| 0.6225349369600102, | |
| 1.3236704498055218e-50 | |
| ], | |
| "nsamples": 459 | |
| }, | |
| "surprise.OnWN": { | |
| "pearson": [ | |
| 0.7340552729415586, | |
| 7.588168789192366e-128 | |
| ], | |
| "spearman": [ | |
| 0.6862387036955726, | |
| 1.6734444180335097e-105 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "surprise.SMTnews": { | |
| "pearson": [ | |
| 0.6554833920285361, | |
| 2.382715639542081e-50 | |
| ], | |
| "spearman": [ | |
| 0.6338572026855626, | |
| 3.2084519789805647e-46 | |
| ], | |
| "nsamples": 399 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.667994262204242, | |
| "mean": 0.6482801255612287, | |
| "wmean": 0.6587597745452379 | |
| }, | |
| "spearman": { | |
| "all": 0.6198695741812278, | |
| "mean": 0.6541841792461144, | |
| "wmean": 0.659443091175976 | |
| } | |
| } | |
| }, | |
| "STS13": { | |
| "FNWN": { | |
| "pearson": [ | |
| 0.570828218970704, | |
| 9.847987645161228e-18 | |
| ], | |
| "spearman": [ | |
| 0.6046232001507675, | |
| 3.1840503190651897e-20 | |
| ], | |
| "nsamples": 189 | |
| }, | |
| "headlines": { | |
| "pearson": [ | |
| 0.7995020825200164, | |
| 9.500980524179015e-168 | |
| ], | |
| "spearman": [ | |
| 0.8125460785135767, | |
| 1.649528631120915e-177 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "OnWN": { | |
| "pearson": [ | |
| 0.7844322816916485, | |
| 4.612357080156043e-118 | |
| ], | |
| "spearman": [ | |
| 0.7801367000550998, | |
| 5.871872377147612e-116 | |
| ], | |
| "nsamples": 561 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.7407620228952401, | |
| "mean": 0.718254194394123, | |
| "wmean": 0.7650530702029934 | |
| }, | |
| "spearman": { | |
| "all": 0.7458274512781141, | |
| "mean": 0.7324353262398148, | |
| "wmean": 0.7742266882963925 | |
| } | |
| } | |
| }, | |
| "STS14": { | |
| "deft-forum": { | |
| "pearson": [ | |
| 0.5631321216962222, | |
| 5.210192672663357e-39 | |
| ], | |
| "spearman": [ | |
| 0.5511772152789268, | |
| 4.031968923211026e-37 | |
| ], | |
| "nsamples": 450 | |
| }, | |
| "deft-news": { | |
| "pearson": [ | |
| 0.7692532638001923, | |
| 6.387272025432673e-60 | |
| ], | |
| "spearman": [ | |
| 0.7225821241233733, | |
| 1.0522226454927607e-49 | |
| ], | |
| "nsamples": 300 | |
| }, | |
| "headlines": { | |
| "pearson": [ | |
| 0.7685532605008147, | |
| 3.1092413652073723e-147 | |
| ], | |
| "spearman": [ | |
| 0.72597520728351, | |
| 9.721513948491652e-124 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "images": { | |
| "pearson": [ | |
| 0.8353848860067018, | |
| 1.3687751285353585e-196 | |
| ], | |
| "spearman": [ | |
| 0.8008852796417836, | |
| 9.492757773828998e-169 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "OnWN": { | |
| "pearson": [ | |
| 0.8314850837955547, | |
| 3.9440310563877586e-193 | |
| ], | |
| "spearman": [ | |
| 0.8316458764161619, | |
| 2.851406716141642e-193 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "tweet-news": { | |
| "pearson": [ | |
| 0.7778758524787571, | |
| 4.607989885863323e-153 | |
| ], | |
| "spearman": [ | |
| 0.6958906739489547, | |
| 1.1738361486418922e-109 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.73992310716163, | |
| "mean": 0.7576140780463737, | |
| "wmean": 0.7717759322639277 | |
| }, | |
| "spearman": { | |
| "all": 0.6864467990669687, | |
| "mean": 0.7213593961154516, | |
| "wmean": 0.734827243221423 | |
| } | |
| } | |
| }, | |
| "STS15": { | |
| "answers-forums": { | |
| "pearson": [ | |
| 0.7391296029963142, | |
| 5.385805927099778e-66 | |
| ], | |
| "spearman": [ | |
| 0.7409312029744309, | |
| 1.789609179402688e-66 | |
| ], | |
| "nsamples": 375 | |
| }, | |
| "answers-students": { | |
| "pearson": [ | |
| 0.7026326622651042, | |
| 1.1706959560110963e-112 | |
| ], | |
| "spearman": [ | |
| 0.7107886172788801, | |
| 2.1014876744572312e-116 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "belief": { | |
| "pearson": [ | |
| 0.8006281964159404, | |
| 5.439878843301103e-85 | |
| ], | |
| "spearman": [ | |
| 0.8172868949869829, | |
| 2.543177483321865e-91 | |
| ], | |
| "nsamples": 375 | |
| }, | |
| "headlines": { | |
| "pearson": [ | |
| 0.8172091326355518, | |
| 3.472486751097505e-181 | |
| ], | |
| "spearman": [ | |
| 0.8158723780481846, | |
| 4.030205865269612e-180 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "images": { | |
| "pearson": [ | |
| 0.8784869743544939, | |
| 3.750940760878824e-242 | |
| ], | |
| "spearman": [ | |
| 0.8806887757856461, | |
| 6.211464354884257e-245 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.7913051772787048, | |
| "mean": 0.787617313733481, | |
| "wmean": 0.7920519172403193 | |
| }, | |
| "spearman": { | |
| "all": 0.7986117512115241, | |
| "mean": 0.7931135738148248, | |
| "wmean": 0.7966147050233544 | |
| } | |
| } | |
| }, | |
| "STS16": { | |
| "answer-answer": { | |
| "pearson": [ | |
| 0.7177168935513755, | |
| 1.7118144003107402e-41 | |
| ], | |
| "spearman": [ | |
| 0.7132893030185777, | |
| 8.841119832138394e-41 | |
| ], | |
| "nsamples": 254 | |
| }, | |
| "headlines": { | |
| "pearson": [ | |
| 0.7910112149047425, | |
| 1.2553390356155912e-54 | |
| ], | |
| "spearman": [ | |
| 0.8059772680734066, | |
| 3.561704789876049e-58 | |
| ], | |
| "nsamples": 249 | |
| }, | |
| "plagiarism": { | |
| "pearson": [ | |
| 0.8146246368802861, | |
| 7.416724178775043e-56 | |
| ], | |
| "spearman": [ | |
| 0.8287733749128491, | |
| 2.0728091350119785e-59 | |
| ], | |
| "nsamples": 230 | |
| }, | |
| "postediting": { | |
| "pearson": [ | |
| 0.8358664674126319, | |
| 5.627683437146896e-65 | |
| ], | |
| "spearman": [ | |
| 0.8514410852739017, | |
| 8.855209165489321e-70 | |
| ], | |
| "nsamples": 244 | |
| }, | |
| "question-question": { | |
| "pearson": [ | |
| 0.7757563825820808, | |
| 2.9043591729136265e-43 | |
| ], | |
| "spearman": [ | |
| 0.7882765418102572, | |
| 1.5472955880308416e-45 | |
| ], | |
| "nsamples": 209 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.7727412422624585, | |
| "mean": 0.7869951190662233, | |
| "wmean": 0.7864334333593027 | |
| }, | |
| "spearman": { | |
| "all": 0.7823935956761426, | |
| "mean": 0.7975515146177985, | |
| "wmean": 0.7967817209039866 | |
| } | |
| } | |
| }, | |
| "eval_senteval-STS12": 0.6198695741812278, | |
| "eval_senteval-STS13": 0.7458274512781141, | |
| "eval_senteval-STS14": 0.6864467990669687, | |
| "eval_senteval-STS15": 0.7986117512115241, | |
| "eval_senteval-STS16": 0.7823935956761426, | |
| "eval_senteval-STSBenchmark": 0.7668425138024548, | |
| "eval_senteval-SICKRelatedness": 0.7025121676824595, | |
| "eval_senteval-avg_sts_7": 0.7289291218426988, | |
| "eval_senteval-MR": 81.09, | |
| "eval_senteval-CR": 87.01, | |
| "eval_senteval-SUBJ": 95.34, | |
| "eval_senteval-MPQA": 88.89, | |
| "eval_senteval-SST2": 84.06, | |
| "eval_senteval-TREC": 82.7, | |
| "eval_senteval-MRPC": 70.8, | |
| "eval_senteval-avg_transfer": 84.27 | |
| } | |