{ "STSBenchmark": { "train": { "pearson": [ 0.759143306453737, 0.0 ], "spearman": [ 0.732317209733238, 0.0 ], "nsamples": 5749 }, "dev": { "pearson": [ 0.7746081783379711, 2.2556430824992415e-300 ], "spearman": [ 0.7800424504336374, 2.5466091811087085e-307 ], "nsamples": 1500 }, "test": { "pearson": [ 0.7580850707783013, 6.51859239939566e-258 ], "spearman": [ 0.7560754511600052, 8.850873074119353e-256 ], "nsamples": 1379 }, "all": { "pearson": { "all": 0.7620304519086235, "mean": 0.7639455185233365, "wmean": 0.7616627780381048 }, "spearman": { "all": 0.7488342650348635, "mean": 0.7561450371089601, "wmean": 0.7444116088962087 } } }, "SICKRelatedness": { "train": { "pearson": [ 0.7660762263543586, 0.0 ], "spearman": [ 0.6943387839868075, 0.0 ], "nsamples": 4500 }, "dev": { "pearson": [ 0.7787463535407387, 6.565381473913046e-103 ], "spearman": [ 0.724076491124677, 2.2464753888818782e-82 ], "nsamples": 500 }, "test": { "pearson": [ 0.7585953923013815, 0.0 ], "spearman": [ 0.6847050750070173, 0.0 ], "nsamples": 4927 }, "all": { "pearson": { "all": 0.7629449380408607, "mean": 0.7678059907321595, "wmean": 0.7630014801283258 }, "spearman": { "all": 0.6909722716792152, "mean": 0.7010401167061673, "wmean": 0.6910551705512789 } } }, "MR": { "devacc": 81.13, "acc": 80.72, "ndev": 10662, "ntest": 10662 }, "CR": { "devacc": 87.1, "acc": 85.35, "ndev": 3775, "ntest": 3775 }, "SUBJ": { "devacc": 95.37, "acc": 94.83, "ndev": 10000, "ntest": 10000 }, "MPQA": { "devacc": 88.87, "acc": 88.84, "ndev": 10606, "ntest": 10606 }, "SST2": { "devacc": 83.72, "acc": 84.84, "ndev": 872, "ntest": 1821 }, "TREC": { "devacc": 79.31, "acc": 85.4, "ndev": 5452, "ntest": 500 }, "MRPC": { "devacc": 71.32, "acc": 68.93, "f1": 78.8, "ndev": 4076, "ntest": 1725 }, "STS12": { "MSRpar": { "pearson": [ 0.4802609709856894, 1.5548549952823705e-44 ], "spearman": [ 0.4768543559246008, 7.614791350529852e-44 ], "nsamples": 750 }, "MSRvid": { "pearson": [ 0.8263319209416204, 1.080221265522035e-188 ], "spearman": [ 0.8252791857619619, 8.356577398931468e-188 ], "nsamples": 750 }, "SMTeuroparl": { "pearson": [ 0.5184489923467462, 6.1406360676578446e-33 ], "spearman": [ 0.6230131131079721, 1.0590062591812851e-50 ], "nsamples": 459 }, "surprise.OnWN": { "pearson": [ 0.7315874470125636, 1.414322871092147e-126 ], "spearman": [ 0.6731301571473148, 4.0960228359103546e-100 ], "nsamples": 750 }, "surprise.SMTnews": { "pearson": [ 0.6259156885670977, 8.739441951714742e-45 ], "spearman": [ 0.6148988213247114, 7.33074623929432e-43 ], "nsamples": 399 }, "all": { "pearson": { "all": 0.623383289827507, "mean": 0.6365090039707434, "wmean": 0.6487592346944445 }, "spearman": { "all": 0.606079829047924, "mean": 0.6426351266533122, "wmean": 0.6476047048746869 } } }, "STS13": { "FNWN": { "pearson": [ 0.5661863937046698, 2.0580012282455086e-17 ], "spearman": [ 0.5912700246061035, 3.3253838782016723e-19 ], "nsamples": 189 }, "headlines": { "pearson": [ 0.7825601400543155, 4.2299360961974125e-156 ], "spearman": [ 0.7841745697183816, 3.6476628073513e-157 ], "nsamples": 750 }, "OnWN": { "pearson": [ 0.794529783804957, 3.311330886766186e-123 ], "spearman": [ 0.7834024923042585, 1.4890472209808275e-117 ], "nsamples": 561 }, "all": { "pearson": { "all": 0.7471361547131256, "mean": 0.7144254391879808, "wmean": 0.7597736947770001 }, "spearman": { "all": 0.7564536182897523, "mean": 0.7196156955429145, "wmean": 0.7595798400813525 } } }, "STS14": { "deft-forum": { "pearson": [ 0.5270887834584671, 1.541470939768766e-33 ], "spearman": [ 0.5112851401228272, 2.4363986591055635e-31 ], "nsamples": 450 }, "deft-news": { "pearson": [ 0.7735386895034391, 5.576824423082716e-61 ], "spearman": [ 0.721172124183031, 1.986170265393856e-49 ], "nsamples": 300 }, "headlines": { "pearson": [ 0.7545133819317689, 5.912159102599232e-139 ], "spearman": [ 0.7126401838119187, 2.8437071308486684e-117 ], "nsamples": 750 }, "images": { "pearson": [ 0.8323456329452497, 6.921690784176308e-194 ], "spearman": [ 0.7937433816131229, 1.1474485172111374e-163 ], "nsamples": 750 }, "OnWN": { "pearson": [ 0.8297923551143387, 1.175057263022769e-191 ], "spearman": [ 0.826236205234858, 1.3018028294306147e-188 ], "nsamples": 750 }, "tweet-news": { "pearson": [ 0.7558799639584549, 9.788201645414497e-140 ], "spearman": [ 0.6845652322701915, 8.460582099195036e-105 ], "nsamples": 750 }, "all": { "pearson": { "all": 0.7159862351857734, "mean": 0.7455264678186196, "wmean": 0.7596400159652537 }, "spearman": { "all": 0.6704800621775756, "mean": 0.7082737112059916, "wmean": 0.7224849873354 } } }, "STS15": { "answers-forums": { "pearson": [ 0.7317275686935678, 4.532297271121567e-64 ], "spearman": [ 0.7407727747177331, 1.972401483293033e-66 ], "nsamples": 375 }, "answers-students": { "pearson": [ 0.7050737483230498, 9.136794973091405e-114 ], "spearman": [ 0.7064217578173171, 2.2091874983666628e-114 ], "nsamples": 750 }, "belief": { "pearson": [ 0.783394736380794, 4.7974747142716894e-79 ], "spearman": [ 0.7998064345059992, 1.0770053295383067e-84 ], "nsamples": 375 }, "headlines": { "pearson": [ 0.813583787598092, 2.5592603175894345e-178 ], "spearman": [ 0.8095733522782433, 3.220682833365799e-175 ], "nsamples": 750 }, "images": { "pearson": [ 0.8641888241232765, 2.5653257492197874e-225 ], "spearman": [ 0.869236692916196, 4.9395933314044357e-231 ], "nsamples": 750 }, "all": { "pearson": { "all": 0.7901758719755425, "mean": 0.779593733023756, "wmean": 0.7851018781453998 }, "spearman": { "all": 0.7993930145058169, "mean": 0.7851622024470977, "wmean": 0.7888803519059057 } } }, "STS16": { "answer-answer": { "pearson": [ 0.732325384111771, 6.031948033432114e-44 ], "spearman": [ 0.7331760524557248, 4.2918539327373104e-44 ], "nsamples": 254 }, "headlines": { "pearson": [ 0.7751129917838497, 3.6445698860447696e-51 ], "spearman": [ 0.7759120405190852, 2.4798809912199763e-51 ], "nsamples": 249 }, "plagiarism": { "pearson": [ 0.8073018229023751, 3.908432651513822e-54 ], "spearman": [ 0.8201128848961844, 3.380055386535779e-57 ], "nsamples": 230 }, "postediting": { "pearson": [ 0.8262293698872258, 2.999128754083028e-62 ], "spearman": [ 0.8467835716176686, 2.7503849185111653e-68 ], "nsamples": 244 }, "question-question": { "pearson": [ 0.7751346076550414, 3.733475629577834e-43 ], "spearman": [ 0.7850104595896853, 6.270227463768292e-45 ], "nsamples": 209 }, "all": { "pearson": { "all": 0.7817024776005017, "mean": 0.7832208352680526, "wmean": 0.7827118895771512 }, "spearman": { "all": 0.7931904794778135, "mean": 0.7921990018156697, "wmean": 0.7915153089950118 } } }, "eval_senteval-STS12": 0.606079829047924, "eval_senteval-STS13": 0.7564536182897523, "eval_senteval-STS14": 0.6704800621775756, "eval_senteval-STS15": 0.7993930145058169, "eval_senteval-STS16": 0.7931904794778135, "eval_senteval-STSBenchmark": 0.7488342650348635, "eval_senteval-SICKRelatedness": 0.6909722716792152, "eval_senteval-avg_sts_7": 0.7236290771732802, "eval_senteval-MR": 81.13, "eval_senteval-CR": 87.1, "eval_senteval-SUBJ": 95.37, "eval_senteval-MPQA": 88.87, "eval_senteval-SST2": 83.72, "eval_senteval-TREC": 79.31, "eval_senteval-MRPC": 71.32, "eval_senteval-avg_transfer": 83.83142857142856 }