Compute CIs

README.md

@@ -1,17 +1,17 @@
 ---
-title: Pair2Rank
+title: Evalica
 emoji: 💞
 colorFrom: green
 colorTo: purple
 sdk: gradio
 python_version: 3.11
-sdk_version: 5.9.1
+sdk_version: 5.12.0
 app_file: app.py
 pinned: true
 license: apache-2.0
 ---
 
-# Pair2Rank
+# Evalica
 
 [Evalica](https://github.com/dustalov/evalica) is a library for pairwise comparisons as described in paper
 Reliable, Reproducible, and Really Fast Leaderboards with Evalica

app.py

@@ -17,6 +17,7 @@
 __author__ = "Dmitry Ustalov"
 __license__ = "Apache 2.0"
 
+from collections.abc import Callable
 from typing import BinaryIO, cast
 
 import evalica
@@ -42,45 +43,45 @@ def visualize(df_pairwise: pd.DataFrame) -> Figure:
 
 
 def counting(xs: "pd.Series[str]", ys: "pd.Series[str]",
-             ws: "pd.Series[Winner]") -> tuple["pd.Series[str]", dict[str, int]]:  # type: ignore[type-var]
-    result = evalica.counting(xs, ys, ws)
-    return result.scores, result.index
+             ws: "pd.Series[Winner]", index: dict[str, int]) -> "pd.Series[float]":  # type: ignore[type-var]
+    result = evalica.counting(xs, ys, ws, index=index)
+    return result.scores
 
 
 def average_win_rate(xs: "pd.Series[str]", ys: "pd.Series[str]",
-                     ws: "pd.Series[Winner]") -> tuple["pd.Series[str]", dict[str, int]]:  # type: ignore[type-var]
-    result = evalica.counting(xs, ys, ws)
-    return result.scores, result.index
+                     ws: "pd.Series[Winner]", index: dict[str, int]) -> "pd.Series[float]":  # type: ignore[type-var]
+    result = evalica.counting(xs, ys, ws, index=index)
+    return result.scores
 
 
 def bradley_terry(xs: "pd.Series[str]", ys: "pd.Series[str]",
-                  ws: "pd.Series[Winner]") -> tuple["pd.Series[str]", dict[str, int]]:  # type: ignore[type-var]
-    result = evalica.bradley_terry(xs, ys, ws, tolerance=TOLERANCE, limit=LIMIT)
-    return result.scores, result.index
+                  ws: "pd.Series[Winner]", index: dict[str, int]) -> "pd.Series[float]":  # type: ignore[type-var]
+    result = evalica.bradley_terry(xs, ys, ws, index=index, tolerance=TOLERANCE, limit=LIMIT)
+    return result.scores
 
 
 def elo(xs: "pd.Series[str]", ys: "pd.Series[str]",
-        ws: "pd.Series[Winner]") -> tuple["pd.Series[str]", dict[str, int]]:  # type: ignore[type-var]
-    result = evalica.elo(xs, ys, ws)
-    return result.scores, result.index
+        ws: "pd.Series[Winner]", index: dict[str, int]) -> "pd.Series[float]":  # type: ignore[type-var]
+    result = evalica.elo(xs, ys, ws, index=index)
+    return result.scores
 
 
 def eigen(xs: "pd.Series[str]", ys: "pd.Series[str]",
-          ws: "pd.Series[Winner]") -> tuple["pd.Series[str]", dict[str, int]]:  # type: ignore[type-var]
-    result = evalica.eigen(xs, ys, ws, tolerance=TOLERANCE, limit=LIMIT)
-    return result.scores, result.index
+          ws: "pd.Series[Winner]", index: dict[str, int]) -> "pd.Series[float]":  # type: ignore[type-var]
+    result = evalica.eigen(xs, ys, ws, index=index, tolerance=TOLERANCE, limit=LIMIT)
+    return result.scores
 
 
 def pagerank(xs: "pd.Series[str]", ys: "pd.Series[str]",
-             ws: "pd.Series[Winner]") -> tuple["pd.Series[str]", dict[str, int]]:  # type: ignore[type-var]
-    result = evalica.pagerank(xs, ys, ws, tolerance=TOLERANCE, limit=LIMIT)
-    return result.scores, result.index
+             ws: "pd.Series[Winner]", index: dict[str, int]) -> "pd.Series[float]":  # type: ignore[type-var]
+    result = evalica.pagerank(xs, ys, ws, index=index, tolerance=TOLERANCE, limit=LIMIT)
+    return result.scores
 
 
 def newman(xs: "pd.Series[str]", ys: "pd.Series[str]",
-           ws: "pd.Series[Winner]") -> tuple["pd.Series[str]", dict[str, int]]:  # type: ignore[type-var]
-    result = evalica.newman(xs, ys, ws, tolerance=TOLERANCE, limit=LIMIT)
-    return result.scores, result.index
+           ws: "pd.Series[Winner]", index: dict[str, int]) -> "pd.Series[float]":  # type: ignore[type-var]
+    result = evalica.newman(xs, ys, ws, index=index, tolerance=TOLERANCE, limit=LIMIT)
+    return result.scores
 
 
 ALGORITHMS = {
@@ -103,11 +104,56 @@ def largest_strongly_connected_component(df_pairs: pd.DataFrame) -> set[str]:
     return cast(set[str], largest)
 
 
+def estimate(df_pairs: pd.DataFrame,
+             algorithm: Callable[[  # type: ignore[type-var]
+                 "pd.Series[str]", "pd.Series[str]", "pd.Series[Winner]", dict[str, int]],
+             "pd.Series[float]",
+             ],
+             index: dict[str, int]) -> pd.DataFrame:
+    scores = algorithm(df_pairs["left"], df_pairs["right"], df_pairs["winner"], index)
+
+    df_result = pd.DataFrame(data={"score": scores}, index=index)
+    df_result.index.name = "item"
+
+    return df_result
+
+
+def bootstrap(df_pairs: pd.DataFrame,
+              algorithm: Callable[[  # type: ignore[type-var]
+                  "pd.Series[str]", "pd.Series[str]", "pd.Series[Winner]", dict[str, int]],
+              "pd.Series[float]",
+              ],
+              index: dict[str, int],
+              rounds: int) -> pd.DataFrame:
+    scores: list[pd.Series[float]] = []  # assuming model names are strings
+
+    for r in range(rounds):
+        df_sample = df_pairs.sample(frac=1.0, replace=True, random_state=r)
+
+        sample_scores = algorithm(df_sample["left"], df_sample["right"], df_sample["winner"], index)
+
+        scores.append(sample_scores)
+
+    df_bootstrap = pd.DataFrame(scores, columns=index)
+
+    ratings = df_bootstrap.quantile(.5)
+
+    ci = df_bootstrap.apply(lambda row: (
+        row.quantile(.025).item(), row.quantile(.975).item(),
+    ), axis=0, result_type="reduce")
+
+    df_result = pd.DataFrame({"score": ratings, "ci": ci})
+    df_result.index.name = "item"
+
+    return df_result
+
+
 def handler(
         file: BinaryIO,
         algorithm: str,
         filtered: bool,
         truncated: bool,
+        rounds: int,
 ) -> tuple[pd.DataFrame, Figure]:
     if file is None:
         raise gr.Error("File must be uploaded")
@@ -127,6 +173,9 @@ def handler(
         raise gr.Error("Allowed winner values: left, right, tie")
 
     df_pairs = df_pairs[["left", "right", "winner"]]
+    df_pairs["winner"] = df_pairs["winner"].map(
+        {"left": Winner.X, "right": Winner.Y, "tie": Winner.Draw},
+    )
 
     df_pairs = df_pairs.dropna(axis=0)
 
@@ -135,13 +184,12 @@ def handler(
 
         df_pairs = df_pairs.drop(df_pairs[~(df_pairs["left"].isin(largest) & df_pairs["right"].isin(largest))].index)
 
-    xs, ys = df_pairs["left"], df_pairs["right"]
-    ws = df_pairs["winner"].map({"left": Winner.X, "right": Winner.Y, "tie": Winner.Draw})
+    *_, index = evalica.indexing(xs=df_pairs["left"], ys=df_pairs["right"])
 
-    scores, index = ALGORITHMS[algorithm](xs, ys, ws)
-
-    df_result = pd.DataFrame(data={"score": scores}, index=index)
-    df_result.index.name = "item"
+    if rounds:
+        df_result = bootstrap(df_pairs, ALGORITHMS[algorithm], index, rounds)
+    else:
+        df_result = estimate(df_pairs, ALGORITHMS[algorithm], index)
 
     df_result["pairs"] = pd.Series(0, dtype=int, index=index).add(
         df_pairs.groupby("left")["left"].count(), fill_value=0,
@@ -165,6 +213,14 @@ def handler(
 
     fig = visualize(df_pairwise)
 
+    df_result["score"] = df_result["score"].apply(lambda x: f"{x:.03f}")
+
+    if "ci" in df_result.columns:
+        df_result["ci"] = df_result.apply(
+            lambda row: f"({row['score'] - row['ci'][0]:.03f}; {row['ci'][1] - row['score']:.03f})",
+            axis=1,
+        )
+
     return df_result, fig
 
 
@@ -195,10 +251,17 @@ def main() -> None:
                 info="Perform the entire computation but output only five head and five tail items, "
                      "avoiding overlap.",
             ),
+            gr.Number(
+                value=0,
+                minimum=0,
+                maximum=10000,
+                label="Bootstrap Rounds",
+                info="Number of bootstrap rounds to perform for estimating the confidence interval.",
+            ),
         ],
         outputs=[
             gr.Dataframe(
-                headers=["item", "score", "pairs", "rank"],
+                headers=["item", "score", "ci", "pairs", "rank"],
                 label="Ranking",
             ),
             gr.Plot(
@@ -215,9 +278,11 @@ def main() -> None:
             ["llmfao.csv", "Bradley-Terry (1952)", False, True],
             ["llmfao.csv", "Elo (1960)", False, True],
         ],
-        title="Pair2Rank: Turn Your Side-by-Side Comparisons into Ranking!",
+        title="Evalica: Turn Your Side-by-Side Comparisons into Ranking!",
         description="""
-This easy-to-use tool transforms pairwise comparisons (aka side-by-side) to a meaningful ranking of items.
+        """.strip(),
+        article="""
+This easy-to-use tool transforms pairwise comparisons (*aka* side-by-side) to a meaningful ranking of items.
 
 As an input, it expects a comma-separated (CSV) file with a header containing the following columns:
 
@@ -228,8 +293,7 @@ As an input, it expects a comma-separated (CSV) file with a header containing th
 Possible values for `winner` are `left`, `right`, or `tie`. The provided examples might be a good starting point.
 
 As the output, this tool provides a table with items, their estimated scores, and ranks.
-        """.strip(),
-        article="""
+
 **More Evalica:**
 
 - Paper: TBD ([arXiv](https://arxiv.org/abs/2412.11314))

requirements.txt

@@ -1,3 +1,3 @@
-evalica
+evalica[gradio]
 networkx
 plotly