File size: 4,516 Bytes
ee6e792 23fcc73 0b04461 23fcc73 d977853 0b04461 23fcc73 ee6e792 23fcc73 ee6e792 d977853 ee6e792 d977853 ee6e792 0b04461 ee6e792 d977853 ee6e792 d977853 ee6e792 d977853 c52bc65 0b04461 e2b8c08 0b04461 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 |
import pandas as pd
import gradio as gr
import matplotlib.pyplot as plt
import seaborn as sns
from typing import Tuple
import plotly.express as px
VOLUME_FACTOR_REGULARIZATION = 0.5
UNSCALED_WEIGHTED_ACCURACY_INTERVAL = (-0.5, 100.5)
SCALED_WEIGHTED_ACCURACY_INTERVAL = (0, 1)
# tools palette as dictionary
tools_palette = {
"prediction-request-reasoning": "darkorchid",
"claude-prediction-offline": "rebeccapurple",
"prediction-request-reasoning-claude": "slateblue",
"prediction-request-rag-claude": "steelblue",
"prediction-online": "darkcyan",
"prediction-offline": "mediumaquamarine",
"claude-prediction-online": "mediumseagreen",
"prediction-online-sme": "yellowgreen",
"prediction-url-cot-claude": "gold",
"prediction-offline-sme": "orange",
"prediction-request-rag": "chocolate",
}
HEIGHT = 400
WIDTH = 1100
def scale_value(
value: float,
min_max_bounds: Tuple[float, float],
scale_bounds: Tuple[float, float] = (0, 1),
) -> float:
"""Perform min-max scaling on a value."""
min_, max_ = min_max_bounds
current_range = max_ - min_
# normalize between 0-1
std = (value - min_) / current_range
# scale between min_bound and max_bound
min_bound, max_bound = scale_bounds
target_range = max_bound - min_bound
return std * target_range + min_bound
def get_weighted_accuracy(row, global_requests: int):
"""Function to compute the weighted accuracy of a tool"""
return scale_value(
(
row["tool_accuracy"]
+ (row["total_requests"] / global_requests) * VOLUME_FACTOR_REGULARIZATION
),
UNSCALED_WEIGHTED_ACCURACY_INTERVAL,
SCALED_WEIGHTED_ACCURACY_INTERVAL,
)
def compute_weighted_accuracy(tools_accuracy: pd.DataFrame):
global_requests = tools_accuracy.total_requests.sum()
tools_accuracy["weighted_accuracy"] = tools_accuracy.apply(
lambda x: get_weighted_accuracy(x, global_requests), axis=1
)
return tools_accuracy
def plot_tools_accuracy_graph(tools_accuracy_info: pd.DataFrame):
tools_accuracy_info = tools_accuracy_info.sort_values(
by="tool_accuracy", ascending=False
)
plt.figure(figsize=(25, 10))
plot = sns.barplot(
tools_accuracy_info,
x="tool_accuracy",
y="tool",
hue="tool",
dodge=False,
palette=tools_palette,
)
plt.xlabel("Mech tool_accuracy (%)", fontsize=20)
plt.ylabel("tool", fontsize=20)
plt.tick_params(axis="y", labelsize=12)
return gr.Plot(value=plot.get_figure())
def plot_tools_accuracy_rotated_graph(tools_accuracy_info: pd.DataFrame):
tools_accuracy_info = tools_accuracy_info.sort_values(
by="tool_accuracy", ascending=False
)
fig = px.bar(
tools_accuracy_info,
x="tool",
y="tool_accuracy",
color="tool",
color_discrete_map=tools_palette,
)
fig.update_layout(
xaxis_title="Tool",
yaxis_title="Mech tool_accuracy (%)",
)
fig.update_layout(width=WIDTH, height=HEIGHT)
# fig.update_xaxes(tickangle=45)
fig.update_xaxes(showticklabels=False)
return gr.Plot(
value=fig,
)
def plot_tools_weighted_accuracy_graph(tools_accuracy_info: pd.DataFrame):
tools_accuracy_info = tools_accuracy_info.sort_values(
by="weighted_accuracy", ascending=False
)
# Create the Seaborn bar plot
# sns.set_theme(palette="viridis")
plt.figure(figsize=(25, 10))
plot = sns.barplot(
tools_accuracy_info,
x="weighted_accuracy",
y="tool",
hue="tool",
dodge=False,
palette=tools_palette,
)
plt.xlabel("Weighted accuracy metric", fontsize=20)
plt.ylabel("tool", fontsize=20)
plt.tick_params(axis="y", labelsize=12)
return gr.Plot(value=plot.get_figure())
def plot_tools_weighted_accuracy_rotated_graph(
tools_accuracy_info: pd.DataFrame,
) -> gr.Plot:
tools_accuracy_info = tools_accuracy_info.sort_values(
by="weighted_accuracy", ascending=False
)
fig = px.bar(
tools_accuracy_info,
x="tool",
y="weighted_accuracy",
color="tool",
color_discrete_map=tools_palette,
)
fig.update_layout(
xaxis_title="Tool",
yaxis_title="Weighted accuracy metric",
)
fig.update_layout(width=WIDTH, height=HEIGHT)
# fig.update_xaxes(tickangle=45)
fig.update_xaxes(showticklabels=False)
return gr.Plot(
value=fig,
)
|