model-evaluation / utils.py
AlekseyKorshuk's picture
updates
dd2beab
import itertools
import random
def get_matchmaking(client, models, is_anonymous=True):
model_a, model_b = random.sample(models, k=2)
return model_a, model_b
sheet = client.open("Chat Arena").sheet1
records = sheet.get_all_records()
records = [
{
col: record.get(col, None)
for col in ['model_a', 'model_b']
} for record in records if record["is_anonymous"] == is_anonymous
]
combinations = list(itertools.combinations_with_replacement(models, 2))
combinations = [frozenset(combination) for combination in combinations if len(set(combination)) > 1]
records = [
frozenset(record.values()) for record in records
]
repetitions_count = {combination: 0 for combination in combinations}
for record in records:
repetitions_count[record] += 1
sorted_repetitions = dict(sorted(repetitions_count.items(), key=lambda item: item[1]))
less_common = list(sorted_repetitions.keys())[0]
less_common = list(less_common)
random.shuffle(less_common)
model_a, model_b = less_common
return model_a, model_b