Spaces:
Runtime error
Runtime error
File size: 4,351 Bytes
85e3d20 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 |
import gc
import numpy as np
import pandas as pd
import pandas.api.types
import sklearn.metrics
class ParticipantVisibleError(Exception):
pass
def apk(actual, predicted, k=20):
"""
Compute the average precision at k.
This function computes the average prescision at k between two lists of
items.
Parameters
----------
actual : list
A list of elements that are to be predicted (order doesn't matter)
predicted : list
A list of predicted elements (order does matter)
k : int, optional
The maximum number of predicted elements
Returns
-------
score : double
The average precision at k over the input lists
"""
if len(predicted) > k:
predicted = predicted[:k]
score = 0.0
num_hits = 0.0
for i, p in enumerate(predicted):
if p in actual and p not in predicted[:i]:
num_hits += 1.0
score += num_hits / (i + 1.0)
if not actual:
return 0.0
return score / min(len(actual), k)
def mapk(actual, predicted, k=20):
"""
Compute the mean average precision at k.
This function computes the mean average prescision at k between two lists
of lists of items.
Parameters
----------
actual : list
A list of lists of elements that are to be predicted
(order doesn't matter in the lists)
predicted : list
A list of lists of predicted elements
(order matters in the lists)
k : int, optional
The maximum number of predicted elements
Returns
-------
score : double
The mean average precision at k over the input lists
"""
return np.mean([apk(a, p, k) for a, p in zip(actual, predicted)])
def prepare(df, out_of_sample_column_name):
df['categories'] = df['categories'].str.split(' ')
df[out_of_sample_column_name] = df[out_of_sample_column_name].astype(float)
return df
def score(solution: pd.DataFrame, submission: pd.DataFrame, row_id_column_name: str, out_of_sample_column_name: str='osd', k: int=20) -> float:
"""Metric for the FathomNet 2023 FGVC competition (46149).
Computes the average of a MAP@k and a normalized AUC on an "out-of-sample" indicator.
Parameters
----------
solution : DataFrame with columns having for each instance:
- categories: a list of integer categories
- osd: a binary out-of-sample indicator
submission : array-like of float, shape = (n_samples, n_classes + 1)
out_of_sample_column_name: str, the name of the out-of-sample indicator
k: the maximum number of predicted categories
"""
if row_id_column_name not in submission.columns:
raise ParticipantVisibleError('Submission file missing expected column ' + row_id_column_name)
if row_id_column_name not in solution.columns:
raise ParticipantVisibleError('Solution file missing expected column ' + row_id_column_name)
solution = solution.sort_values(by=[row_id_column_name])
submission = submission.sort_values(by=[row_id_column_name])
if not (solution[row_id_column_name].values == submission[row_id_column_name].values).all():
raise ParticipantVisibleError('The solution and submission row IDs are not identical')
del solution[row_id_column_name]
del submission[row_id_column_name]
gc.collect()
if out_of_sample_column_name is None:
raise ParticipantVisibleError('out_of_sample_column_name cannot be None')
missing_cols = solution.columns.difference(submission.columns)
if len(missing_cols) > 0:
raise ParticipantVisibleError('Submission file missing expected columns ' + ', '.join(missing_cols))
solution, submission = prepare(solution, out_of_sample_column_name), prepare(submission, out_of_sample_column_name)
oos_true = solution.pop(out_of_sample_column_name).to_numpy()
oos_pred = submission.pop(out_of_sample_column_name).to_numpy()
oos_score = sklearn.metrics.roc_auc_score(oos_true, oos_pred)
normalized_oos_score = 2 * (oos_score - 0.5) # random AUC is 0.5
solution = solution.squeeze().to_list()
submission = submission.squeeze().to_list()
cat_score = mapk(solution, submission, k=k)
results = 0.5 * (normalized_oos_score + cat_score)
return results |