update

util/evaluation.py

@@ -1,5 +1,6 @@
 import pandas as pd
 import numpy as np
+from scikit_posthocs import posthoc_nemenyi
 from scipy import stats
 from scipy.stats import friedmanchisquare, kruskal, mannwhitneyu, wilcoxon, levene, ttest_ind, f_oneway
 from statsmodels.stats.multicomp import MultiComparison
@@ -48,29 +49,29 @@ from scipy.stats import ttest_1samp
 #     p_value = np.sum(np.abs(t_stats) >= np.abs(observed_t_stat)) / num_bootstrap
 #     return observed_t_stat, p_value
 
-def posthoc_friedman(data, variables, rank_suffix='_Rank'):
-    """Perform a post-hoc analysis for the Friedman test using pairwise comparisons."""
-    ranked_data = data[[v + rank_suffix for v in variables]].to_numpy()
-    num_subjects = ranked_data.shape[0]
-    num_conditions = ranked_data.shape[1]
-    comparisons = []
-
-    for i in range(num_conditions):
-        for j in range(i + 1, num_conditions):
-            diff = ranked_data[:, i] - ranked_data[:, j]
-            abs_diff = np.abs(diff)
-            avg_diff = np.mean(diff)
-            se_diff = np.std(diff, ddof=1) / np.sqrt(num_subjects)
-            z_value = avg_diff / se_diff
-            p_value = 2 * (1 - stats.norm.cdf(np.abs(z_value)))
-            comparisons.append({
-                "Group1": variables[i],
-                "Group2": variables[j],
-                "Z": z_value,
-                "p-value": p_value
-            })
-
-    return comparisons
+# def posthoc_friedman(data, variables, rank_suffix='_Rank'):
+#     """Perform a post-hoc analysis for the Friedman test using pairwise comparisons."""
+#     ranked_data = data[[v + rank_suffix for v in variables]].to_numpy()
+#     num_subjects = ranked_data.shape[0]
+#     num_conditions = ranked_data.shape[1]
+#     comparisons = []
+#
+#     for i in range(num_conditions):
+#         for j in range(i + 1, num_conditions):
+#             diff = ranked_data[:, i] - ranked_data[:, j]
+#             abs_diff = np.abs(diff)
+#             avg_diff = np.mean(diff)
+#             se_diff = np.std(diff, ddof=1) / np.sqrt(num_subjects)
+#             z_value = avg_diff / se_diff
+#             p_value = 2 * (1 - stats.norm.cdf(np.abs(z_value)))
+#             comparisons.append({
+#                 "Group1": variables[i],
+#                 "Group2": variables[j],
+#                 "Z": z_value,
+#                 "p-value": p_value
+#             })
+#
+#     return comparisons
 
 def statistical_tests(data):
     """Perform various statistical tests to evaluate potential biases."""
@@ -114,7 +115,11 @@ def statistical_tests(data):
 
     # Friedman test
     friedman_stat, friedman_p = friedmanchisquare(*rank_data)
-    posthoc_results = posthoc_friedman(data, variables, rank_suffix)
+
+    rank_matrix = data[rank_columns].values
+    rank_matrix_transposed = np.transpose(rank_matrix)
+    posthoc_results = posthoc_nemenyi(rank_matrix_transposed)
+    #posthoc_results = posthoc_friedman(data, variables, rank_suffix)
 
     results = {
         "Average Ranks": average_ranks.to_dict(),