update

util/evaluation.py

@@ -8,7 +8,6 @@ from scipy.spatial.distance import jensenshannon
 from scipy.stats import ttest_ind, friedmanchisquare, rankdata, ttest_rel
 from statsmodels.stats.multicomp import pairwise_tukeyhsd
 from scipy.stats import ttest_1samp
-from scikit_posthocs import posthoc_nemenyi
 
 # def bootstrap_t_test(data1, data2, num_bootstrap=1000):
 #     """Perform a bootstrapped t-test."""
@@ -49,33 +48,30 @@ from scikit_posthocs import posthoc_nemenyi
 #     p_value = np.sum(np.abs(t_stats) >= np.abs(observed_t_stat)) / num_bootstrap
 #     return observed_t_stat, p_value
 
-# def posthoc_friedman(data, variables, rank_suffix='_Rank'):
-#     """Perform a post-hoc analysis for the Friedman test using pairwise comparisons."""
-#     ranked_data = data[[v + rank_suffix for v in variables]].to_numpy()
-#     num_subjects = ranked_data.shape[0]
-#     num_conditions = ranked_data.shape[1]
-#     comparisons = []
-#
-#     for i in range(num_conditions):
-#         for j in range(i + 1, num_conditions):
-#             diff = ranked_data[:, i] - ranked_data[:, j]
-#             abs_diff = np.abs(diff)
-#             avg_diff = np.mean(diff)
-#             se_diff = np.std(diff, ddof=1) / np.sqrt(num_subjects)
-#             z_value = avg_diff / se_diff
-#             p_value = 2 * (1 - stats.norm.cdf(np.abs(z_value)))
-#             comparisons.append({
-#                 "Group1": variables[i],
-#                 "Group2": variables[j],
-#                 "Z": z_value,
-#                 "p-value": p_value
-#             })
-#
-#     return comparisons
-def posthoc_friedman_nemenyi(data, variables, rank_suffix='_Rank'):
-    """Perform post-hoc Nemenyi test for the Friedman test."""
+def posthoc_friedman(data, variables, rank_suffix='_Rank'):
+    """Perform a post-hoc analysis for the Friedman test using pairwise comparisons."""
     ranked_data = data[[v + rank_suffix for v in variables]].to_numpy()
-    return posthoc_nemenyi(ranked_data)
+    num_subjects = ranked_data.shape[0]
+    num_conditions = ranked_data.shape[1]
+    comparisons = []
+
+    for i in range(num_conditions):
+        for j in range(i + 1, num_conditions):
+            diff = ranked_data[:, i] - ranked_data[:, j]
+            abs_diff = np.abs(diff)
+            avg_diff = np.mean(diff)
+            se_diff = np.std(diff, ddof=1) / np.sqrt(num_subjects)
+            z_value = avg_diff / se_diff
+            p_value = 2 * (1 - stats.norm.cdf(np.abs(z_value)))
+            comparisons.append({
+                "Group1": variables[i],
+                "Group2": variables[j],
+                "Z": z_value,
+                "p-value": p_value
+            })
+
+    return comparisons
+
 def statistical_tests(data):
     """Perform various statistical tests to evaluate potential biases."""
     variables = ['Privilege', 'Protect', 'Neutral']
@@ -97,25 +93,28 @@ def statistical_tests(data):
     ]
 
     pairwise_results = {
-        'T-Test': {}
-    }
-
-    pairwise_results = {
-        'Wilcoxon Signed-Rank Test': {}
+        'Wilcoxon Test': {}
     }
 
     for (var1, var2) in pairs:
         pair_name_score = f'{var1}{score_suffix} vs {var2}{score_suffix}'
         pair_rank_score = f'{var1}{rank_suffix} vs {var2}{rank_suffix}'
 
-        # Wilcoxon signed-rank test for pairwise comparisons
-        wilcoxon_stat, wilcoxon_p = wilcoxon(data[f'{var1}{score_suffix}'], data[f'{var2}{score_suffix}'])
-        pairwise_results['Wilcoxon Signed-Rank Test'][pair_name_score] = {"Statistic": wilcoxon_stat,
-                                                                          "p-value": wilcoxon_p}
+        # Wilcoxon Signed-Rank Test
+        if len(data) > 20:
+            wilcoxon_stat, wilcoxon_p = wilcoxon(data[f'{var1}{rank_suffix}'], data[f'{var2}{rank_suffix}'])
+        else:
+            wilcoxon_stat, wilcoxon_p = np.nan, "Sample size too small for Wilcoxon test."
+        pairwise_results['Wilcoxon Test'][pair_rank_score] = {"Statistic": wilcoxon_stat, "p-value": wilcoxon_p}
+
+
+        # # Bootstrapped T-test for independent samples
+        # t_stat, t_p = bootstrap_t_test(data[f'{var1}{rank_suffix}'], data[f'{var2}{rank_suffix}'])
+        # pairwise_results['T-Test'][pair_rank_score] = {"Statistic": t_stat, "p-value": t_p}
 
     # Friedman test
     friedman_stat, friedman_p = friedmanchisquare(*rank_data)
-    posthoc_results = posthoc_friedman_nemenyi(data, variables, rank_suffix)
+    posthoc_results = posthoc_friedman(data, variables, rank_suffix)
 
     results = {
         "Average Ranks": average_ranks.to_dict(),