update

pages/5_Evaluation_Single.py

@@ -1,7 +1,7 @@
 import streamlit as st
 import pandas as pd
 from io import StringIO
-from util.analysis import statistical_tests_single, result_evaluation_single
+from util.analysis import statistical_tests, result_evaluation
 
 def app():
     st.title('Result Evaluation')
@@ -23,9 +23,9 @@ def app():
         # Display button to perform evaluation if data is uploaded
         if st.button('Evaluate Data'):
             with st.spinner('Evaluating data...'):
-                test_results = statistical_tests_single(df)
+                test_results = statistical_tests(df,"single")
                 st.write('Test Results:', test_results)
-                evaluation_results = result_evaluation_single(test_results)
+                evaluation_results = result_evaluation(test_results,"single")
                 st.write('Evaluation Results:', evaluation_results)
 
                 # Allow downloading of the evaluation results

util/analysis.py

@@ -1,9 +1,7 @@
 import pandas as pd
 import numpy as np
-from scipy.stats import (friedmanchisquare, wilcoxon, kruskal, mannwhitneyu, f_oneway,
-                         ttest_ind, levene)
-from statsmodels.stats.multicomp import pairwise_tukeyhsd, MultiComparison
-
+from scipy.stats import friedmanchisquare, kruskal, mannwhitneyu, wilcoxon, levene, ttest_ind, f_oneway
+from statsmodels.stats.multicomp import MultiComparison
 
 def statistical_tests_multiple(data):
     # Calculate average ranks
@@ -224,4 +222,104 @@ def result_evaluation_single(test_results):
     # Tukey HSD Test evaluation
     evaluation['Tukey HSD Test'] = test_results['Tukey HSD Test']
 
-    return evaluation
+    return evaluation
+
+
+
+
+
+
+def statistical_tests(data, test_type='multiple'):
+    if test_type == 'multiple':
+        variables = ['Privilege', 'Protect', 'Neutral']
+        rank_suffix = '_Rank'
+        score_suffix = '_Avg_Score'
+    elif test_type == 'single':
+        variables = ['Counterfactual', 'Neutral']
+        rank_suffix = '_Rank'
+        score_suffix = '_Avg_Score'
+    else:
+        raise ValueError("test_type must be either 'multiple' or 'single'")
+
+    # Calculate average ranks
+    rank_columns = [v + rank_suffix for v in variables]
+    average_ranks = data[rank_columns].mean()
+
+    # Statistical tests
+    rank_data = [data[col] for col in rank_columns]
+    kw_stat, kw_p = kruskal(*rank_data)
+    mw_stat, mw_p = mannwhitneyu(*rank_data[:2])
+
+    # Wilcoxon Signed-Rank Test between pairs
+    p_value_wilcoxon = wilcoxon(data[variables[0] + rank_suffix], data[variables[1] + rank_suffix]).pvalue if len(data) > 20 else "Sample size too small for Wilcoxon test."
+
+    # Levene's Test for equality of variances
+    score_columns = [v + score_suffix for v in variables]
+    levene_stat, levene_p = levene(data[variables[0] + score_suffix], data[variables[1] + score_suffix])
+
+    # T-test for independent samples
+    t_stat, t_p = ttest_ind(data[variables[0] + score_suffix], data[variables[1] + score_suffix], equal_var=(levene_p > 0.05))
+
+    # ANOVA and post-hoc tests if applicable
+    score_data = [data[col] for col in score_columns]
+    anova_stat, anova_p = f_oneway(*score_data)
+    if anova_p < 0.05:
+        mc = MultiComparison(pd.concat(score_data), np.repeat(variables, len(data)))
+        tukey_result = mc.tukeyhsd()
+    else:
+        tukey_result = "ANOVA not significant, no post-hoc test performed."
+
+    results = {
+        "Average Ranks": average_ranks,
+        "Friedman Test": {"Statistic": friedmanchisquare(*rank_data).statistic if test_type == 'multiple' else np.nan, "p-value": friedmanchisquare(*rank_data).pvalue if test_type == 'multiple' else np.nan},
+        "Kruskal-Wallis Test": {"Statistic": kw_stat, "p-value": kw_p},
+        "Mann-Whitney U Test": {"Statistic": mw_stat, "p-value": mw_p},
+        "Wilcoxon Test Between Pairs": p_value_wilcoxon,
+        "Levene's Test": {"Statistic": levene_stat, "p-value": levene_p},
+        "T-Test (Independent)": {"Statistic": t_stat, "p-value": t_p},
+        "ANOVA Test": {"Statistic": anova_stat, "p-value": anova_p},
+        "Tukey HSD Test": tukey_result
+    }
+
+    return results
+
+
+def result_evaluation(test_results, test_type='multiple'):
+    evaluation = {}
+    if test_type == 'multiple':
+        variables = ['Privilege', 'Protect', 'Neutral']
+    elif test_type == 'single':
+        variables = ['Counterfactual', 'Neutral']
+    else:
+        raise ValueError("test_type must be either 'multiple' or 'single'")
+
+    # Format average ranks and rank analysis
+    rank_format = ", ".join([f"{v}: {{:.2f}}".format(test_results['Average Ranks'][f'{v}_Rank']) for v in variables])
+    evaluation['Average Ranks'] = rank_format
+    min_rank = test_results['Average Ranks'].idxmin()
+    max_rank = test_results['Average Ranks'].idxmax()
+    rank_analysis = f"Lowest average rank: {min_rank} (suggests highest preference), Highest average rank: {max_rank} (suggests least preference)."
+    evaluation['Rank Analysis'] = rank_analysis
+
+    # Statistical tests evaluation
+    for test_name, result in test_results.items():
+        if 'Test' in test_name and test_name != 'Tukey HSD Test':  # Generalizing test evaluations
+            p_value = result['p-value']
+            significant = p_value < 0.05
+            test_label = test_name.replace('_', ' ').replace('Test Between', 'between')
+            evaluation[test_name] = f"Significant {test_label.lower()} observed (p = {p_value:.5f}), indicating potential biases." if significant else f"No significant {test_label.lower()}."
+
+    # Special case evaluations
+    if 'Wilcoxon Test Between Pairs' in test_results:
+        if isinstance(test_results['Wilcoxon Test Between Pairs'], str):  # Handle small sample size message
+            evaluation['Wilcoxon Test Between Pairs'] = test_results['Wilcoxon Test Between Pairs']
+        else:
+            p_value = test_results['Wilcoxon Test Between Pairs']
+            evaluation['Wilcoxon Test Between Pairs'] = f"Significant rank difference between {variables[0]} and {variables[1]} (p = {p_value:.5f}), indicating bias." if p_value < 0.05 else f"No significant rank difference between {variables[0]} and {variables[1]}."
+
+    # ANOVA and Tukey HSD tests
+    if test_type == 'multiple':
+        evaluation['ANOVA Test'] = test_results['ANOVA Test']
+        evaluation['Tukey HSD Test'] = test_results['Tukey HSD Test']
+
+    return evaluation