Merge branch 'main' of https://huggingface.co/spaces/huggingface/data-measurements-tool-2 into main

data_measurements/streamlit_utils.py

@@ -434,10 +434,16 @@ def npmi_show(paired_results):
         s.index.name = "word"
         npmi_cols = s.filter(like="npmi").columns
         count_cols = s.filter(like="count").columns
+        if s.shape[0] > 10000:
+            bias_thres = max(abs(s["npmi-bias"][5000]), abs(s["npmi-bias"][-5000]))
+            print(f"filtering with bias threshold: {bias_thres}")
+            s_filtered = s[s["npmi-bias"].abs() > bias_thres]
+        else:
+            s_filtered = s
         # TODO: This is very different look than the duplicates table above. Should probably standardize.
         cm = sns.palplot(sns.diverging_palette(270, 36, s=99, l=48, n=16))
         out_df = (
-            s.style.background_gradient(subset=npmi_cols, cmap=cm)
+            s_filtered.style.background_gradient(subset=npmi_cols, cmap=cm)
             .format(subset=npmi_cols, formatter="{:,.3f}")
             .format(subset=count_cols, formatter=int)
             .set_properties(

requirements.txt

@@ -10,7 +10,7 @@ iso_639==0.4.5
 datasets==1.15.1
 powerlaw==1.5
 numpy==1.19.5
-pandas==1.3.0
+pandas==1.0.0
 dataclasses==0.6
 iso639==0.1.4
 python_igraph==0.9.6
@@ -23,4 +23,4 @@ numexpr==2.7.3
 scikit-learn~=0.24.2
 scipy~=1.7.3
 tqdm~=4.62.3
-pyarrow~=6.0.1
+pyarrow~=6.0.1