Updated mc with isotonic.

nohup.out

@@ -99,3 +99,47 @@ error: externally-managed-environment
 
 note: If you believe this is a mistake, please contact your Python installation or OS distribution provider. You can override this, at the risk of breaking your Python installation or OS, by passing --break-system-packages.
 hint: See PEP 668 for the detailed specification.
+2024-03-27 15:11:04.526493: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
+2024-03-27 15:11:04.526578: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
+2024-03-27 15:11:04.528324: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
+2024-03-27 15:11:04.536839: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
+To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.
+2024-03-27 15:11:05.847612: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
+[nltk_data] Downloading package punkt to /root/nltk_data...
+[nltk_data]   Package punkt is already up-to-date!
+[nltk_data] Downloading package punkt to /root/nltk_data...
+[nltk_data]   Package punkt is already up-to-date!
+[nltk_data] Downloading package stopwords to /root/nltk_data...
+[nltk_data]   Package stopwords is already up-to-date!
+[nltk_data] Downloading package punkt to /root/nltk_data...
+[nltk_data]   Package punkt is already up-to-date!
+[nltk_data] Downloading package punkt to /root/nltk_data...
+[nltk_data]   Package punkt is already up-to-date!
+[nltk_data] Downloading package stopwords to /root/nltk_data...
+[nltk_data]   Package stopwords is already up-to-date!
+error: externally-managed-environment
+
+× This environment is externally managed
+╰─> To install Python packages system-wide, try apt install
+    python3-xyz, where xyz is the package you are trying to
+    install.
+    
+    If you wish to install a non-Debian-packaged Python package,
+    create a virtual environment using python3 -m venv path/to/venv.
+    Then use path/to/venv/bin/python and path/to/venv/bin/pip. Make
+    sure you have python3-full installed.
+    
+    If you wish to install a non-Debian packaged Python application,
+    it may be easiest to use pipx install xyz, which will manage a
+    virtual environment for you. Make sure you have pipx installed.
+    
+    See /usr/share/doc/python3.11/README.venv for more information.
+
+note: If you believe this is a mistake, please contact your Python installation or OS distribution provider. You can override this, at the risk of breaking your Python installation or OS, by passing --break-system-packages.
+hint: See PEP 668 for the detailed specification.
+/home/aliasgarov/copyright_checker/predictors.py:197: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.
+  probas = F.softmax(tensor_logits).detach().cpu().numpy()
+/home/aliasgarov/copyright_checker/predictors.py:197: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.
+  probas = F.softmax(tensor_logits).detach().cpu().numpy()
+/home/aliasgarov/copyright_checker/predictors.py:197: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.
+  probas = F.softmax(tensor_logits).detach().cpu().numpy()

predictors.py

@@ -276,11 +276,11 @@ def predict_bc_scores(input):
     average_bc_scores = np.mean(bc_scores_array, axis=0)
     bc_score_list = average_bc_scores.tolist()
     print(f"Original BC scores: AI: {bc_score_list[1]}, HUMAN: {bc_score_list[0]}")
-    # # isotonic regression calibration
-    # ai_score = iso_reg.predict([bc_score_list[1]])[0]
-    # human_score = 1 - ai_score
-    bc_score = {"AI": bc_score_list[1], "HUMAN": bc_score_list[0]}
-    # print(f"Calibration BC scores: AI: {ai_score}, HUMAN: {human_score}")
+    # isotonic regression calibration
+    ai_score = iso_reg.predict([bc_score_list[1]])[0]
+    human_score = 1 - ai_score
+    bc_score = {"AI": ai_score, "HUMAN": human_score}
+    print(f"Calibration BC scores: AI: {ai_score}, HUMAN: {human_score}")
     return bc_score
 
 
@@ -330,8 +330,13 @@ def predict_1on1_scores(input, models):
     bc_scores_array = np.array(bc_scores)
     average_bc_scores = np.mean(bc_scores_array, axis=0)
     bc_score_list = average_bc_scores.tolist()
-    bc_score = {"AI": bc_score_list[1], "HUMAN": bc_score_list[0]}
-
+    print(f"Original BC scores: AI: {bc_score_list[1]}, HUMAN: {bc_score_list[0]}")
+    # isotonic regression calibration
+    ai_score = iso_reg.predict([bc_score_list[1]])[0]
+    human_score = 1 - ai_score
+    bc_score = {"AI": ai_score, "HUMAN": human_score}
+    print(f"Calibration BC scores: AI: {ai_score}, HUMAN: {human_score}")
+    
     # MC SCORE
     if len(models) > 1:
         print("Starting MC")