added confidence

generic_nel.py

@@ -120,9 +120,6 @@ class NelPipeline(Pipeline):
         wikipedia_predictions = self.tokenizer.batch_decode(
             outputs.sequences, skip_special_tokens=True
         )
-        # Initialize confidence list for each token
-        token_confidences = []
-
         # Process the scores for each token
         all_probabilities = []
         import torch.nn.functional as F
@@ -139,6 +136,7 @@ class NelPipeline(Pipeline):
 
             # Store the probabilities
             all_probabilities.append(top_probabilities)
+            print(f"Top probabilities: {top_probabilities}")
 
         # Convert probabilities into percentages if needed
         percentages = [(prob * 100.0).tolist() for prob in all_probabilities]