add custom handler

handler.py

@@ -0,0 +1,32 @@
+from typing import Dict, List, Any
+from transformers import AutoModelForMultipleChoice, AutoTokenizer
+import torch
+
+
+
+class EndpointHandler:
+    def __init__(self, path=""):
+        # load model and processor from path
+        self.model =  AutoModelForMultipleChoice.from_pretrained(path)
+        self.tokenizer = AutoTokenizer.from_pretrained(path)
+
+    def __call__(self, data: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Args:
+            data (:dict:):
+                The payload with the text prompt.
+        """
+        # process input
+        input = data.pop("input", data)
+        gen_outputs_no_input_decoded = data.pop("gen_outputs_no_input_decoded", None)
+
+        rank_inputs = self.tokenizer([[input, gen_output_no_input_decoded] for gen_output_no_input_decoded in gen_outputs_no_input_decoded],
+                                        return_tensors="pt",
+                                        padding=True)
+
+        rank_labels = torch.tensor(0).unsqueeze(0)
+
+        rank_outputs = self.model(**{k: v.unsqueeze(0) for k, v in rank_inputs.items()}, labels=rank_labels)
+
+        rank_predictions = torch.nn.functional.softmax(rank_outputs.logits, dim=-1)[0].tolist()
+        return {"rank_predictions": rank_predictions}

requirements.txt

@@ -0,0 +1,3 @@
+accelerate
+bitsandbytes
+transformers