Update handler.py

handler.py

@@ -50,10 +50,12 @@ class EndpointHandler():
 
         with torch.no_grad():
             out = self.model.generate(**processed, 
-                                      max_new_tokens=512,
-                                      temperature = 0.1,
-                                      do_sample=True,
-                                      repetition_penalty=1.2  
+                                     max_new_tokens=50,
+                                     temperature = 0.5,
+                                     do_sample=True,
+                                     top_k=50,
+                                     top_p=0.9,
+                                     repetition_penalty=1.2  
                                      ).to(self.device)
         
         result = {}