hansihe commited on
Commit
22e45de
1 Parent(s): b303079

create handler.py

Browse files

This should make this model directly compatible with HuggingFace inference endpoints.

Files changed (1) hide show
  1. handler.py +29 -0
handler.py ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Dict, List, Any
2
+ from transformers import pipeline
3
+
4
+ import torch.nn.functional as F
5
+ from torch import Tensor
6
+ from transformers import AutoTokenizer, AutoModel
7
+
8
+ def average_pool(last_hidden_states: Tensor,
9
+ attention_mask: Tensor) -> Tensor:
10
+ last_hidden = last_hidden_states.masked_fill(~attention_mask[..., None].bool(), 0.0)
11
+ return last_hidden.sum(dim=1) / attention_mask.sum(dim=1)[..., None]
12
+
13
+ class EndpointHandler():
14
+ def __init__(self, path=""):
15
+ self.pipeline = pipeline("feature-extraction", model=path)
16
+ self.tokenizer = AutoTokenizer.from_pretrained(path)
17
+ self.model = AutoModel.from_pretrained(path)
18
+
19
+ def __call__(self, data: Dict[str, Any]) -> List[List[int]]:
20
+ inputs = data.pop("inputs",data)
21
+
22
+ batch_dict = self.tokenizer(inputs, max_length=512, padding=True, truncation=True, return_tensors='pt')
23
+
24
+ outputs = self.model(**batch_dict)
25
+
26
+ embeddings = average_pool(outputs.last_hidden_state, batch_dict['attention_mask'])
27
+ embeddings = F.normalize(embeddings, p=2, dim=1).tolist()
28
+
29
+ return embeddings