KoichiYasuoka commited on
Commit
6fd5751
1 Parent(s): ed37fa0

universal-dependencies pipeline

Browse files
Files changed (3) hide show
  1. README.md +8 -0
  2. config.json +5 -0
  3. ud.py +35 -0
README.md CHANGED
@@ -56,4 +56,12 @@ nlp=UDgoeswith("KoichiYasuoka/roberta-base-english-ud-goeswith")
56
  print(nlp("I saw a horse yesterday which had no name"))
57
  ```
58
 
 
 
 
 
 
 
 
 
59
  [ufal.chu-liu-edmonds](https://pypi.org/project/ufal.chu-liu-edmonds/) is required.
 
56
  print(nlp("I saw a horse yesterday which had no name"))
57
  ```
58
 
59
+ or
60
+
61
+ ```
62
+ from transformers import pipeline
63
+ nlp=pipeline("universal-dependencies","KoichiYasuoka/roberta-base-english-ud-goeswith",trust_remote_code=True)
64
+ print(nlp("I saw a horse yesterday which had no name"))
65
+ ```
66
+
67
  [ufal.chu-liu-edmonds](https://pypi.org/project/ufal.chu-liu-edmonds/) is required.
config.json CHANGED
@@ -5,6 +5,11 @@
5
  "attention_probs_dropout_prob": 0.1,
6
  "bos_token_id": 0,
7
  "classifier_dropout": null,
 
 
 
 
 
8
  "eos_token_id": 2,
9
  "hidden_act": "gelu",
10
  "hidden_dropout_prob": 0.1,
 
5
  "attention_probs_dropout_prob": 0.1,
6
  "bos_token_id": 0,
7
  "classifier_dropout": null,
8
+ "custom_pipelines": {
9
+ "universal-dependencies": {
10
+ "impl": "ud.UniversalDependenciesPipeline"
11
+ }
12
+ },
13
  "eos_token_id": 2,
14
  "hidden_act": "gelu",
15
  "hidden_dropout_prob": 0.1,
ud.py ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from transformers import TokenClassificationPipeline
2
+ from transformers.pipelines import PIPELINE_REGISTRY
3
+
4
+ class UniversalDependenciesPipeline(TokenClassificationPipeline):
5
+ def _forward(self,model_input):
6
+ import torch
7
+ v=model_input["input_ids"][0].tolist()
8
+ with torch.no_grad():
9
+ e=self.model(input_ids=torch.tensor([v[0:i]+[self.tokenizer.mask_token_id]+v[i+1:]+[j] for i,j in enumerate(v[1:-1],1)]))
10
+ return {"logits":e.logits[:,1:-2,:],**model_input}
11
+ def postprocess(self,model_output,**kwargs):
12
+ import numpy
13
+ import ufal.chu_liu_edmonds
14
+ e=model_output["logits"].numpy()
15
+ r=[1 if i==0 else -1 if j.endswith("|root") else 0 for i,j in sorted(self.model.config.id2label.items())]
16
+ e+=numpy.where(numpy.add.outer(numpy.identity(e.shape[0]),r)==0,0,numpy.nan)
17
+ m=numpy.full((e.shape[0]+1,e.shape[1]+1),numpy.nan)
18
+ m[1:,1:]=numpy.nanmax(e,axis=2).transpose()
19
+ p=numpy.zeros(m.shape)
20
+ p[1:,1:]=numpy.nanargmax(e,axis=2).transpose()
21
+ for i in range(1,m.shape[0]):
22
+ m[i,0],m[i,i],p[i,0]=m[i,i],numpy.nan,p[i,i]
23
+ h=ufal.chu_liu_edmonds.chu_liu_edmonds(m)[0]
24
+ if [0 for i in h if i==0]!=[0]:
25
+ m[:,0]+=numpy.where(m[:,0]<numpy.nanmax(m[:,0]),numpy.nan,0)
26
+ h=ufal.chu_liu_edmonds.chu_liu_edmonds(m)[0]
27
+ t=model_output["sentence"]
28
+ u="# text = "+t+"\n"
29
+ v=[(s,e) for s,e in model_output["offset_mapping"][0].tolist() if s<e]
30
+ for i,(s,e) in enumerate(v,1):
31
+ q=self.model.config.id2label[p[i,h[i]]].split("|")
32
+ u+="\t".join([str(i),t[s:e],"_",q[0],"_","|".join(q[1:-1]),str(h[i]),q[-1],"_","_" if i<len(v) and e<v[i][0] else "SpaceAfter=No"])+"\n"
33
+ return u+"\n"
34
+
35
+ PIPELINE_REGISTRY.register_pipeline("universal-dependencies",pipeline_class=UniversalDependenciesPipeline)