Duplicate from DD0101/Disfluency-base

.gitattributes

@@ -0,0 +1,37 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+jdk-8u361-linux-aarch64.rpm filter=lfs diff=lfs merge=lfs -text
+VnCoreNLP-1.2.jar filter=lfs diff=lfs merge=lfs -text
+models/postagger/vi-tagger filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,13 @@
+---
+title: Disfluency Base
+emoji: 😻
+colorFrom: gray
+colorTo: pink
+sdk: gradio
+sdk_version: 3.23.0
+app_file: app.py
+pinned: false
+duplicated_from: DD0101/Disfluency-base
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

VnCoreNLP-1.2.jar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e2811cdbc2ddfc71d04be5dc36e185c88dcd1ad4d5d69e4ff2e1369dccf7793
+size 27412703

VnCoreNLP-master.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d8e31f5b5293c18691ea0d23eb64e504838222d71f020a84496fa8b5c64ec7f
+size 168648997

app.py

@@ -0,0 +1,88 @@
+import os
+
+import transformers
+from transformers import pipeline
+from transformers.pipelines.token_classification import TokenClassificationPipeline
+import py_vncorenlp
+
+os.system('pwd')
+os.system('sudo update-alternatives --config java')
+os.mkdir('/home/user/app/vncorenlp')
+py_vncorenlp.download_model(save_dir='/home/user/app/vncorenlp')
+rdrsegmenter = py_vncorenlp.VnCoreNLP(annotators=["wseg"], save_dir='/home/user/app/vncorenlp')
+
+class MyPipeline(TokenClassificationPipeline):
+  def preprocess(self, sentence, offset_mapping=None):
+      truncation = True if self.tokenizer.model_max_length and self.tokenizer.model_max_length > 0 else False 
+      
+      model_inputs = self.tokenizer(
+          sentence,
+          return_tensors=self.framework,
+          truncation=truncation,
+          return_special_tokens_mask=True,
+          return_offsets_mapping=self.tokenizer.is_fast,
+      )
+
+
+      length = len(model_inputs['input_ids'][0]) - 2
+      tokens = self.tokenizer.tokenize(sentence)
+      seek = 0
+      offset_mapping_list = [[(0, 0)]]
+      for i in range(length):
+        if tokens[i][-2:] == '@@':
+          offset_mapping_list[0].append((seek, seek + len(tokens[i]) - 2))
+          seek += len(tokens[i]) - 2            
+        else:
+          offset_mapping_list[0].append((seek, seek + len(tokens[i])))
+          seek += len(tokens[i]) + 1
+      offset_mapping_list[0].append((0, 0))
+
+      # if offset_mapping:
+      #     model_inputs["offset_mapping"] = offset_mapping
+
+      model_inputs['offset_mapping'] = offset_mapping_list
+      model_inputs["sentence"] = sentence
+
+      return model_inputs
+
+model_checkpoint = "DD0101/disfluency-base"
+
+my_classifier = pipeline(
+  "token-classification", model=model_checkpoint, aggregation_strategy="simple", pipeline_class=MyPipeline)
+
+
+import gradio as gr
+
+def ner(text):
+  text = " ".join(rdrsegmenter.word_segment(text))
+    
+  output = my_classifier(text)
+  for entity in output:
+    entity['entity'] = entity.pop('entity_group')
+      
+  return {'text': text, 'entities': output}, text
+
+examples = ['Tôi cần thuê à tôi muốn bay một chuyến khứ hồi từ Đà Nẵng đến Đà Lạt', 
+            'Giá vé một chiều à không khứ hồi từ Đà Nẵng đến Vinh dưới 2 triệu đồng giá vé khứ hồi từ Quy Nhơn đến Vinh dưới 3 triệu đồng giá vé khứ hồi từ Buôn Ma Thuột đến Quy Nhơn à đến Vinh dưới 4 triệu rưỡi', 
+            'Cho tôi biết các chuyến bay đến Đà Nẵng vào ngày 12 mà không ngày 14 tháng sáu',
+            'Những chuyến bay nào khởi hành từ Thành phố Hồ Chí Minh bay đến Frankfurt mà nối chuyến ở Singapore và hạ cánh trước 10 giờ ý tôi là 9 giờ tối'
+] 
+
+demo = gr.Interface(ner, 
+                    gr.Textbox(label='Text', placeholder="Enter sentence here..."),
+                    outputs=[gr.HighlightedText(label='Highlighted Output'), gr.Textbox(label='Word-Segmentation Preprocessing')],
+                    examples=examples,
+                    title="Disfluency Detection",
+                    description="This is an easy-to-use built in Gradio for desmontrating a NER System that identifies disfluency-entities in \
+                    Vietnamese utterances",
+                    theme=gr.themes.Soft())
+
+demo.launch() 
+
+
+
+
+
+
+
+

jdk-8u361-linux-aarch64.rpm

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:316f980932185e2cf38797783bc38f9cf095cb125d11cf37bda91e30aed4ecce
+size 73402781

models/dep/vi-dep.xz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:266e4a3a55d5edd1607d5f036c2f95b70c0a6c80f58b57fd9962677a6ef331b7
+size 16048864

models/ner/vi-500brownclusters.xz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d30f9cfdf0af193a69e185d1acda0306a9fbe1321f8a700f7c66557a90f92b8c
+size 5599844

models/ner/vi-ner.xz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f04c5e752d7f99a6313b758fc2607a2c3906e58b1d60a37eb0192aead73d61f7
+size 9956876

models/ner/vi-pretrainedembeddings.xz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:00d3d034f1b23a8bfe5168195741fde845808c212e6dfcd4c94bead1665eb0fc
+size 57313672

models/postagger/vi-tagger

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a95608a5982db89c11353b451154ec396eccc0ff1f5b22874935ecdf4e0ace01
+size 29709468

packages.txt

@@ -0,0 +1 @@
+default-jdk

requirements.txt

@@ -0,0 +1,3 @@
+transformers
+torch
+py_vncorenlp