Spaces:
Sleeping
Sleeping
File size: 1,897 Bytes
74a60bc |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 |
# coding=utf-8
# author: xusong
# time: 2022/8/23 16:06
from perplexity import PerplexityPipeline
from transformers import BertTokenizer, BertForMaskedLM
import gradio as gr
import time
en_tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
en_model = BertForMaskedLM.from_pretrained("bert-base-uncased")
en_pipeline = PerplexityPipeline(model=en_model, tokenizer=en_tokenizer)
zh_tokenizer = BertTokenizer.from_pretrained('bert-base-chinese')
zh_model = BertForMaskedLM.from_pretrained("bert-base-chinese")
zh_pipeline = PerplexityPipeline(model=zh_model, tokenizer=zh_tokenizer)
def ppl(model_version, text):
print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), model_version, text)
if model_version == "bert-base-uncased":
result = en_pipeline(text)
else:
result = zh_pipeline(text)
return result["ppl"], result
examples = [
["bert-base-uncased", "New York City is located in the northeastern United States."],
["bert-base-uncased", "New York City is located in the western United States."],
["bert-base-chinese", "少先队员因该为老人让坐"],
]
css = "#json-container {height:: 400px; overflow: auto !important}"
corr_iface = gr.Interface(
fn=ppl,
inputs=[
# gr.Dropdown(["bert-base-uncased", "bert-base-chinese"], value="bert-base-uncased"), # TODO 调整大小和位置
gr.Radio(
["bert-base-uncased", "bert-base-chinese"],
value="bert-base-uncased"
),
gr.Textbox(
value="New York City is located in the northeastern United States.",
label="input text"
)],
outputs=[
gr.Textbox(label="Perplexity"),
gr.JSON(label="Tokens", elem_id="json-container")],
examples=examples,
title="BERT as Language Model",
description='',
css=css
)
if __name__ == "__main__":
corr_iface.launch()
|