File size: 2,859 Bytes
0255e9b |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 |
import torch
import orjson
from transformers import pipeline
from transformers import BertTokenizerFast, AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained("dbmdz/bert-base-turkish-cased")
with open("labels.json", "r") as f:
id_to_label = {int(k): v for k, v in orjson.loads(f.read()).items()}
nlp = pipeline(
"ner",
model="./model",
tokenizer=tokenizer,
device=0 if torch.cuda.is_available() else -1,
)
def get_entities(tokens):
entities = []
entity = None
for token in tokens:
label_id = int(token["entity"][6:])
label = id_to_label[label_id]
if label.startswith("B-"):
if entity:
entity["score"] /= entity["token_count"]
entities.append(entity)
entity = {
"label": label[2:],
"ranges": [token["start"], token["end"]],
"score": token["score"],
"token_count": 1,
}
elif label.startswith("I-"):
if entity and entity["label"] == label[2:]:
entity["ranges"][1] = token["end"]
entity["token_count"] += 1
entity["score"] += token["score"]
else:
if entity:
entity["ranges"][1] = token["end"]
entity["token_count"] += 1
entity["score"] += token["score"]
entity["score"] /= entity["token_count"]
entities.append(entity)
entity = None
else:
if entity:
entity["score"] /= entity["token_count"]
entities.append(entity)
entity = None
if entity:
entity["score"] /= entity["token_count"]
entities.append(entity)
return entities
def process(text):
nlp_output = nlp(text)
entities = get_entities(nlp_output)
for entity in entities:
print(f"{text[entity['ranges'][0]:entity['ranges'][1]]:<35} {entity['label']:>15} {entity['score'] * 100:.2f}%")
print("Average Score: ", sum([token["score"] for token in nlp_output]) / len(nlp_output))
print("Labels Found: ", len(entities))
print("-" * 70)
if __name__ == "__main__":
examples = [
"Osmangazi Mahallesi, Hoca Ahmet Yesevi Cd. No:34, 16050 Osmangazi/Bursa",
"Karşıyaka Mahallesi, Mavişehir Caddesi No: 91, Daire 4, 35540 Karşıyaka/İzmir",
"Selçuklu Mahallesi, Atatürk Bulvarı No: 55, 42050 Selçuklu/Konya",
"Alsancak Mahallesi, 1475. Sk. No:3, 35220 Konak/İzmir",
"Kocatepe Mahallesi, Yaşam Caddesi 3. Sokak No:4, 06420 Bayrampaşa/İstanbul",
]
for example in examples:
print(example)
process(example)
while True:
text = input("Enter text: ")
if not text:
break
process(text) |