Update model files and tokenizer configuration

README.md

@@ -30,55 +30,55 @@ The model is based on [dbmdz/bert-base-turkish-cased](https://huggingface.co/dbm
 ```
 (g:\projects\address-extraction\venv) G:\projects\address-extraction>python predict.py
 Osmangazi Mahallesi, Hoca Ahmet Yesevi Cd. No:34, 16050 Osmangazi/Bursa
-Osmangazi                                   Mahalle 98.65%
-Hoca Ahmet Yesevi                             Cadde 97.63%
-34                                    Bina Numarası 98.92%
-16050                                    Posta Kodu 97.83%
-Osmangazi                                      İlçe 98.97%
+Osmangazi                                   Mahalle 98.80%
+Hoca Ahmet Yesevi                             Cadde 98.55%
+34                                    Bina Numarası 99.50%
+16050                                    Posta Kodu 98.49%
+Osmangazi                                      İlçe 98.71%
 Bursa                                            İl 99.21%
-Average Score:  0.9902257982053255
+Average Score:  0.9874102413654328
 Labels Found:  6
 ----------------------------------------------------------------------
 Karşıyaka Mahallesi, Mavişehir Caddesi No: 91, Daire 4, 35540 Karşıyaka/İzmir
-Karşıyaka                                   Mahalle 99.11%
-Mavişehir                                     Cadde 97.16%
-91                                    Bina Numarası 98.73%
-4                                               Kat 29.06%
-35540                                    Posta Kodu 98.65%
-Karşıyaka                                      İlçe 99.17%
-İzmir                                            İl 99.16%
-Average Score:  0.9237866433043229
+Karşıyaka                                   Mahalle 98.93%
+Mavişehir                                     Cadde 96.90%
+91                                    Bina Numarası 99.25%
+4                                     Bina Numarası 30.75%
+35540                                    Posta Kodu 98.97%
+Karşıyaka                                      İlçe 98.84%
+İzmir                                            İl 98.86%
+Average Score:  0.9173339426517486
 Labels Found:  7
 ----------------------------------------------------------------------
 Selçuklu Mahallesi, Atatürk Bulvarı No: 55, 42050 Selçuklu/Konya
-Selçuklu                                    Mahalle 98.67%
-Atatürk                                       Cadde 57.06%
-55                                    Bina Numarası 98.94%
-42050                                    Posta Kodu 98.81%
-Selçuklu                                       İlçe 99.06%
-Konya                                            İl 99.22%
-Average Score:  0.9659512996673584
+Selçuklu                                    Mahalle 98.53%
+Atatürk                                       Cadde 47.01%
+55                                    Bina Numarası 99.49%
+42050                                    Posta Kodu 98.78%
+Selçuklu                                       İlçe 98.74%
+Konya                                            İl 99.16%
+Average Score:  0.9240859523415565
 Labels Found:  6
 ----------------------------------------------------------------------
 Alsancak Mahallesi, 1475. Sk. No:3, 35220 Konak/İzmir
-Alsancak                                    Mahalle 99.38%
-1475                                          Sokak 96.04%
-3                                     Bina Numarası 98.06%
-35220                                    Posta Kodu 98.75%
-Konak                                          İlçe 99.23%
-İzmir                                            İl 99.16%
-Average Score:  0.9909308176291617
+Alsancak                                    Mahalle 99.35%
+1475                                          Sokak 97.71%
+3                                     Bina Numarası 99.18%
+35220                                    Posta Kodu 99.00%
+Konak                                          İlçe 98.90%
+İzmir                                            İl 98.95%
+Average Score:  0.9881603717803955
 Labels Found:  6
 ----------------------------------------------------------------------
 Kocatepe Mahallesi, Yaşam Caddesi 3. Sokak No:4, 06420 Bayrampaşa/İstanbul
-Kocatepe                                    Mahalle 99.46%
-Yaşam                                         Cadde 94.07%
-3                                             Sokak 84.07%
-4                                     Bina Numarası 98.42%
-06420                                    Posta Kodu 98.54%
-Bayrampaşa                                     İlçe 98.97%
-İstanbul                                         İl 98.98%
-Average Score:  0.9832726591511777
+Kocatepe                                    Mahalle 99.44%
+Yaşam                                         Cadde 92.45%
+3                                             Sokak 70.61%
+4                                     Bina Numarası 99.18%
+06420                                    Posta Kodu 99.00%
+Bayrampaşa                                     İlçe 98.86%
+İstanbul                                         İl 98.90%
+Average Score:  0.9558616995811462
 Labels Found:  7
 ----------------------------------------------------------------------
 ```

config.json

@@ -9,46 +9,76 @@
   "hidden_dropout_prob": 0.1,
   "hidden_size": 768,
   "id2label": {
-    "0": "[PAD]",
-    "1": "\u00dclke",
-    "2": "\u0130l",
-    "3": "\u0130l\u00e7e",
-    "4": "Mahalle",
-    "5": "Cadde",
-    "6": "Sokak",
-    "7": "Bina Ad\u0131",
-    "8": "Bina Numaras\u0131",
-    "9": "Yer Ad\u0131",
-    "10": "Site",
-    "11": "Adres Detay",
-    "12": "Blok No",
-    "13": "Bulvar",
-    "14": "Daire No",
-    "15": "Posta Kodu",
-    "16": "Kat",
-    "17": "[UNK]"
+    "0": "O",
+    "1": "B-\u00dclke",
+    "2": "I-\u00dclke",
+    "3": "B-\u0130l",
+    "4": "I-\u0130l",
+    "5": "B-\u0130l\u00e7e",
+    "6": "I-\u0130l\u00e7e",
+    "7": "B-Mahalle",
+    "8": "I-Mahalle",
+    "9": "B-Cadde",
+    "10": "I-Cadde",
+    "11": "B-Sokak",
+    "12": "I-Sokak",
+    "13": "B-Bina Ad\u0131",
+    "14": "I-Bina Ad\u0131",
+    "15": "B-Bina Numaras\u0131",
+    "16": "I-Bina Numaras\u0131",
+    "17": "B-Yer Ad\u0131",
+    "18": "I-Yer Ad\u0131",
+    "19": "B-Site",
+    "20": "I-Site",
+    "21": "B-Adres Detay",
+    "22": "I-Adres Detay",
+    "23": "B-Blok No",
+    "24": "I-Blok No",
+    "25": "B-Bulvar",
+    "26": "I-Bulvar",
+    "27": "B-Daire No",
+    "28": "I-Daire No",
+    "29": "B-Posta Kodu",
+    "30": "I-Posta Kodu",
+    "31": "B-Kat",
+    "32": "I-Kat"
   },
   "initializer_range": 0.02,
   "intermediate_size": 3072,
   "label2id": {
-    "Adres Detay": 11,
-    "Bina Ad\u0131": 7,
-    "Bina Numaras\u0131": 8,
-    "Blok No": 12,
-    "Bulvar": 13,
-    "Cadde": 5,
-    "Daire No": 14,
-    "Kat": 16,
-    "Mahalle": 4,
-    "Posta Kodu": 15,
-    "Site": 10,
-    "Sokak": 6,
-    "Yer Ad\u0131": 9,
-    "[PAD]": 0,
-    "[UNK]": 17,
-    "\u00dclke": 1,
-    "\u0130l": 2,
-    "\u0130l\u00e7e": 3
+    "B-Adres Detay": 21,
+    "B-Bina Ad\u0131": 13,
+    "B-Bina Numaras\u0131": 15,
+    "B-Blok No": 23,
+    "B-Bulvar": 25,
+    "B-Cadde": 9,
+    "B-Daire No": 27,
+    "B-Kat": 31,
+    "B-Mahalle": 7,
+    "B-Posta Kodu": 29,
+    "B-Site": 19,
+    "B-Sokak": 11,
+    "B-Yer Ad\u0131": 17,
+    "B-\u00dclke": 1,
+    "B-\u0130l": 3,
+    "B-\u0130l\u00e7e": 5,
+    "I-Adres Detay": 22,
+    "I-Bina Ad\u0131": 14,
+    "I-Bina Numaras\u0131": 16,
+    "I-Blok No": 24,
+    "I-Bulvar": 26,
+    "I-Cadde": 10,
+    "I-Daire No": 28,
+    "I-Kat": 32,
+    "I-Mahalle": 8,
+    "I-Posta Kodu": 30,
+    "I-Site": 20,
+    "I-Sokak": 12,
+    "I-Yer Ad\u0131": 18,
+    "I-\u00dclke": 2,
+    "I-\u0130l": 4,
+    "I-\u0130l\u00e7e": 6,
+    "O": 0
   },
   "layer_norm_eps": 1e-12,
   "max_position_embeddings": 512,

model/config.json

@@ -9,46 +9,76 @@
   "hidden_dropout_prob": 0.1,
   "hidden_size": 768,
   "id2label": {
-    "0": "[PAD]",
-    "1": "\u00dclke",
-    "2": "\u0130l",
-    "3": "\u0130l\u00e7e",
-    "4": "Mahalle",
-    "5": "Cadde",
-    "6": "Sokak",
-    "7": "Bina Ad\u0131",
-    "8": "Bina Numaras\u0131",
-    "9": "Yer Ad\u0131",
-    "10": "Site",
-    "11": "Adres Detay",
-    "12": "Blok No",
-    "13": "Bulvar",
-    "14": "Daire No",
-    "15": "Posta Kodu",
-    "16": "Kat",
-    "17": "[UNK]"
+    "0": "O",
+    "1": "B-\u00dclke",
+    "2": "I-\u00dclke",
+    "3": "B-\u0130l",
+    "4": "I-\u0130l",
+    "5": "B-\u0130l\u00e7e",
+    "6": "I-\u0130l\u00e7e",
+    "7": "B-Mahalle",
+    "8": "I-Mahalle",
+    "9": "B-Cadde",
+    "10": "I-Cadde",
+    "11": "B-Sokak",
+    "12": "I-Sokak",
+    "13": "B-Bina Ad\u0131",
+    "14": "I-Bina Ad\u0131",
+    "15": "B-Bina Numaras\u0131",
+    "16": "I-Bina Numaras\u0131",
+    "17": "B-Yer Ad\u0131",
+    "18": "I-Yer Ad\u0131",
+    "19": "B-Site",
+    "20": "I-Site",
+    "21": "B-Adres Detay",
+    "22": "I-Adres Detay",
+    "23": "B-Blok No",
+    "24": "I-Blok No",
+    "25": "B-Bulvar",
+    "26": "I-Bulvar",
+    "27": "B-Daire No",
+    "28": "I-Daire No",
+    "29": "B-Posta Kodu",
+    "30": "I-Posta Kodu",
+    "31": "B-Kat",
+    "32": "I-Kat"
   },
   "initializer_range": 0.02,
   "intermediate_size": 3072,
   "label2id": {
-    "Adres Detay": 11,
-    "Bina Ad\u0131": 7,
-    "Bina Numaras\u0131": 8,
-    "Blok No": 12,
-    "Bulvar": 13,
-    "Cadde": 5,
-    "Daire No": 14,
-    "Kat": 16,
-    "Mahalle": 4,
-    "Posta Kodu": 15,
-    "Site": 10,
-    "Sokak": 6,
-    "Yer Ad\u0131": 9,
-    "[PAD]": 0,
-    "[UNK]": 17,
-    "\u00dclke": 1,
-    "\u0130l": 2,
-    "\u0130l\u00e7e": 3
+    "B-Adres Detay": 21,
+    "B-Bina Ad\u0131": 13,
+    "B-Bina Numaras\u0131": 15,
+    "B-Blok No": 23,
+    "B-Bulvar": 25,
+    "B-Cadde": 9,
+    "B-Daire No": 27,
+    "B-Kat": 31,
+    "B-Mahalle": 7,
+    "B-Posta Kodu": 29,
+    "B-Site": 19,
+    "B-Sokak": 11,
+    "B-Yer Ad\u0131": 17,
+    "B-\u00dclke": 1,
+    "B-\u0130l": 3,
+    "B-\u0130l\u00e7e": 5,
+    "I-Adres Detay": 22,
+    "I-Bina Ad\u0131": 14,
+    "I-Bina Numaras\u0131": 16,
+    "I-Blok No": 24,
+    "I-Bulvar": 26,
+    "I-Cadde": 10,
+    "I-Daire No": 28,
+    "I-Kat": 32,
+    "I-Mahalle": 8,
+    "I-Posta Kodu": 30,
+    "I-Site": 20,
+    "I-Sokak": 12,
+    "I-Yer Ad\u0131": 18,
+    "I-\u00dclke": 2,
+    "I-\u0130l": 4,
+    "I-\u0130l\u00e7e": 6,
+    "O": 0
   },
   "layer_norm_eps": 1e-12,
   "max_position_embeddings": 512,

model/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3f61796d22b89ac6c4b5bf7cd5932198148f721b23b684a10950709b692328c6
-size 440185728
+oid sha256:2ff0f793d2c61260659c6a327c27dd0ea1d632bc0e5fc51da60d20d3caf3f7f3
+size 440231868

model/tokenizer.json

@@ -1,7 +1,21 @@
 {
   "version": "1.0",
-  "truncation": null,
-  "padding": null,
+  "truncation": {
+    "direction": "Right",
+    "max_length": 128,
+    "strategy": "LongestFirst",
+    "stride": 0
+  },
+  "padding": {
+    "strategy": {
+      "Fixed": 128
+    },
+    "direction": "Right",
+    "pad_to_multiple_of": null,
+    "pad_id": 0,
+    "pad_type_id": 0,
+    "pad_token": "[PAD]"
+  },
   "added_tokens": [
     {
       "id": 0,

model/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:300abae98dafa01f4daa08ba322e5f0ec434e9a6823866fb12dde9fb1397ba62
+oid sha256:551b4a0b8523f76d65879932b7a7ba98935984c8de39d14af0fd2659e2aadadc
 size 4664

predict.py

@@ -5,9 +5,6 @@ from transformers import BertTokenizerFast, AutoTokenizer
 
 tokenizer = AutoTokenizer.from_pretrained("dbmdz/bert-base-turkish-cased")
 
-with open("labels.json", "r") as f:
-    id_to_label = {int(k): v for k, v in orjson.loads(f.read()).items()}
-
 nlp = pipeline(
     "ner",
     model="./model",
@@ -19,20 +16,18 @@ def get_entities(tokens):
     entities = []
     entity = None
     for token in tokens:
-        label_id = int(token["entity"][6:])
-        label = id_to_label[label_id]
-        if label.startswith("B-"):
+        if token["entity"].startswith("B-"):
             if entity:
                 entity["score"] /= entity["token_count"]
                 entities.append(entity)
             entity = {
-                "label": label[2:],
+                "label": token["entity"][2:],
                 "ranges": [token["start"], token["end"]],
                 "score": token["score"],
                 "token_count": 1,
             }
-        elif label.startswith("I-"):
-            if entity and entity["label"] == label[2:]:
+        elif token["entity"].startswith("I-"):
+            if entity and entity["label"] == token["entity"][2:]:
                 entity["ranges"][1] = token["end"]
                 entity["token_count"] += 1
                 entity["score"] += token["score"]

train.py

@@ -1,4 +1,5 @@
 import io
+import shutil
 import requests
 import json
 import time
@@ -63,14 +64,11 @@ def load_data():
     return labels, [orjson.loads(line) for line in data.split("\n") if line]
 
 labels, data = load_data()
-# label_to_id = {}
-# for i, label in enumerate(labels):
-#     label_to_id["B-" + label["text"]] = i * 2 + 1
-#     label_to_id["I-" + label["text"]] = i * 2 + 2
-# label_to_id["O"] = 0
-label_to_id = {label["text"]: i + 1 for i, label in enumerate(labels)}
-label_to_id["[PAD]"] = 0
-label_to_id["[UNK]"] = len(label_to_id)
+label_to_id = {}
+for i, label in enumerate(labels):
+    label_to_id["B-" + label["text"]] = i * 2 + 1
+    label_to_id["I-" + label["text"]] = i * 2 + 2
+label_to_id["O"] = 0
 id_to_label = {v: k for k, v in label_to_id.items()}
 
 tokenizer = AutoTokenizer.from_pretrained("dbmdz/bert-base-turkish-cased")
@@ -97,18 +95,20 @@ def preprocess_data(item, tokenizer, label_to_id):
     attention_mask = inputs["attention_mask"]
     offset_mapping = inputs["offset_mapping"]
 
-    labels = ["[PAD]"] * 128
+    labels = ["O"] * 128
+    last_label = "O"
     for token_idx, [off_start, off_end] in enumerate(offset_mapping[0]):
         if off_start == off_end:
             continue
 
         for start, end, label in item['label']:
             if start <= off_start and off_end <= end:
-                labels[token_idx] = label
+                if last_label == label:
+                    labels[token_idx] = "I-" + label
+                else:
+                    labels[token_idx] = "B-" + label
+                last_label = label
                 break
-        
-        if labels[token_idx] == "[PAD]":
-            labels[token_idx] = "[UNK]"
     
     # Convert labels to ids
     labels = [label_to_id[label] for label in labels]
@@ -132,6 +132,7 @@ class AddressDataset(Dataset):
         return {key: torch.tensor(val) for key, val in item.items()}
 
 
+
 dataset = Dataset.from_generator(
     lambda: (preprocess_data(item, tokenizer, label_to_id) for item in data),
 )
@@ -166,8 +167,8 @@ def compute_metrics(pred, id_to_label):
     labels = [[id_to_label[label_id] for label_id in label_ids] for label_ids in labels]
     preds = [[id_to_label[pred] for pred in preds] for preds in preds]
 
-    labels = [set(label) for label in labels]
-    preds = [set(pred) for pred in preds]
+    labels = [label for label in labels if label != "O"]
+    preds = [pred for pred in preds if pred != "O"]
 
     mlb = MultiLabelBinarizer()
     mlb.fit([id_to_label.values()])
@@ -194,4 +195,6 @@ trainer = Trainer(
 trainer.train()
 trainer.evaluate()
 
-trainer.save_model("./model")
+trainer.save_model("./model")
+
+shutil.copy("./model/config.json", "./config.json")