Spaces:
Running
Running
pragnakalp
commited on
Commit
•
84a67cd
1
Parent(s):
6520201
Update bert_ner_model_loader.py
Browse files- bert_ner_model_loader.py +2 -0
bert_ner_model_loader.py
CHANGED
@@ -11,6 +11,8 @@ from torch.nn import CrossEntropyLoss
|
|
11 |
from torch.utils.data import DataLoader, RandomSampler, SequentialSampler, TensorDataset
|
12 |
from torch.utils.data.distributed import DistributedSampler
|
13 |
from tqdm import tqdm, trange
|
|
|
|
|
14 |
from nltk import word_tokenize
|
15 |
# from transformers import (BertConfig, BertForTokenClassification,
|
16 |
# BertTokenizer)
|
|
|
11 |
from torch.utils.data import DataLoader, RandomSampler, SequentialSampler, TensorDataset
|
12 |
from torch.utils.data.distributed import DistributedSampler
|
13 |
from tqdm import tqdm, trange
|
14 |
+
import nltk
|
15 |
+
nltk.download('punkt')
|
16 |
from nltk import word_tokenize
|
17 |
# from transformers import (BertConfig, BertForTokenClassification,
|
18 |
# BertTokenizer)
|