update model.py to work
Browse files
model.py
CHANGED
@@ -16,8 +16,7 @@ def load_embedder_and_tokenizer(name: str) -> Tuple[
|
|
16 |
transformers.PreTrainedTokenizer
|
17 |
]:
|
18 |
if name.startswith("nomic") or (name == "bert-base-uncased"):
|
19 |
-
|
20 |
-
model = transformers.AutoModel.from_pretrained(name, config=config, trust_remote_code=True)
|
21 |
tokenizer = transformers.AutoTokenizer.from_pretrained(name)
|
22 |
elif name in ["gtr-base", "gtr_base"]:
|
23 |
model = transformers.AutoModel.from_pretrained(
|
|
|
16 |
transformers.PreTrainedTokenizer
|
17 |
]:
|
18 |
if name.startswith("nomic") or (name == "bert-base-uncased"):
|
19 |
+
model = transformers.AutoModelForMaskedLM.from_pretrained(name, trust_remote_code=True).bert
|
|
|
20 |
tokenizer = transformers.AutoTokenizer.from_pretrained(name)
|
21 |
elif name in ["gtr-base", "gtr_base"]:
|
22 |
model = transformers.AutoModel.from_pretrained(
|