Upload bge_custom_impl.py with huggingface_hub
Browse files- bge_custom_impl.py +4 -2
bge_custom_impl.py
CHANGED
@@ -1,3 +1,4 @@
|
|
|
|
1 |
import json
|
2 |
import os
|
3 |
from typing import Any
|
@@ -7,7 +8,7 @@ from transformers import AutoModel, AutoTokenizer
|
|
7 |
|
8 |
class ConcatCustomPooling(nn.Module):
|
9 |
def __init__(self, model_name_or_path="BAAI/bge-large-en-v1.5",layers=None,max_seq_len=512, **kwargs):
|
10 |
-
super().__init__(
|
11 |
self.layers = layers
|
12 |
self.base_name = model_name_or_path
|
13 |
self.tokenizer = AutoTokenizer.from_pretrained(model_name_or_path)
|
@@ -47,7 +48,8 @@ class ConcatCustomPooling(nn.Module):
|
|
47 |
with open(os.path.join(save_dir, "config.json"), "w") as fOut:
|
48 |
json.dump(self.get_config_dict(), fOut, indent=4)
|
49 |
|
50 |
-
|
|
|
51 |
with open(os.path.join(load_dir, "config.json")) as fIn:
|
52 |
config = json.load(fIn)
|
53 |
|
|
|
1 |
+
from calendar import c
|
2 |
import json
|
3 |
import os
|
4 |
from typing import Any
|
|
|
8 |
|
9 |
class ConcatCustomPooling(nn.Module):
|
10 |
def __init__(self, model_name_or_path="BAAI/bge-large-en-v1.5",layers=None,max_seq_len=512, **kwargs):
|
11 |
+
super().__init__()
|
12 |
self.layers = layers
|
13 |
self.base_name = model_name_or_path
|
14 |
self.tokenizer = AutoTokenizer.from_pretrained(model_name_or_path)
|
|
|
48 |
with open(os.path.join(save_dir, "config.json"), "w") as fOut:
|
49 |
json.dump(self.get_config_dict(), fOut, indent=4)
|
50 |
|
51 |
+
@classmethod
|
52 |
+
def load(cls,load_dir: str, **kwargs) -> "ConcatCustomPooling":
|
53 |
with open(os.path.join(load_dir, "config.json")) as fIn:
|
54 |
config = json.load(fIn)
|
55 |
|