Narsil HF staff commited on
Commit
ff90924
1 Parent(s): dede4f0

Returning the PR as per old code (still passing `api` instead of `token` directly

Browse files
Files changed (1) hide show
  1. convert.py +14 -7
convert.py CHANGED
@@ -7,9 +7,9 @@ import torch
7
 
8
  from huggingface_hub import CommitOperationAdd, HfApi, hf_hub_download
9
  from huggingface_hub.file_download import repo_folder_name
 
10
  from transformers import AutoConfig
11
  from transformers.pipelines.base import infer_framework_load_model
12
- from safetensors.torch import save_file
13
 
14
 
15
  def check_file_size(sf_filename, pt_filename):
@@ -17,10 +17,12 @@ def check_file_size(sf_filename, pt_filename):
17
  pt_size = os.stat(pt_filename).st_size
18
 
19
  if (sf_size - pt_size) / pt_size > 0.01:
20
- raise RuntimeError(f"""The file size different is more than 1%:
 
21
  - {sf_filename}: {sf_size}
22
  - {pt_filename}: {pt_size}
23
- """)
 
24
 
25
 
26
  def rename(pt_filename) -> str:
@@ -29,12 +31,13 @@ def rename(pt_filename) -> str:
29
  return local
30
 
31
 
32
- def convert_multi(model_id):
33
  filename = hf_hub_download(repo_id=model_id, filename="pytorch_model.bin.index.json")
34
  with open(filename, "r") as f:
35
  data = json.load(f)
36
 
37
  filenames = set(data["weight_map"].values())
 
38
  for filename in filenames:
39
  cached_filename = hf_hub_download(repo_id=model_id, filename=filename)
40
  loaded = torch.load(cached_filename)
@@ -53,7 +56,9 @@ def convert_multi(model_id):
53
  json.dump(newdata, f)
54
  local_filenames.append(index)
55
 
56
- operations = [CommitOperationAdd(path_in_repo=local.split("/")[-1], path_or_fileobj=local) for local in local_filenames]
 
 
57
 
58
  return operations
59
 
@@ -71,6 +76,7 @@ def convert_single(model_id, folder):
71
  operations = [CommitOperationAdd(path_in_repo=sf_filename, path_or_fileobj=local)]
72
  return operations
73
 
 
74
  def check_final_model(model_id, folder):
75
  config = hf_hub_download(repo_id=model_id, filename="config.json")
76
  shutil.copy(config, os.path.join(folder, "config.json"))
@@ -91,6 +97,7 @@ def convert(api, model_id):
91
 
92
  folder = repo_folder_name(repo_id=model_id, repo_type="models")
93
  os.makedirs(folder)
 
94
  try:
95
  operations = None
96
  if "model.safetensors" in filenames or "model_index.safetensors.index.json" in filenames:
@@ -104,7 +111,7 @@ def convert(api, model_id):
104
 
105
  if operations:
106
  check_final_model(model_id, folder)
107
- api.create_commit(
108
  repo_id=model_id,
109
  operations=operations,
110
  commit_message="Adding `safetensors` variant of this model",
@@ -112,7 +119,7 @@ def convert(api, model_id):
112
  )
113
  finally:
114
  shutil.rmtree(folder)
115
- return 1
116
 
117
 
118
  if __name__ == "__main__":
 
7
 
8
  from huggingface_hub import CommitOperationAdd, HfApi, hf_hub_download
9
  from huggingface_hub.file_download import repo_folder_name
10
+ from safetensors.torch import save_file
11
  from transformers import AutoConfig
12
  from transformers.pipelines.base import infer_framework_load_model
 
13
 
14
 
15
  def check_file_size(sf_filename, pt_filename):
 
17
  pt_size = os.stat(pt_filename).st_size
18
 
19
  if (sf_size - pt_size) / pt_size > 0.01:
20
+ raise RuntimeError(
21
+ f"""The file size different is more than 1%:
22
  - {sf_filename}: {sf_size}
23
  - {pt_filename}: {pt_size}
24
+ """
25
+ )
26
 
27
 
28
  def rename(pt_filename) -> str:
 
31
  return local
32
 
33
 
34
+ def convert_multi(model_id, folder):
35
  filename = hf_hub_download(repo_id=model_id, filename="pytorch_model.bin.index.json")
36
  with open(filename, "r") as f:
37
  data = json.load(f)
38
 
39
  filenames = set(data["weight_map"].values())
40
+ local_filenames = []
41
  for filename in filenames:
42
  cached_filename = hf_hub_download(repo_id=model_id, filename=filename)
43
  loaded = torch.load(cached_filename)
 
56
  json.dump(newdata, f)
57
  local_filenames.append(index)
58
 
59
+ operations = [
60
+ CommitOperationAdd(path_in_repo=local.split("/")[-1], path_or_fileobj=local) for local in local_filenames
61
+ ]
62
 
63
  return operations
64
 
 
76
  operations = [CommitOperationAdd(path_in_repo=sf_filename, path_or_fileobj=local)]
77
  return operations
78
 
79
+
80
  def check_final_model(model_id, folder):
81
  config = hf_hub_download(repo_id=model_id, filename="config.json")
82
  shutil.copy(config, os.path.join(folder, "config.json"))
 
97
 
98
  folder = repo_folder_name(repo_id=model_id, repo_type="models")
99
  os.makedirs(folder)
100
+ new_pr = None
101
  try:
102
  operations = None
103
  if "model.safetensors" in filenames or "model_index.safetensors.index.json" in filenames:
 
111
 
112
  if operations:
113
  check_final_model(model_id, folder)
114
+ new_pr = api.create_commit(
115
  repo_id=model_id,
116
  operations=operations,
117
  commit_message="Adding `safetensors` variant of this model",
 
119
  )
120
  finally:
121
  shutil.rmtree(folder)
122
+ return new_pr
123
 
124
 
125
  if __name__ == "__main__":