Xmaster6y commited on
Commit
de554eb
1 Parent(s): 24b09f4

votes not in metadata

Browse files
src/global_variables.py CHANGED
@@ -1,6 +1,8 @@
1
  """Global variables used in the space.
2
  """
3
 
 
 
4
  from huggingface_hub import HfApi
5
  import jsonlines
6
 
@@ -10,23 +12,30 @@ from src.constants import DATASET_NAME, HF_TOKEN, ASSETS_FOLDER
10
 
11
  hf_api: HfApi
12
  all_metadata: dict
 
13
 
14
 
15
  def setup():
16
  global hf_api
17
  global all_metadata
 
18
  hf_api = HfApi(token=HF_TOKEN)
19
- hf_api.snapshot_download(
20
- local_dir=f"{ASSETS_FOLDER}/{DATASET_NAME}",
21
- repo_id=DATASET_NAME,
22
- repo_type="dataset",
23
- )
24
  all_metadata = {}
25
  for split in ["train", "validation", "test"]:
26
  all_metadata[split] = []
27
  with jsonlines.open(f"{ASSETS_FOLDER}/{DATASET_NAME}/data/{split}/metadata.jsonl") as reader:
28
  for row in reader:
29
  all_metadata[split].append(row)
 
 
 
 
 
30
 
31
  def get_metadata(split):
32
  global all_metadata
@@ -55,5 +64,32 @@ def save_metadata(split):
55
  repo_type="dataset",
56
  )
57
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
58
  if gr.NO_RELOAD:
59
- setup()
 
1
  """Global variables used in the space.
2
  """
3
 
4
+ import os
5
+ import json
6
  from huggingface_hub import HfApi
7
  import jsonlines
8
 
 
12
 
13
  hf_api: HfApi
14
  all_metadata: dict
15
+ all_votes: dict
16
 
17
 
18
  def setup():
19
  global hf_api
20
  global all_metadata
21
+ global all_votes
22
  hf_api = HfApi(token=HF_TOKEN)
23
+ # hf_api.snapshot_download(
24
+ # local_dir=f"{ASSETS_FOLDER}/{DATASET_NAME}",
25
+ # repo_id=DATASET_NAME,
26
+ # repo_type="dataset",
27
+ # )
28
  all_metadata = {}
29
  for split in ["train", "validation", "test"]:
30
  all_metadata[split] = []
31
  with jsonlines.open(f"{ASSETS_FOLDER}/{DATASET_NAME}/data/{split}/metadata.jsonl") as reader:
32
  for row in reader:
33
  all_metadata[split].append(row)
34
+ all_votes = {}
35
+ for file in os.listdir(f"{ASSETS_FOLDER}/{DATASET_NAME}/votes"):
36
+ with open(f"{ASSETS_FOLDER}/{DATASET_NAME}/votes/{file}") as f:
37
+ key = file.split(".")[0]
38
+ all_votes[key] = json.load(f)
39
 
40
  def get_metadata(split):
41
  global all_metadata
 
64
  repo_type="dataset",
65
  )
66
 
67
+ def get_votes(key):
68
+ global all_votes
69
+ global hf_api
70
+ try:
71
+ hf_api.hf_hub_download(
72
+ repo_id=DATASET_NAME,
73
+ filename=f"votes/{key}.json",
74
+ repo_type="dataset",
75
+ local_dir=f"{ASSETS_FOLDER}/{DATASET_NAME}",
76
+ )
77
+ with open(f"{ASSETS_FOLDER}/{DATASET_NAME}/votes/{key}.json") as f:
78
+ all_votes[key] = json.load(f)
79
+ except:
80
+ pass
81
+
82
+ def save_votes(key):
83
+ global all_votes
84
+ global hf_api
85
+ with open(f"{ASSETS_FOLDER}/{DATASET_NAME}/votes/{key}.json", "w") as f:
86
+ json.dump(all_votes[key], f)
87
+ hf_api.upload_file(
88
+ path_or_fileobj=f"{ASSETS_FOLDER}/{DATASET_NAME}/votes/{key}.json",
89
+ path_in_repo=f"votes/{key}.json",
90
+ repo_id=DATASET_NAME,
91
+ repo_type="dataset",
92
+ )
93
+
94
  if gr.NO_RELOAD:
95
+ setup()
src/label_interface.py CHANGED
@@ -47,7 +47,7 @@ def get_next_image(
47
  sample = global_variables.all_metadata[split][sample_idx]
48
  image_path = f"{ASSETS_FOLDER}/{DATASET_NAME}/data/{split}/{sample['file_name']}"
49
  try:
50
- username_votes = sample["votes"][username]
51
  voted_concepts = [c for c in CONCEPTS if username_votes.get(c, False)]
52
  unseen_concepts = [c for c in CONCEPTS if c not in username_votes]
53
  except KeyError:
@@ -89,19 +89,22 @@ def submit_label(
89
  current_split, idx = current_image.split(":")
90
  idx = int(idx)
91
  global_variables.get_metadata(current_split)
92
- if "votes" not in global_variables.all_metadata[current_split][idx]:
93
- global_variables.all_metadata[current_split][idx]["votes"] = {}
94
- global_variables.all_metadata[current_split][idx]["votes"][username] = {c: c in voted_concepts for c in CONCEPTS}
 
 
95
  vote_sum = {c: 0 for c in CONCEPTS}
96
  new_concepts = {}
97
  for c in CONCEPTS:
98
- for vote in global_variables.all_metadata[current_split][idx]["votes"].values():
99
  if c not in vote:
100
  continue
101
  vote_sum[c] += 2 * vote[c] - 1
102
  new_concepts[c] = vote_sum[c] > 0 if vote_sum[c] != 0 else None
103
  global_variables.all_metadata[current_split][idx]["concepts"] = new_concepts
104
  global_variables.save_metadata(current_split)
 
105
  gr.Info("Submit success")
106
  return get_next_image(
107
  split,
 
47
  sample = global_variables.all_metadata[split][sample_idx]
48
  image_path = f"{ASSETS_FOLDER}/{DATASET_NAME}/data/{split}/{sample['file_name']}"
49
  try:
50
+ username_votes = global_variables.all_votes[sample["id"]][username]
51
  voted_concepts = [c for c in CONCEPTS if username_votes.get(c, False)]
52
  unseen_concepts = [c for c in CONCEPTS if c not in username_votes]
53
  except KeyError:
 
89
  current_split, idx = current_image.split(":")
90
  idx = int(idx)
91
  global_variables.get_metadata(current_split)
92
+ s_id = global_variables.all_metadata[current_split][idx]["id"]
93
+ global_variables.get_votes(s_id)
94
+ if s_id not in global_variables.all_votes:
95
+ global_variables.all_votes[s_id] = {}
96
+ global_variables.all_votes[s_id][username] = {c: c in voted_concepts for c in CONCEPTS}
97
  vote_sum = {c: 0 for c in CONCEPTS}
98
  new_concepts = {}
99
  for c in CONCEPTS:
100
+ for vote in global_variables.all_votes[s_id].values():
101
  if c not in vote:
102
  continue
103
  vote_sum[c] += 2 * vote[c] - 1
104
  new_concepts[c] = vote_sum[c] > 0 if vote_sum[c] != 0 else None
105
  global_variables.all_metadata[current_split][idx]["concepts"] = new_concepts
106
  global_variables.save_metadata(current_split)
107
+ global_variables.save_votes(s_id)
108
  gr.Info("Submit success")
109
  return get_next_image(
110
  split,
src/sample_interface.py CHANGED
@@ -31,7 +31,7 @@ def get_image(
31
  sample = global_variables.all_metadata[split][sample_idx]
32
  image_path = f"{ASSETS_FOLDER}/{DATASET_NAME}/data/{split}/{sample['file_name']}"
33
  try:
34
- username_votes = sample["votes"][username]
35
  voted_concepts = [c for c in CONCEPTS if username_votes.get(c, False)]
36
  unseen_concepts = [c for c in CONCEPTS if c not in username_votes]
37
  except KeyError:
@@ -80,19 +80,22 @@ def submit_label(
80
  current_split, idx = current_image.split(":")
81
  idx = int(idx)
82
  global_variables.get_metadata(current_split)
83
- if "votes" not in global_variables.all_metadata[current_split][idx]:
84
- global_variables.all_metadata[current_split][idx]["votes"] = {}
85
- global_variables.all_metadata[current_split][idx]["votes"][username] = {c: c in voted_concepts for c in CONCEPTS}
 
 
86
  vote_sum = {c: 0 for c in CONCEPTS}
87
  new_concepts = {}
88
  for c in CONCEPTS:
89
- for vote in global_variables.all_metadata[current_split][idx]["votes"].values():
90
  if c not in vote:
91
  continue
92
  vote_sum[c] += 2 * vote[c] - 1
93
  new_concepts[c] = vote_sum[c] > 0 if vote_sum[c] != 0 else None
94
  global_variables.all_metadata[current_split][idx]["concepts"] = new_concepts
95
  global_variables.save_metadata(current_split)
 
96
  gr.Info("Submit success")
97
  return get_next_image(
98
  split,
 
31
  sample = global_variables.all_metadata[split][sample_idx]
32
  image_path = f"{ASSETS_FOLDER}/{DATASET_NAME}/data/{split}/{sample['file_name']}"
33
  try:
34
+ username_votes = global_variables.all_votes[sample["id"]][username]
35
  voted_concepts = [c for c in CONCEPTS if username_votes.get(c, False)]
36
  unseen_concepts = [c for c in CONCEPTS if c not in username_votes]
37
  except KeyError:
 
80
  current_split, idx = current_image.split(":")
81
  idx = int(idx)
82
  global_variables.get_metadata(current_split)
83
+ s_id = global_variables.all_metadata[current_split][idx]["id"]
84
+ global_variables.get_votes(s_id)
85
+ if s_id not in global_variables.all_votes:
86
+ global_variables.all_votes[s_id] = {}
87
+ global_variables.all_votes[s_id][username] = {c: c in voted_concepts for c in CONCEPTS}
88
  vote_sum = {c: 0 for c in CONCEPTS}
89
  new_concepts = {}
90
  for c in CONCEPTS:
91
+ for vote in global_variables.all_votes[s_id].values():
92
  if c not in vote:
93
  continue
94
  vote_sum[c] += 2 * vote[c] - 1
95
  new_concepts[c] = vote_sum[c] > 0 if vote_sum[c] != 0 else None
96
  global_variables.all_metadata[current_split][idx]["concepts"] = new_concepts
97
  global_variables.save_metadata(current_split)
98
+ global_variables.save_votes(s_id)
99
  gr.Info("Submit success")
100
  return get_next_image(
101
  split,
src/vote_interface.py CHANGED
@@ -13,8 +13,8 @@ def get_votes(
13
  username = profile.username
14
  vote_list = []
15
  for i,s in enumerate(global_variables.all_metadata[split]):
16
- if "votes" in s and username in s["votes"]:
17
- vote_list.append(f'[{i}]: {s["votes"][username]}')
18
 
19
  return "\n".join(vote_list)
20
 
 
13
  username = profile.username
14
  vote_list = []
15
  for i,s in enumerate(global_variables.all_metadata[split]):
16
+ if s["id"] in global_variables.all_votes and username in global_variables.all_votes[s["id"]]:
17
+ vote_list.append(f'[{i}]: {global_variables.all_votes[s["id"]][username]}')
18
 
19
  return "\n".join(vote_list)
20