Spaces:
Sleeping
Sleeping
inoki-giskard
commited on
Commit
•
a89f9d8
1
Parent(s):
136af2d
Format and remove duplicated file close
Browse files- io_utils.py +14 -16
- text_classification_ui_helpers.py +8 -12
io_utils.py
CHANGED
@@ -1,14 +1,18 @@
|
|
1 |
import os
|
2 |
import subprocess
|
3 |
-
|
4 |
import yaml
|
5 |
|
|
|
|
|
6 |
YAML_PATH = "./configs"
|
7 |
|
|
|
8 |
class Dumper(yaml.Dumper):
|
9 |
def increase_indent(self, flow=False, *args, **kwargs):
|
10 |
return super().increase_indent(flow=flow, indentless=False)
|
11 |
|
|
|
12 |
def get_yaml_path(uid):
|
13 |
if not os.path.exists(YAML_PATH):
|
14 |
os.makedirs(YAML_PATH)
|
@@ -16,6 +20,7 @@ def get_yaml_path(uid):
|
|
16 |
os.system(f"cp {YAML_PATH}/config.yaml {YAML_PATH}/{uid}_config.yaml")
|
17 |
return f"{YAML_PATH}/{uid}_config.yaml"
|
18 |
|
|
|
19 |
# read scanners from yaml file
|
20 |
# return a list of scanners
|
21 |
def read_scanners(uid):
|
@@ -23,7 +28,6 @@ def read_scanners(uid):
|
|
23 |
with open(get_yaml_path(uid), "r") as f:
|
24 |
config = yaml.load(f, Loader=yaml.FullLoader)
|
25 |
scanners = config.get("detectors", [])
|
26 |
-
f.close()
|
27 |
return scanners
|
28 |
|
29 |
|
@@ -35,7 +39,6 @@ def write_scanners(scanners, uid):
|
|
35 |
config["detectors"] = scanners
|
36 |
# save scanners to detectors in yaml
|
37 |
yaml.dump(config, f, Dumper=Dumper)
|
38 |
-
f.close()
|
39 |
|
40 |
|
41 |
# read model_type from yaml file
|
@@ -44,7 +47,6 @@ def read_inference_type(uid):
|
|
44 |
with open(get_yaml_path(uid), "r") as f:
|
45 |
config = yaml.load(f, Loader=yaml.FullLoader)
|
46 |
inference_type = config.get("inference_type", "")
|
47 |
-
f.close()
|
48 |
return inference_type
|
49 |
|
50 |
|
@@ -52,13 +54,13 @@ def read_inference_type(uid):
|
|
52 |
def write_inference_type(use_inference, uid):
|
53 |
with open(get_yaml_path(uid), "r+") as f:
|
54 |
config = yaml.load(f, Loader=yaml.FullLoader)
|
55 |
-
|
56 |
-
|
57 |
-
|
58 |
-
|
59 |
-
|
60 |
-
|
61 |
-
|
62 |
|
63 |
# read column mapping from yaml file
|
64 |
def read_column_mapping(uid):
|
@@ -67,7 +69,6 @@ def read_column_mapping(uid):
|
|
67 |
config = yaml.load(f, Loader=yaml.FullLoader)
|
68 |
if config:
|
69 |
column_mapping = config.get("column_mapping", dict())
|
70 |
-
f.close()
|
71 |
return column_mapping
|
72 |
|
73 |
|
@@ -75,7 +76,6 @@ def read_column_mapping(uid):
|
|
75 |
def write_column_mapping(mapping, uid):
|
76 |
with open(get_yaml_path(uid), "r") as f:
|
77 |
config = yaml.load(f, Loader=yaml.FullLoader)
|
78 |
-
f.close()
|
79 |
if config is None:
|
80 |
return
|
81 |
if mapping is None and "column_mapping" in config.keys():
|
@@ -85,7 +85,6 @@ def write_column_mapping(mapping, uid):
|
|
85 |
with open(get_yaml_path(uid), "w") as f:
|
86 |
# save column_mapping to column_mapping in yaml
|
87 |
yaml.dump(config, f, Dumper=Dumper)
|
88 |
-
f.close()
|
89 |
|
90 |
|
91 |
# convert column mapping dataframe to json
|
@@ -114,6 +113,7 @@ def save_job_to_pipe(id, job, lock):
|
|
114 |
with lock:
|
115 |
pipe.jobs.append((id, job))
|
116 |
|
|
|
117 |
def pop_job_from_pipe():
|
118 |
if len(pipe.jobs) == 0:
|
119 |
return
|
@@ -128,5 +128,3 @@ def pop_job_from_pipe():
|
|
128 |
stdout=log_file,
|
129 |
stderr=log_file,
|
130 |
)
|
131 |
-
|
132 |
-
|
|
|
1 |
import os
|
2 |
import subprocess
|
3 |
+
|
4 |
import yaml
|
5 |
|
6 |
+
import pipe
|
7 |
+
|
8 |
YAML_PATH = "./configs"
|
9 |
|
10 |
+
|
11 |
class Dumper(yaml.Dumper):
|
12 |
def increase_indent(self, flow=False, *args, **kwargs):
|
13 |
return super().increase_indent(flow=flow, indentless=False)
|
14 |
|
15 |
+
|
16 |
def get_yaml_path(uid):
|
17 |
if not os.path.exists(YAML_PATH):
|
18 |
os.makedirs(YAML_PATH)
|
|
|
20 |
os.system(f"cp {YAML_PATH}/config.yaml {YAML_PATH}/{uid}_config.yaml")
|
21 |
return f"{YAML_PATH}/{uid}_config.yaml"
|
22 |
|
23 |
+
|
24 |
# read scanners from yaml file
|
25 |
# return a list of scanners
|
26 |
def read_scanners(uid):
|
|
|
28 |
with open(get_yaml_path(uid), "r") as f:
|
29 |
config = yaml.load(f, Loader=yaml.FullLoader)
|
30 |
scanners = config.get("detectors", [])
|
|
|
31 |
return scanners
|
32 |
|
33 |
|
|
|
39 |
config["detectors"] = scanners
|
40 |
# save scanners to detectors in yaml
|
41 |
yaml.dump(config, f, Dumper=Dumper)
|
|
|
42 |
|
43 |
|
44 |
# read model_type from yaml file
|
|
|
47 |
with open(get_yaml_path(uid), "r") as f:
|
48 |
config = yaml.load(f, Loader=yaml.FullLoader)
|
49 |
inference_type = config.get("inference_type", "")
|
|
|
50 |
return inference_type
|
51 |
|
52 |
|
|
|
54 |
def write_inference_type(use_inference, uid):
|
55 |
with open(get_yaml_path(uid), "r+") as f:
|
56 |
config = yaml.load(f, Loader=yaml.FullLoader)
|
57 |
+
if use_inference:
|
58 |
+
config["inference_type"] = "hf_inference_api"
|
59 |
+
else:
|
60 |
+
config["inference_type"] = "hf_pipeline"
|
61 |
+
# save inference_type to inference_type in yaml
|
62 |
+
yaml.dump(config, f, Dumper=Dumper)
|
63 |
+
|
64 |
|
65 |
# read column mapping from yaml file
|
66 |
def read_column_mapping(uid):
|
|
|
69 |
config = yaml.load(f, Loader=yaml.FullLoader)
|
70 |
if config:
|
71 |
column_mapping = config.get("column_mapping", dict())
|
|
|
72 |
return column_mapping
|
73 |
|
74 |
|
|
|
76 |
def write_column_mapping(mapping, uid):
|
77 |
with open(get_yaml_path(uid), "r") as f:
|
78 |
config = yaml.load(f, Loader=yaml.FullLoader)
|
|
|
79 |
if config is None:
|
80 |
return
|
81 |
if mapping is None and "column_mapping" in config.keys():
|
|
|
85 |
with open(get_yaml_path(uid), "w") as f:
|
86 |
# save column_mapping to column_mapping in yaml
|
87 |
yaml.dump(config, f, Dumper=Dumper)
|
|
|
88 |
|
89 |
|
90 |
# convert column mapping dataframe to json
|
|
|
113 |
with lock:
|
114 |
pipe.jobs.append((id, job))
|
115 |
|
116 |
+
|
117 |
def pop_job_from_pipe():
|
118 |
if len(pipe.jobs) == 0:
|
119 |
return
|
|
|
128 |
stdout=log_file,
|
129 |
stderr=log_file,
|
130 |
)
|
|
|
|
text_classification_ui_helpers.py
CHANGED
@@ -8,17 +8,10 @@ import datasets
|
|
8 |
import gradio as gr
|
9 |
from transformers.pipelines import TextClassificationPipeline
|
10 |
|
11 |
-
from io_utils import (
|
12 |
-
|
13 |
-
|
14 |
-
|
15 |
-
write_log_to_user_file,
|
16 |
-
)
|
17 |
-
from text_classification import (
|
18 |
-
check_model,
|
19 |
-
get_example_prediction,
|
20 |
-
get_labels_and_features_from_dataset,
|
21 |
-
)
|
22 |
from wordings import CONFIRM_MAPPING_DETAILS_FAIL_RAW
|
23 |
|
24 |
MAX_LABELS = 20
|
@@ -28,6 +21,7 @@ HF_REPO_ID = "HF_REPO_ID"
|
|
28 |
HF_SPACE_ID = "SPACE_ID"
|
29 |
HF_WRITE_TOKEN = "HF_WRITE_TOKEN"
|
30 |
|
|
|
31 |
def check_dataset_and_get_config(dataset_id, uid):
|
32 |
try:
|
33 |
write_column_mapping(None, uid) # reset column mapping
|
@@ -48,7 +42,9 @@ def check_dataset_and_get_split(dataset_id, dataset_config):
|
|
48 |
pass
|
49 |
|
50 |
|
51 |
-
def write_column_mapping_to_config(
|
|
|
|
|
52 |
# TODO: Substitute 'text' with more features for zero-shot
|
53 |
# we are not using ds features because we only support "text" for now
|
54 |
ds_labels, _ = get_labels_and_features_from_dataset(
|
|
|
8 |
import gradio as gr
|
9 |
from transformers.pipelines import TextClassificationPipeline
|
10 |
|
11 |
+
from io_utils import (read_column_mapping, save_job_to_pipe,
|
12 |
+
write_column_mapping, write_log_to_user_file)
|
13 |
+
from text_classification import (check_model, get_example_prediction,
|
14 |
+
get_labels_and_features_from_dataset)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
15 |
from wordings import CONFIRM_MAPPING_DETAILS_FAIL_RAW
|
16 |
|
17 |
MAX_LABELS = 20
|
|
|
21 |
HF_SPACE_ID = "SPACE_ID"
|
22 |
HF_WRITE_TOKEN = "HF_WRITE_TOKEN"
|
23 |
|
24 |
+
|
25 |
def check_dataset_and_get_config(dataset_id, uid):
|
26 |
try:
|
27 |
write_column_mapping(None, uid) # reset column mapping
|
|
|
42 |
pass
|
43 |
|
44 |
|
45 |
+
def write_column_mapping_to_config(
|
46 |
+
dataset_id, dataset_config, dataset_split, uid, *labels
|
47 |
+
):
|
48 |
# TODO: Substitute 'text' with more features for zero-shot
|
49 |
# we are not using ds features because we only support "text" for now
|
50 |
ds_labels, _ = get_labels_and_features_from_dataset(
|