Merge branch 'main' of https://huggingface.co/spaces/GEM/DatasetCardForm

formatting/README.md

@@ -1,7 +1,9 @@
 # Formatting Utilities
 
-The resources in this folder are used to format the saved datacard in a readable format.
-The file `key_to_question.json` maps from the saved key back to the original question which can then be parsed into question/answer pairs.
+The resources in this folder are used to format the saved dataset card in a readable format (JSON, Markdown).
 
-
-In the json, everything labeled `N/A` will get rendered, empty fields will be completely omitted.
+File | Description
+--- | ---
+[`key_to_question.json`](https://huggingface.co/spaces/GEM/DatasetCardForm/blob/main/formatting/key_to_question.json) | Maps from the saved key back to the original question, which can then be parsed into question/answer pairs.
+[`reformat_json.py`](https://huggingface.co/spaces/GEM/DatasetCardForm/blob/main/formatting/reformat_json.py) | Uses `key_to_question.json` to transform dataset cards into a JSON organized by Data Card standards (e.g., sections, subsections, scopes). Everything labeled `N/A` in the original card will be rendered; empty fields will be completely omitted.
+[`json_to_md.py`](https://huggingface.co/spaces/GEM/DatasetCardForm/blob/main/formatting/json_to_md.py) | Transforms output from `reformat_json.py` into Markdown that is compatible with Data Cards Labs (e.g., special comment syntax).

formatting/format_as_md.py

@@ -1,78 +0,0 @@
-import argparse
-import json
-import pathlib
-import os
-
-parser = argparse.ArgumentParser(
-    description="Format the output of the data card tool as .md for the hub."
-)
-parser.add_argument("--input_path", "-i", type=pathlib.Path, required=True)
-parser.add_argument("--output_path", "-o", type=pathlib.Path, required=True)
-args = parser.parse_args()
-
-def read_json_file(json_path: pathlib.Path):
-    """Load a json file and return it as object."""
-    with open(json_path, "r") as f:
-        data = json.load(f)
-    return data
-
-def save_md_file(md_path: pathlib.Path, md_string: str):
-    """Takes a string and saves it as .md file."""
-    with open(md_path, 'w') as f:
-        f.write(md_string)
-
-def construct_md(data_card_data: dict, text_by_key: dict):
-    """Constructs the markdown file
-
-    This function iterates through text_by_key and extracts all answers from
-    the data_card_data object. It uses the levels of hierarchy as indicator for
-    the heading indentation and does not change the order in which anything
-    appears.
-
-    Args:
-        data_card_data: Output from the data card tool
-        text_by_key: configuration defined in key_to_question.json
-
-    Returns:
-        data_card_md_string: Markdown-formatted content
-    """
-
-    data_card_md_string = ""
-
-    for main_key, main_content in text_by_key.items():
-        section_header = main_content['section-title']
-        # Remove it so that we don't iterate over it.
-        del main_content['section-title']
-        # Add it to string.
-        data_card_md_string += f"## {section_header} \n\n"
-        # Iterate over the subsections.
-        for second_key, second_content in main_content.items():
-            subsection_header = second_content['section-title']
-            # Remove it so that we don't iterate over it.
-            del second_content['section-title']
-            # Add it to string.
-            data_card_md_string += f"### {subsection_header} \n\n"
-            # Finally, iterate over actual questions.
-            for question_key, question_text in second_content.items():
-                # Now grab the answer text.
-                reply = data_card_data[main_key][second_key].get(question_key, "N/A")
-                if reply and reply != "N/A":
-                    data_card_md_string += f"#### {question_text} \n\n"
-                    # Special parsing options here.
-                    if question_key == question_key == "paper-bibtext":
-                        data_card_md_string += f"```\n{reply}\n``` \n\n"
-                    elif question_key == "structure-example":
-                        data_card_md_string += f"```json\n{reply}\n``` \n\n"
-                    elif isinstance(reply, list):
-                        data_card_md_string += f"{' '.join(reply)} \n\n"
-                    else:
-                        data_card_md_string += f"{reply} \n\n"
-
-    return data_card_md_string
-
-
-if __name__ == "__main__":
-    data_card_data = read_json_file(args.input_path)
-    text_by_key = read_json_file(os.path.join(os.path.dirname(__file__), "key_to_question.json"))
-    data_card_md_string = construct_md(data_card_data, text_by_key)
-    save_md_file(args.output_path, data_card_md_string)

formatting/json_to_md.py

@@ -0,0 +1,72 @@
+from argparse import ArgumentParser
+from json import load
+
+def parse_args():
+    parser = ArgumentParser()
+    parser.add_argument('input', type=str, nargs='+', \
+                        help='Specify paths to files (e.g., path/to/*.json)')
+
+    return parser.parse_args()
+
+
+def json_to_markdown(filename):
+    json = load(open(filename))
+
+    markdown = f'# {json["name"]}\n\n'
+    markdown += json['summary'] + '\n\n'
+
+    for key in json:
+        if key not in ('name', 'summary', 'sections'):
+            markdown += f'#### {key}\n{json[key]}\n\n'
+
+    markdown += '\n'.join(section_to_markdown(section) \
+                          for section in json['sections'])
+
+    with open(f'{filename[:-5]}.md', 'w') as f:
+        f.write(markdown)
+
+
+def section_to_markdown(section):
+    markdown = f'{"#" * section["level"]} {section["title"]}\n\n'
+    markdown += '\n'.join(subsection_to_markdown(subsection) \
+                          for subsection in section['subsections'])
+
+    return markdown + '\n'
+
+
+def subsection_to_markdown(subsection):
+    markdown = f'{"#" * subsection["level"]} {subsection["title"]}\n\n'
+    markdown += '\n'.join(field_to_markdown(field) \
+                          for field in subsection['fields'])
+
+    return markdown + '\n'
+
+
+def field_to_markdown(field):
+    markdown = f'{"#" * field["level"]} {field["title"]}\n\n'
+
+    if 'flags' in field and 'quick' in field['flags']:
+        markdown += f'<!-- quick -->\n'
+
+    if field.get('info', False):
+        markdown += f'<!-- info: {field["info"]} -->\n'
+
+    if field.get('scope', False):
+        markdown += f'<!-- scope: {field["scope"]} -->\n'
+
+    markdown += field.get('content', '')
+
+    return markdown + '\n'
+
+
+def main():
+    """Converts JSON output from `reformat_json.py`
+       to Markdown input for Data Cards Labs."""
+    args = parse_args()
+    for filename in args.input:
+        if filename[-5:] == '.json':
+            json_to_markdown(filename)
+
+
+if __name__ == '__main__':
+    main()