Spaces:
Sleeping
Sleeping
File size: 5,768 Bytes
739cf2e |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 |
import logging
from pathlib import Path
import gradio as gr
from datasets import Dataset
from gradio_log import Log
from huggingface_hub import DatasetCard
from llama_index.core import SimpleDirectoryReader
from llama_index.core.node_parser import SentenceSplitter
from llama_index.core.schema import MetadataMode
from tqdm.auto import tqdm
log_file = "logs.txt"
Path(log_file).touch(exist_ok=True)
logging.basicConfig(filename="logs.txt", level=logging.INFO)
logging.getLogger().addHandler(logging.FileHandler(log_file))
def load_corpus(files, chunk_size=256, chunk_overlap=0, verbose=True):
if verbose:
gr.Info("Loading files...")
reader = SimpleDirectoryReader(input_files=files)
docs = reader.load_data()
if verbose:
print(f"Loaded {len(docs)} docs")
parser = SentenceSplitter.from_defaults(
chunk_size=chunk_size, chunk_overlap=chunk_overlap
)
nodes = parser.get_nodes_from_documents(docs, show_progress=verbose)
if verbose:
print(f"Parsed {len(nodes)} nodes")
docs = {
node.node_id: node.get_content(metadata_mode=MetadataMode.NONE)
for node in tqdm(nodes)
}
# remove empty docs
docs = {k: v for k, v in docs.items() if v}
return docs
def upload_file(
files,
chunk_size: int = 256,
chunk_overlap: int = 0,
hub_id: str = None,
private: bool = False,
oauth_token: gr.OAuthToken = None,
):
print("loading files")
file_paths = [file.name for file in files]
print("parsing into sentences")
corpus = load_corpus(file_paths, chunk_size=chunk_size, chunk_overlap=chunk_overlap)
print("Creating dataset")
dataset = Dataset.from_dict({"ids": corpus.keys(), "texts": corpus.values()})
message = f"Dataset created has: \n - {len(dataset)} rows"
if hub_id:
if oauth_token is not None:
gr.Info("Uploading to Hugging Face Hub")
dataset.push_to_hub(hub_id, token=oauth_token.token, private=private)
update_dataset_card(hub_id, oauth_token.token, chunk_size, chunk_overlap)
message += f"\n\nUploaded to [{hub_id}](https://huggingface.co/{hub_id}"
else:
raise gr.Error("Please login to Hugging Face Hub to push to hub")
return dataset.to_pandas(), message
def update_dataset_card(
hub_id,
token,
chunk_size,
chunk_overlap,
):
card = DatasetCard.load(hub_id, token=token)
if not card.text:
# add template description to card text
card.text += f"""This dataset was created using [Corpus Creator](https://huggingface.co/spaces/davanstrien/corpus-creator). This dataset was created by parsing a corpus of text files into chunks of sentences using Llama Index.
This processing was done with a chunk size of {chunk_size} and a chunk overlap of {chunk_overlap}."""
tags = card.data.get("tags", [])
tags.append("corpus-creator")
card.data["tags"] = tags
card.push_to_hub(hub_id, token=token)
description = """
Corpus Creator is a tool designed to help you easily convert a collection of text files into a dataset suitable for various natural language processing (NLP) tasks.
In particular the app is focused on splitting texts into chunks of a specified size and overlap. This can be useful for preparing data for synthetic data generation, pipelines or annotation tasks.
The resulting text chunks are stored in a dataset that can be previewed and uploaded to the Hugging Face Hub for easy sharing and access by the community.
The chunking is done using `Llama-index`'s [`SentenceSplitter`](https://docs.llamaindex.ai/en/stable/module_guides/loading/node_parsers/modules/?h=sentencesplitter#sentencesplitter) classes.
### Usage:
- Login: Start by logging in to your Hugging Face account using the provided login button.
- Set Parameters: Customize the chunk size and overlap according to your requirements.
- Upload Files: Use the upload button to load file(s) for processing.
- Preview Dataset: View the created dataset in a dataframe format before uploading it to the Hugging Face Hub.
- Upload to Hub: Optionally, specify the Hub ID and choose whether to make the dataset private before pushing it to the Hugging Face Hub."""
with gr.Blocks() as demo:
gr.HTML(
"""<h1 style='text-align: center;'> Corpus Creator</h1>
<center><i> 📁 From random files to a Hugging Face dataset in a single step 📁 </i></center>"""
)
gr.Markdown(description)
with gr.Row():
gr.LoginButton()
with gr.Column():
gr.Markdown(
"To upload to the Hub, add an ID for where you want to push the dataset"
)
hub_id = gr.Textbox(value=None, label="Hub ID")
with gr.Row():
chunk_size = gr.Number(
256,
label="Chunk size (size to split text into)",
minimum=10,
maximum=4096,
step=1,
)
chunk_overlap = gr.Number(
0,
label="Chunk overlap (overlap size between chunks)",
minimum=0,
maximum=4096,
step=1,
)
private = gr.Checkbox(False, label="Upload dataset to a private repo?")
upload_button = gr.UploadButton(
"Load files to corpus",
file_types=[
"text",
],
file_count="multiple",
)
summary = gr.Markdown()
with gr.Accordion("detailed logs", open=False):
Log(log_file, dark=True, xterm_font_size=12)
corpus_preview_df = gr.DataFrame()
upload_button.upload(
upload_file,
inputs=[upload_button, chunk_size, chunk_overlap, hub_id, private],
outputs=[corpus_preview_df, summary],
)
demo.launch(debug=True)
|