Update curated.py

curated.py

@@ -46,19 +46,12 @@ treemap_data = {
     'Deep Mind Maths dataset with generated questions.'
   ]
 }
-# Calculate percentage for each data source
 total_count = sum(treemap_data['Count'])
 treemap_data['Percentage'] = [count / total_count * 100 for count in treemap_data['Count']]
-
-# Create treemap
 fig = px.treemap(treemap_data, path=['Category', 'Source'], values='Count', hover_data=['Details', 'Percentage'], hover_name='Source')
-
-# Set the size of the chart
-
-
-# Display treemap if you want to update the size.update_layout(width=800, height=600)
 treemap_chart = fig
 
+#start individual tables showing filterin
 wikipedia_filter = pd.DataFrame(
         {
             "Dataset": [
@@ -438,291 +431,23 @@ phil_filter = pd.DataFrame(
 
 table_html_phil = phil_filter.to_html(index=False, border=0)
 table_div_phil = Div(NotStr(table_html_phil), style="margin: 40px;")
+## end individual tables showing filterin
 
-data_sources = [
-    "Freelaw",
-    "Wikipedia",
-    "PhilPapers",
-    "Arxiv",
-    "S2ORC",
-    "S2ORC Abstract",
-    "Pubmed",
-    "USPTO",
-    "Hackernews",
-    "Ubuntu IRC",
-    "StackExchange",
-    "DM Maths",
-    "PG19",
-    "Europarl",
-]
-
-
-
-def get_wiki_data(data_source: str = "Wikipedia", doc_id: int = 3, target: str = "foo"):
-    doc_id = max(0, min(int(doc_id), 9))
-
-    if data_source == "Wikipedia":
-        raw_sample_doc = extracted_sample_doc = json.load(
-            open("data/curated_samples/wiki.json")
-        )
-    else:
-        raw_sample_doc = extracted_sample_doc = [{} for _ in range(10)]
-
-    raw_json = raw_sample_doc[doc_id]
-    extracted_json = extracted_sample_doc[doc_id]
-    return view_data(
-        raw_json,
-        extracted_json,
-        doc_id=doc_id,
-        data_source="Wikipedia",
-        data_sources="Wikipedia",
-        target=target,
-    )
-
-wiki_examples = Div(
-    Div(
-        get_wiki_data(target=gen_random_id()),
-        style="border: 1px solid #ccc; padding: 20px;",
-    ),
-)
 
+## start filtered examples
 wiki_examples = DV("data/curated_samples/wiki.json", 0, "Wikipedia")
-
-
-def get_freelaw_data(data_source: str = "Freelaw", doc_id: int = 3, target: str = "foo"):
-    doc_id = max(0, min(int(doc_id), 9))
-
-    if data_source == "Freelaw":
-        raw_sample_doc = json.load(open("data/curated_samples/freelaw_raw.json"))
-        extracted_sample_doc = json.load(
-            open("data/curated_samples/freelaw_extract.json")
-        )
-    else:
-        raw_sample_doc = extracted_sample_doc = [{} for _ in range(10)]
-
-    raw_json = raw_sample_doc[doc_id]
-    extracted_json = extracted_sample_doc[doc_id]
-    return view_data(
-        raw_json,
-        extracted_json,
-        doc_id=doc_id,
-        data_source="Freelaw",
-        data_sources="Freelaw",
-        target=target,
-    )
-
 freelaw_examples = DV2("data/curated_samples/freelaw_raw.json", "data/curated_samples/freelaw_extract.json", 2)
-
-def get_se_data(data_source: str = "StackExchange", doc_id: int = 3, target: str = "foo"):
-    doc_id = max(0, min(int(doc_id), 9))
-
-    if data_source == "StackExchange":
-        raw_sample_doc = json.load(open("data/curated_samples/stackexchange_raw.json"))
-        extracted_sample_doc = json.load(
-            open("data/curated_samples/stackexchange_extract.json")
-        )
-    else:
-        raw_sample_doc = extracted_sample_doc = [{} for _ in range(10)]
-
-    raw_json = raw_sample_doc[doc_id]
-    extracted_json = extracted_sample_doc[doc_id]
-    return view_data(
-        raw_json,
-        extracted_json,
-        doc_id=doc_id,
-        data_source="StackExchange",
-        data_sources="StackExchange",
-        target=target,
-    )
-
 se_examples = DV2("data/curated_samples/stackexchange_raw.json", "data/curated_samples/stackexchange_extract.json", 3)
-
-def get_phil_data(data_source: str = "PhilPapers", doc_id: int = 3, target: str = "foo"):
-    doc_id = max(0, min(int(doc_id), 9))
-
-    if data_source == "PhilPapers":
-        raw_sample_doc = extracted_sample_doc = json.load(
-            open("data/curated_samples/philpapers_raw.json")
-        )
-    else:
-        raw_sample_doc = extracted_sample_doc = [{} for _ in range(10)]
-
-    raw_json = raw_sample_doc[doc_id]
-    extracted_json = extracted_sample_doc[doc_id]
-    return view_data(
-        raw_json,
-        extracted_json,
-        doc_id=doc_id,
-        data_source="PhilPapers",
-        data_sources="PhilPapers",
-        target=target,
-    )
-
 phil_examples = DV("data/curated_samples/philpapers_raw.json", 2, "PhilPapers")
-
-def get_arx_data(data_source: str = "Arxiv", doc_id: int = 3, target: str = "foo"):
-    doc_id = max(0, min(int(doc_id), 9))
-
-    if data_source == "Arxiv":
-        raw_sample_doc = json.load(open("data/curated_samples/arxiv_raw.json"))
-        extracted_sample_doc = json.load(
-            open("data/curated_samples/arxiv_extract.json")
-        )
-    else:
-        raw_sample_doc = extracted_sample_doc = [{} for _ in range(10)]
-
-    raw_json = raw_sample_doc[doc_id]
-    extracted_json = extracted_sample_doc[doc_id]
-    return view_data(
-        raw_json,
-        extracted_json,
-        doc_id=doc_id,
-        data_source="Arxiv",
-        data_sources="Arxiv",
-        target=target,
-    )
-
 arx_examples = DV2("data/curated_samples/arxiv_raw.json", "data/curated_samples/arxiv_extract.json", 3)
-
-def get_S2ORC_data(data_source: str = "S2ORC", doc_id: int = 3, target: str = "foo"):
-    doc_id = max(0, min(int(doc_id), 9))
-
-    if data_source == "S2ORC":
-        raw_sample_doc = extracted_sample_doc = json.load(
-            open("data/curated_samples/s2orc_raw.json")
-        )
-    else:
-        raw_sample_doc = extracted_sample_doc = [{} for _ in range(10)]
-
-    raw_json = raw_sample_doc[doc_id]
-    extracted_json = extracted_sample_doc[doc_id]
-    return view_data(
-        raw_json,
-        extracted_json,
-        doc_id=doc_id,
-        data_source="S2ORC",
-        data_sources="S2ORC",
-        target=target,
-    )
-
 s2o_examples = DV("data/curated_samples/s2orc_raw.json", 0, "S2ORC")
-
-def get_S2ORCA_data(data_source: str = "S2ORC Abstract", doc_id: int = 3, target: str = "foo"):
-    doc_id = max(0, min(int(doc_id), 9))
-
-    if data_source == "S2ORC":
-        raw_sample_doc = extracted_sample_doc = json.load(
-            open("data/curated_samples/s2orc_abstract_raw.json")
-        )
-    else:
-        raw_sample_doc = extracted_sample_doc = [{} for _ in range(10)]
-
-    raw_json = raw_sample_doc[doc_id]
-    extracted_json = extracted_sample_doc[doc_id]
-    return view_data(
-        raw_json,
-        extracted_json,
-        doc_id=doc_id,
-        data_source="S2ORC Abstract",
-        data_sources="S2ORC Abstract",
-        target=target,
-    )
-
 s2oa_examples = DV("data/curated_samples/s2orc_abstract_raw.json", 0, "S2ORC Abstract")
-
-def get_pubmed_data(data_source: str = "Pubmed", doc_id: int = 3, target: str = "foo"):
-    doc_id = max(0, min(int(doc_id), 9))
-
-    if data_source == "Pubmed":
-        raw_sample_doc = json.load(open("data/curated_samples/pubmed_raw.json"))
-        extracted_sample_doc = json.load(
-            open("data/curated_samples/pubmed_extract.json")
-        )
-    else:
-        raw_sample_doc = extracted_sample_doc = [{} for _ in range(10)]
-
-    raw_json = raw_sample_doc[doc_id]
-    extracted_json = extracted_sample_doc[doc_id]
-    return view_data(
-        raw_json,
-        extracted_json,
-        doc_id=doc_id,
-        data_source="Pubmed",
-        data_sources="Pubmed",
-        target=target,
-    )
-
 pubmed_examples = DV2("data/curated_samples/pubmed_raw.json", "data/curated_samples/pubmed_extract.json", 3)
-
-def get_dmm_data(data_source: str = "DM Math", doc_id: int = 3, target: str = "foo"):
-    doc_id = max(0, min(int(doc_id), 9))
-
-    if data_source == "DM Math":
-        raw_sample_doc = json.load(open("data/curated_samples/dm_maths_raw.json"))
-        extracted_sample_doc = json.load(
-            open("data/curated_samples/dm_maths_extract.json")
-        )
-    else:
-        raw_sample_doc = extracted_sample_doc = [{} for _ in range(10)]
-
-    raw_json = raw_sample_doc[doc_id]
-    extracted_json = extracted_sample_doc[doc_id]
-    return view_data(
-        raw_json,
-        extracted_json,
-        doc_id=doc_id,
-        data_source="DM Math",
-        data_sources="DM Math",
-        target=target,
-    )
-
 dmm_examples = DV2("data/curated_samples/dm_maths_raw.json", "data/curated_samples/dm_maths_extract.json", 3)
-
-def get_pg19_data(data_source: str = "PG19", doc_id: int = 3, target: str = "foo"):
-    doc_id = max(0, min(int(doc_id), 9))
-
-    if data_source == "PG19":
-        raw_sample_doc = extracted_sample_doc = json.load(
-            open("data/curated_samples/pg19_raw.json")
-        )
-    else:
-        raw_sample_doc = extracted_sample_doc = [{} for _ in range(10)]
-
-    raw_json = raw_sample_doc[doc_id]
-    extracted_json = extracted_sample_doc[doc_id]
-    return view_data(
-        raw_json,
-        extracted_json,
-        doc_id=doc_id,
-        data_source="PG19",
-        data_sources="PG19",
-        target=target,
-    )
-
 pg19_examples = DV("data/curated_samples/pg19_raw.json", 0, "PG19")
-
-def get_eu_data(data_source: str = "Europarl", doc_id: int = 3, target: str = "foo"):
-    doc_id = max(0, min(int(doc_id), 9))
-
-    if data_source == "Europarl":
-        raw_sample_doc = extracted_sample_doc = json.load(
-            open("data/curated_samples/europarl_raw.json")
-        )
-    else:
-        raw_sample_doc = extracted_sample_doc = [{} for _ in range(10)]
-
-    raw_json = raw_sample_doc[doc_id]
-    extracted_json = extracted_sample_doc[doc_id]
-    return view_data(
-        raw_json,
-        extracted_json,
-        doc_id=doc_id,
-        data_source="Europarl",
-        data_sources="Europarl",
-        target=target,
-    )
-
 eu_examples = DV("data/curated_samples/europarl_raw.json", 0, "Europarl")
+## end filtered examples
+
 
 filtering_process = Div(
     Section(