Update overview.py

overview.py

@@ -209,6 +209,47 @@ dataset_sources = pd.DataFrame(
 table_html = dataset_sources.to_html(index=False, border=0)
 table_div1 = Div(NotStr(table_html), style="margin: 40px;")
 
+
+def get_curated_chart():
+        # Dataset
+    data = {
+      'Source': ['ArXiv', 'PubMed Central', 'PubMed Abstract', 'S2ORC Full Text', 'S2ORC Abstract', 'PhilPapers', 'Wikipedia', 'StackExchange', 'EuroParl', 'Ubuntu IRC', 'Freelaw', 'PG19', 'USPTO', 'HackerNews', 'DM Maths'],
+      'Category': ['Papers', 'Papers', 'Papers', 'Papers', 'Papers', 'Papers', 'Internet', 'Conversational', 'Legal/Formal', 'Conversational', 'Legal/Formal', 'Books', 'Legal/Formal', 'Conversational', 'Reasoning'],
+      'Count': [100, 200, 150, 120, 80, 90, 300, 250, 180, 150, 150, 250, 180, 120, 90],
+      'Details': [
+        'A repository of scientific papers in various disciplines, including computer science, physics, mathematics, and more.',
+        'A database of biomedical and life sciences research articles.',
+        'Abstracts of biomedical literature from various sources.',
+        'Full-text articles from the Semantic Scholar Open Research Corpus.',
+        'Abstracts of articles from the Semantic Scholar Open Research Corpus.',
+        'Papers from the PhilPapers database, a comprehensive index and bibliography of philosophy research.',
+        'A collaborative online encyclopedia that covers a wide range of topics.',
+        'A network of question-and-answer websites on various subjects, including programming, science, mathematics, and more.',
+        'A collection of multilingual parallel corpora of parliamentary debates from the European Parliament.',
+        'Chat logs from the Ubuntu Internet Relay Chat (IRC) channels.',
+        'Legal documents and court cases from various jurisdictions.',
+        'A collection of books from Project Gutenberg, a digital library of public domain works.',
+        'Patent documents from the United States Patent and Trademark Office.',
+        'User-generated news and discussion platform focused on technology and startups.',
+        'Deep Mind Maths dataset with generated questions.'
+      ]
+    }
+    # Calculate percentage for each data source
+    total_count = sum(data['Count'])
+    data['Percentage'] = [count / total_count * 100 for count in data['Count']]
+    
+    # Create treemap
+    fig = px.treemap(data, path=['Category', 'Source'], values='Count', hover_data=['Details', 'Percentage'], hover_name='Source')
+    
+    # Set the size of the chart
+    fig.update_layout(width=800, height=600)
+    
+    # Display treemap
+    st.plotly_chart(fig)
+
+
+
+
 quality_text = P("""The quality and size of a pre-training dataset play a crucial role in the performance of large language models (LLMs). 
 The community has introduced a variety of datasets for this purpose, including purely web-based datasets like RefinedWeb{citation_obj.display_citation("refinedweb")}, RedPajama-Data-V2{citation_obj.display_citation("redpajama-v2")}, DCLM{citation_obj.display_citation("dclm")}, and FineWeb{citation_obj.display_citation("fineweb")}, 
 as well as comprehensive datasets derived from multiple highly-curated data sources such as The Pile{citation_obj.display_citation("thepile")}, RedPajama-Data-V1{citation_obj.display_citation("redpajama-v1")}, and Dolma{citation_obj.display_citation("dolma")}. 
@@ -239,6 +280,7 @@ def overview():
             table_div,
             P("Table 2: Basic TxT360 Statistics."),
             table_div1,
+            plotly2fasthtml(get_curated_chart()),
             id="inner-text",
         )
     )