Tidy up interface

app.py

@@ -11,17 +11,20 @@ from langchain_text_splitters import TokenTextSplitter
 from process import Process
 from tempfile import NamedTemporaryFile
 from stqdm import stqdm
+from validate import Validation
 
 buffer = io.BytesIO()
 
 st.cache_data()
 st.set_page_config(page_title="NutriGenMe Paper Extractor")
-st.title("NutriGenMe - Paper Extraction")
-st.markdown("<div style='text-align: left; color: white; font-size: 16px'>In its latest version, the app is equipped to extract essential information from papers, including tables in both horizontal and vertical orientations, images, and text exclusively.</div><br>", unsafe_allow_html=True)
+st.title("NutriGenMe - Paper Extractor")
+st.markdown("<div style='text-align: justify;text-justify: inter-word;'>NutriGenMe Paper Extractor is a tool designed to extract relevant information from genomic papers related to the NutriGenMe project. It utilizes natural language processing techniques to parse through documents and extract key data points, enabling researchers and practitioners to efficiently gather insights from a large corpus of literature.</div>", unsafe_allow_html=True)
+st.divider()
 
-uploaded_files = st.file_uploader("Upload Paper(s) here :", type="pdf", accept_multiple_files=True)
-
-col1, col2, col3 = st.columns(3)
+st.markdown("<h4>Extraction</h4>", unsafe_allow_html=True)
+col1, col2 = st.columns(2)
+st.markdown("<h4>Validation</h4>", unsafe_allow_html=True)
+col3, col4 = st.columns(2)
 
 with col1:
     models = (
@@ -30,9 +33,7 @@ with col1:
         # 'llama-3-sonar-large-32k-chat',
         # 'mixtral-8x7b-instruct',
     )
-    model = st.selectbox(
-        'Model selection:', models, key='model'
-    )
+    model = st.selectbox('Model selection:', models, key='model')
 
 with col2:
     tokens = (
@@ -40,118 +41,127 @@ with col2:
         16000,
         24000
     )
-    chunk_option = st.selectbox(
-        'Token amounts per process:', tokens, key='token'
-    )
+    chunk_option = st.selectbox('Token amounts per process:', tokens, key='token')
     chunk_overlap = 0
 
 with col3:
     models_val = (
-        'gemini-1.5-pro-latest',
         'gpt-4-turbo',
+        'gemini-1.5-pro-latest',
         'mixtral-8x7b-instruct',
         # 'llama-3-sonar-large-32k-chat',
     )
-    model_val = st.selectbox(
-        'Model validator selection:', models_val, key='model_val'
-    )
+    model_val = st.selectbox('Model validator selection:', models_val, key='model_val')
+
+with col4:
+    api = st.toggle('Validate with API')
+
+if api:
+    st.warning("""This validation process leverage external application programming interfaces (APIs) from NCBI and EBI to verify information. 
+               These APIs may have limitations on their usage, so please exercise responsible use of this functionality. 
+               If you opt to employ API validation and the process takes a long time (more than 1 hour), consider refreshing the page and proceeding without API validation.""", icon="⚠️")
 
+st.divider()
+st.markdown("<h4>Process</h4>", unsafe_allow_html=True)
+uploaded_files = st.file_uploader("Upload Paper(s) here :", type="pdf", accept_multiple_files=True)
 
 if uploaded_files:
-    journals = []
-    parseButtonHV = st.button("Get Result", key='table_HV')
+    submit = st.button("Get Result", key='submit')
 
-    if parseButtonHV:
-        with st.status("Extraction in progress ...", expanded=True) as status:
-            start_time = datetime.now()
+if uploaded_files and submit:
 
-            for uploaded_file in stqdm(uploaded_files):
-                with NamedTemporaryFile(dir='.', suffix=".pdf", delete=eval(os.getenv('DELETE_TEMP_PDF', 'True'))) as pdf:
-                    pdf.write(uploaded_file.getbuffer())
-
-                    # Load Documents
-                    loader = PyPDFLoader(pdf.name)
-                    pages = loader.load()
-
-                    chunk_size = 120000
-                    chunk_overlap = 0
-                    docs = pages
-
-                    # Split Documents
-                    if chunk_option:
-                        docs = [Document('\n'.join([page.page_content for page in pages]))]
-                        docs[0].metadata = {'source': pages[0].metadata['source']}
-
-                        chunk_size = chunk_option
-                        chunk_overlap = int(0.25 * chunk_size)
-
-                    text_splitter = TokenTextSplitter.from_tiktoken_encoder(
-                        chunk_size=chunk_size, chunk_overlap=chunk_overlap
-                    )
-                    chunks = text_splitter.split_documents(docs)
-
-                    # Start extraction process in parallel
-                    process = Process(model, model_val)
-                    with ThreadPoolExecutor() as executor:
-                        result_gsd = executor.submit(process.get_entity, (chunks, 'gsd'))
-                        result_summ = executor.submit(process.get_entity, (chunks, 'summ'))
-                        result = executor.submit(process.get_entity, (chunks, 'all'))
-                        result_one = executor.submit(process.get_entity_one, [c.page_content for c in chunks[:1]])
-                        result_table = executor.submit(process.get_table, pdf.name)
-
-                        result_gsd = result_gsd.result()
-                        result_summ = result_summ.result()
-                        result = result.result()
-                        result_one = result_one.result()
-                        res_gene, res_snp, res_dis = result_table.result()
-
-                    # Combine Result
-                    result['Genes'] = res_gene + result_gsd['Genes']
-                    result['SNPs'] = res_snp + result_gsd['SNPs']
-                    result['Diseases'] = res_dis + result_gsd['Diseases']
-                    result['Conclusion'] = result_summ
-                    for k in result_one.keys():
-                        result[k] = result_one[k]
-
-                    if len(result['Genes']) == 0:
-                        result['Genes'] = ['']
-                    
-                    num_rows = max(max(len(result['Genes']), len(result['SNPs'])), len(result['Diseases']))
-
-                    # Adjust Genes, SNPs, Diseases
-                    for k in ['Genes', 'SNPs', 'Diseases']:
-                        while len(result[k]) < num_rows:
-                            result[k].append('')
-
-                        # Temporary handling
-                        result[k] = result[k][:num_rows]
-
-                    # Key Column
-                    result = {key: value if isinstance(value, list) else [value] * num_rows for key, value in result.items()}
-
-                    dataframe = pd.DataFrame(result)
-                    dataframe = dataframe[['Genes', 'SNPs', 'Diseases', 'Title', 'Authors', 'Publisher Name', 'Publication Year', 'Population', 'Sample Size', 'Study Methodology', 'Study Level', 'Conclusion']]
-                    dataframe = dataframe[dataframe['Genes'].astype(bool)].reset_index(drop=True)
-                    dataframe.drop_duplicates(['Genes', 'SNPs'], inplace=True)
-                    dataframe.reset_index(drop=True, inplace=True)
-                    
-                    # Validate Result
-                    df, df_no_llm, df_clean = process.validate(dataframe)
-
-                    end_time = datetime.now()
-                    st.write("Success in ", round((end_time.timestamp() - start_time.timestamp()) / 60, 2), "minutes")
-
-                    st.dataframe(df)
-                    with pd.ExcelWriter(buffer, engine='xlsxwriter') as writer:
+    with st.status("Extraction in progress ...", expanded=True) as status:
+        for uploaded_file in stqdm(uploaded_files):
+            start_time = datetime.now()
+            with NamedTemporaryFile(dir='.', suffix=".pdf", delete=eval(os.getenv('DELETE_TEMP_PDF', 'True'))) as pdf:
+
+                pdf.write(uploaded_file.getbuffer())
+                st.markdown(f"Start Extraction process at <code>{datetime.now().strftime('%H:%M')}</code>", unsafe_allow_html=True)
+
+                # Load Documents
+                loader = PyPDFLoader(pdf.name)
+                pages = loader.load()
+
+                chunk_size = 120000
+                chunk_overlap = 0
+                docs = pages
+
+                # Split Documents
+                if chunk_option:
+                    docs = [Document('\n'.join([page.page_content for page in pages]))]
+                    docs[0].metadata = {'source': pages[0].metadata['source']}
+
+                    chunk_size = chunk_option
+                    chunk_overlap = int(0.25 * chunk_size)
+
+                text_splitter = TokenTextSplitter.from_tiktoken_encoder(
+                    chunk_size=chunk_size, chunk_overlap=chunk_overlap
+                )
+                chunks = text_splitter.split_documents(docs)
+
+                # Start extraction process in parallel
+                process = Process(model)
+                with ThreadPoolExecutor() as executor:
+                    result_gsd = executor.submit(process.get_entity, (chunks, 'gsd'))
+                    result_summ = executor.submit(process.get_entity, (chunks, 'summ'))
+                    result = executor.submit(process.get_entity, (chunks, 'all'))
+                    result_one = executor.submit(process.get_entity_one, [c.page_content for c in chunks[:1]])
+                    result_table = executor.submit(process.get_table, pdf.name)
+
+                    result_gsd = result_gsd.result()
+                    result_summ = result_summ.result()
+                    result = result.result()
+                    result_one = result_one.result()
+                    res_gene, res_snp, res_dis = result_table.result()
+
+                # Combine Result
+                result['Genes'] = res_gene + result_gsd['Genes']
+                result['SNPs'] = res_snp + result_gsd['SNPs']
+                result['Diseases'] = res_dis + result_gsd['Diseases']
+                result['Conclusion'] = result_summ
+                for k in result_one.keys():
+                    result[k] = result_one[k]
+
+                if len(result['Genes']) == 0:
+                    result['Genes'] = ['']
+                
+                # Adjust Genes, SNPs, Diseases
+                num_rows = max(max(len(result['Genes']), len(result['SNPs'])), len(result['Diseases']))
+                for k in ['Genes', 'SNPs', 'Diseases']:
+                    while len(result[k]) < num_rows:
+                        result[k].append('')
+
+                    # Temporary handling
+                    result[k] = result[k][:num_rows]
+
+                # Arrange Column 
+                result = {key: value if isinstance(value, list) else [value] * num_rows for key, value in result.items()}
+                dataframe = pd.DataFrame(result)
+                dataframe = dataframe[['Genes', 'SNPs', 'Diseases', 'Title', 'Authors', 'Publisher Name', 'Publication Year', 'Population', 'Sample Size', 'Study Methodology', 'Study Level', 'Conclusion']]
+                dataframe = dataframe[dataframe['Genes'].astype(bool)].reset_index(drop=True)
+                dataframe.reset_index(drop=True, inplace=True)
+                
+                # Validate Result
+                st.markdown(f"Start Validation process at <code>{datetime.now().strftime('%H:%M')}</code>", unsafe_allow_html=True)
+                validation = Validation(model_val)
+                df, df_no_llm, df_clean = validation.validate(dataframe, api)
+                df.drop_duplicates(['Genes', 'SNPs'], inplace=True)
+                st.write("Success in ", round((datetime.now().timestamp() - start_time.timestamp()) / 60, 2), "minutes")
+
+                st.dataframe(df)
+                with pd.ExcelWriter(buffer, engine='xlsxwriter') as writer:
+                    if api:
                         df.to_excel(writer, sheet_name='Result Cleaned API LLM')
                         df_no_llm.to_excel(writer, sheet_name='Result Cleaned API')
-                        df_clean.to_excel(writer, sheet_name='Result Cleaned')
-                        dataframe.to_excel(writer, sheet_name='Original')
-                        writer.close()
-
-                    st.download_button(
-                        label="Save Result",
-                        data=buffer,
-                        file_name=f"{uploaded_file.name.replace('.pdf', '')}_{chunk_option}_{model.split('-')[0]}_{model_val.split('-')[0]}.xlsx",
-                        mime='application/vnd.ms-excel'
-                    )
+                    else:
+                        df.to_excel(writer, sheet_name='Result Cleaned LLM')
+                    df_clean.to_excel(writer, sheet_name='Result Cleaned')
+                    dataframe.to_excel(writer, sheet_name='Original')
+                    writer.close()
+
+                st.download_button(
+                    label="Save Result",
+                    data=buffer,
+                    file_name=f"{uploaded_file.name.replace('.pdf', '')}_{chunk_option}_{model.split('-')[0]}_{model_val.split('-')[0]}.xlsx",
+                    mime='application/vnd.ms-excel'
+                )

resources/experiment.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 1,
    "metadata": {},
    "outputs": [
     {
@@ -38,23 +38,23 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 50,
+   "execution_count": 24,
    "metadata": {},
    "outputs": [],
    "source": [
-    "image = Image('../NutriGenMe-Testing/monogenic-1.png')"
+    "image = Image('testing/ukmss-2.png')"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 51,
+   "execution_count": 25,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "2\n"
+      "1\n"
      ]
     },
     {
@@ -83,108 +83,196 @@
        "      <th>2</th>\n",
        "      <th>3</th>\n",
        "      <th>4</th>\n",
+       "      <th>5</th>\n",
+       "      <th>6</th>\n",
+       "      <th>7</th>\n",
+       "      <th>8</th>\n",
+       "      <th>9</th>\n",
+       "      <th>10</th>\n",
+       "      <th>11</th>\n",
+       "      <th>12</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
-       "      <td>None</td>\n",
-       "      <td>None</td>\n",
-       "      <td>Monogenic Diabetes or</td>\n",
-       "      <td>Associated With Common</td>\n",
-       "      <td>None</td>\n",
+       "      <td>SNP</td>\n",
+       "      <td>Chr.</td>\n",
+       "      <td>Position\\nB36\\n(bp)</td>\n",
+       "      <td>Nearby\\ngenea</td>\n",
+       "      <td>Risk\\nalleleb</td>\n",
+       "      <td>Gene (transcript)</td>\n",
+       "      <td>Tissue</td>\n",
+       "      <td>Effect (s.e.m.)C</td>\n",
+       "      <td>P value</td>\n",
+       "      <td>P d\\nadj</td>\n",
+       "      <td>SNP(2f</td>\n",
+       "      <td>Pvalue</td>\n",
+       "      <td>P g\\nadj</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
-       "      <td>Gene Name</td>\n",
-       "      <td>Major Function</td>\n",
-       "      <td>Syndromes</td>\n",
-       "      <td>T1D and/or T2D</td>\n",
-       "      <td>Refs.</td>\n",
+       "      <td>Novel loci reported in this study</td>\n",
+       "      <td>Novel loci reported in this study</td>\n",
+       "      <td>Novel loci reported in this study</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
-       "      <td>KCNJ11</td>\n",
-       "      <td>Encodes pore-forming inwardly-rectifying</td>\n",
-       "      <td>PNDM (most common cause)</td>\n",
-       "      <td>E23K</td>\n",
-       "      <td>42-46</td>\n",
+       "      <td>rs4457053</td>\n",
+       "      <td>5</td>\n",
+       "      <td>76,460,705</td>\n",
+       "      <td>ZBED3</td>\n",
+       "      <td>G</td>\n",
+       "      <td>PDE8B(NM 003719)</td>\n",
+       "      <td>Adipose</td>\n",
+       "      <td>0.302 (0.070)</td>\n",
+       "      <td>2.8 X 10-5</td>\n",
+       "      <td>0.80</td>\n",
+       "      <td>rs6864250 (0.18)</td>\n",
+       "      <td>3.1 X 10-17</td>\n",
+       "      <td>5.8 X 10-13</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3</th>\n",
        "      <td>None</td>\n",
-       "      <td>potassium channel subunits (Kir6.2)</td>\n",
-       "      <td>and TNDM, CHI, MODY</td>\n",
        "      <td>None</td>\n",
        "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>ZBED3(NM 032367)</td>\n",
+       "      <td>Adipose</td>\n",
+       "      <td>0.429 (0.068)</td>\n",
+       "      <td>1.0: x 10-9</td>\n",
+       "      <td>0.011</td>\n",
+       "      <td>rs4704389 (0.20)</td>\n",
+       "      <td>3.9 x 10-16</td>\n",
+       "      <td>6.0 X 10-9</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4</th>\n",
-       "      <td>ABCC8</td>\n",
-       "      <td>Encodes regulatory SUR1 subunits</td>\n",
-       "      <td>PNDM and TNDM, CHI, MODY</td>\n",
-       "      <td>A1369S, 1273AGA, R1420H</td>\n",
-       "      <td>46,47,52</td>\n",
+       "      <td>rs972283</td>\n",
+       "      <td>7</td>\n",
+       "      <td>130,117,394</td>\n",
+       "      <td>KLF14</td>\n",
+       "      <td>G</td>\n",
+       "      <td>KLF14(NM_138693)</td>\n",
+       "      <td>Adipose</td>\n",
+       "      <td>-0.387 (0.058)</td>\n",
+       "      <td>8.1 X 10-11</td>\n",
+       "      <td>0.058</td>\n",
+       "      <td>rs738134 (0.30)</td>\n",
+       "      <td>2.2 X 10-12</td>\n",
+       "      <td>0.0014</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>5</th>\n",
-       "      <td>GCK</td>\n",
-       "      <td>A key glucose-phosphoryating enzyme;</td>\n",
-       "      <td>GCK-MODY (MODY2), PNDM,</td>\n",
-       "      <td>rs1799884 (G/A), rs4607517 (A/G),</td>\n",
-       "      <td>75,78,79</td>\n",
+       "      <td>rs896854</td>\n",
+       "      <td>8</td>\n",
+       "      <td>96,029,687</td>\n",
+       "      <td>TP53INPI</td>\n",
+       "      <td>T</td>\n",
+       "      <td>CCNE2 (NM 057749)</td>\n",
+       "      <td>Blood</td>\n",
+       "      <td>0.225 (0.053)</td>\n",
+       "      <td>3.8 X 10-5</td>\n",
+       "      <td>0.78</td>\n",
+       "      <td>rs4735339 (0.61)</td>\n",
+       "      <td>5.8 X 10-7</td>\n",
+       "      <td>0.0051</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>6</th>\n",
-       "      <td>None</td>\n",
-       "      <td>a glucose sensor</td>\n",
-       "      <td>CHI</td>\n",
-       "      <td>3'UTR SNP, chr7:44184184-G/A</td>\n",
-       "      <td>None</td>\n",
+       "      <td>rs1552224</td>\n",
+       "      <td>11</td>\n",
+       "      <td>72,110,746</td>\n",
+       "      <td>CENTD2</td>\n",
+       "      <td>A</td>\n",
+       "      <td>STARDIO(NM 006645)</td>\n",
+       "      <td>Blood</td>\n",
+       "      <td>0.337 (0.066)</td>\n",
+       "      <td>8.6 x 10-7</td>\n",
+       "      <td>0.026</td>\n",
+       "      <td>rs519790 (0.04)</td>\n",
+       "      <td>2.7x 10-24</td>\n",
+       "      <td>1.6 X 10-1</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>7</th>\n",
-       "      <td>SLC2A2</td>\n",
-       "      <td>Encodes GLUT2, a high-capacity facilitative</td>\n",
-       "      <td>FBS</td>\n",
-       "      <td>SNPS rs5393 (AA) and rs5394</td>\n",
-       "      <td>93-100</td>\n",
+       "      <td>rs7957197</td>\n",
+       "      <td>12</td>\n",
+       "      <td>119,945,069</td>\n",
+       "      <td>HNFIA</td>\n",
+       "      <td>T</td>\n",
+       "      <td>ACADS (NM 000017)</td>\n",
+       "      <td>Adipose</td>\n",
+       "      <td>0.248 (0.067)</td>\n",
+       "      <td>3.7 x 10-4</td>\n",
+       "      <td>0.29</td>\n",
+       "      <td>rs9204\\n(0.02)</td>\n",
+       "      <td>1.3x 10-53</td>\n",
+       "      <td>5.9 X 10-50</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>8</th>\n",
        "      <td>None</td>\n",
-       "      <td>glucose transporter</td>\n",
        "      <td>None</td>\n",
-       "      <td>(CC) in the promoter region</td>\n",
        "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>PSMD9 (NM 002813)</td>\n",
+       "      <td>Blood</td>\n",
+       "      <td>0.240 (0.065)</td>\n",
+       "      <td>3.9 X 10-4</td>\n",
+       "      <td>0.0088</td>\n",
+       "      <td>rs3741593\\n(0.00)</td>\n",
+       "      <td>8.3x 10-8</td>\n",
+       "      <td>1.7 X 10-6</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>9</th>\n",
        "      <td>None</td>\n",
        "      <td>None</td>\n",
        "      <td>None</td>\n",
-       "      <td>and SNPS rs5400 (T1101) and</td>\n",
        "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>OASL (NM_003733)</td>\n",
+       "      <td>Adipose</td>\n",
+       "      <td>0.318 (0.068)</td>\n",
+       "      <td>6.4 X 10-6</td>\n",
+       "      <td>0.13</td>\n",
+       "      <td>rs2259883\\n(0.19)</td>\n",
+       "      <td>1.1x1 10-7</td>\n",
+       "      <td>0.0018</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "text/plain": [
-       "           0                                            1                         2                                  3         4\n",
-       "0       None                                         None     Monogenic Diabetes or             Associated With Common      None\n",
-       "1  Gene Name                               Major Function                 Syndromes                     T1D and/or T2D     Refs.\n",
-       "2     KCNJ11     Encodes pore-forming inwardly-rectifying  PNDM (most common cause)                               E23K     42-46\n",
-       "3       None          potassium channel subunits (Kir6.2)       and TNDM, CHI, MODY                               None      None\n",
-       "4      ABCC8             Encodes regulatory SUR1 subunits  PNDM and TNDM, CHI, MODY            A1369S, 1273AGA, R1420H  46,47,52\n",
-       "5        GCK         A key glucose-phosphoryating enzyme;   GCK-MODY (MODY2), PNDM,  rs1799884 (G/A), rs4607517 (A/G),  75,78,79\n",
-       "6       None                             a glucose sensor                       CHI       3'UTR SNP, chr7:44184184-G/A      None\n",
-       "7     SLC2A2  Encodes GLUT2, a high-capacity facilitative                       FBS        SNPS rs5393 (AA) and rs5394    93-100\n",
-       "8       None                          glucose transporter                      None        (CC) in the promoter region      None\n",
-       "9       None                                         None                      None        and SNPS rs5400 (T1101) and      None"
+       "                                  0                                  1                                  2              3              4                   5        6                 7            8         9                  10           11           12\n",
+       "0                                SNP                               Chr.                Position\\nB36\\n(bp)  Nearby\\ngenea  Risk\\nalleleb   Gene (transcript)   Tissue  Effect (s.e.m.)C      P value  P d\\nadj             SNP(2f       Pvalue     P g\\nadj\n",
+       "1  Novel loci reported in this study  Novel loci reported in this study  Novel loci reported in this study           None           None                None     None              None         None      None               None         None         None\n",
+       "2                          rs4457053                                  5                         76,460,705          ZBED3              G    PDE8B(NM 003719)  Adipose     0.302 (0.070)   2.8 X 10-5      0.80   rs6864250 (0.18)  3.1 X 10-17  5.8 X 10-13\n",
+       "3                               None                               None                               None           None           None    ZBED3(NM 032367)  Adipose     0.429 (0.068)  1.0: x 10-9     0.011   rs4704389 (0.20)  3.9 x 10-16   6.0 X 10-9\n",
+       "4                           rs972283                                  7                        130,117,394          KLF14              G    KLF14(NM_138693)  Adipose    -0.387 (0.058)  8.1 X 10-11     0.058    rs738134 (0.30)  2.2 X 10-12       0.0014\n",
+       "5                           rs896854                                  8                         96,029,687       TP53INPI              T   CCNE2 (NM 057749)    Blood     0.225 (0.053)   3.8 X 10-5      0.78   rs4735339 (0.61)   5.8 X 10-7       0.0051\n",
+       "6                          rs1552224                                 11                         72,110,746         CENTD2              A  STARDIO(NM 006645)    Blood     0.337 (0.066)   8.6 x 10-7     0.026    rs519790 (0.04)   2.7x 10-24   1.6 X 10-1\n",
+       "7                          rs7957197                                 12                        119,945,069          HNFIA              T   ACADS (NM 000017)  Adipose     0.248 (0.067)   3.7 x 10-4      0.29     rs9204\\n(0.02)   1.3x 10-53  5.9 X 10-50\n",
+       "8                               None                               None                               None           None           None   PSMD9 (NM 002813)    Blood     0.240 (0.065)   3.9 X 10-4    0.0088  rs3741593\\n(0.00)    8.3x 10-8   1.7 X 10-6\n",
+       "9                               None                               None                               None           None           None    OASL (NM_003733)  Adipose     0.318 (0.068)   6.4 X 10-6      0.13  rs2259883\\n(0.19)   1.1x1 10-7       0.0018"
       ]
      },
-     "execution_count": 51,
+     "execution_count": 25,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -204,30 +292,15 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 52,
+   "execution_count": 8,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "                0                                                  1                                                  2                                                  3                                4\n",
-      "0                                                                                                 Monogenic Diabetes or                             Associated With Common                                 \n",
-      "1       Gene Name                                     Major Function                                          Syndromes                                     T1D and/or T2D                            Refs.\n",
-      "2          KCNJ11  Encodes pore-forming inwardly-rectifying potas...       PNDM (most common cause) and TNDM, CHI, MODY                                               E23K                            42-46\n",
-      "3           ABCC8                   Encodes regulatory SUR1 subunits                           PNDM and TNDM, CHI, MODY                            A1369S, 1273AGA, R1420H                         46,47,52\n",
-      "4             GCK  A key glucose-phosphoryating enzyme; a glucose...                        GCK-MODY (MODY2), PNDM, CHI  rs1799884 (G/A), rs4607517 (A/G), 3'UTR SNP, c...                         75,78,79\n",
-      "5          SLC2A2  Encodes GLUT2, a high-capacity facilitative gl...                                                FBS  SNPS rs5393 (AA) and rs5394 (CC) in the promot...                           93-100\n",
-      "6      HNF1A/TCF1  TF; regulator of pancreatic B-cell differentia...  HNF1A-MODY (MODY3), most common cause of MODY,...                         G319S, C.1522G>A (p.E508K)                    114, 118, 119\n",
-      "7           HNF4A                 Key TF for early fetal development                            HNF4A MODY (MODY1), CHI  SNPS rs2144908, rs3818247 and rs884614, rs4810...                     121-124, 274\n",
-      "8      HNF1B/TCF2  TF; required for the generation of pancreatic ...      RCAD syndrome, or MODY5; TNDM and PNDM (rare)       SNP rs757210 A, TS4430796 A, and TS7501939 C                         141, 144\n",
-      "9            PDX1  TF; required for pancreas development, B-cell ...                                        PNDM, MODY4  C18R, Q59L, D76N, R197H, G212R, P239Q, InsCCG2...                     163-165, 167\n",
-      "10           PAX4  Islet TF that functions mainly as a transcript...                                              MODY9                   R121W, R133W, R37W, rs10229583 G                    180, 181, 187\n",
-      "11  NEUROD1/BETA2  TF; required for the development of the endocr...                                     MODY6 and PNDM  R111L and 206 + C; A45T variant at rs1801262 (...                          204-208\n",
-      "12           WFS1  A transmembrane protein; a negative regulator ...             WFS1, sometimes referred to as DIDMOAD  R456 and H611, SNPS at rs10010131, rs6446482; ...                          223-225\n",
-      "13          PPARG  TF; master regulator of adipogenesis, energy b...  Monogenic diabetes   Monogenic Diabetes Genes ...  Pro12Ala variant (rs1801282), SNP at rs4684847...                     240-243, 250\n",
-      "14            INS               Predominant glucose-lowering hormone         PNDM (2nd most common cause), TNDM, MODY10  Class I alleles of INS VNTR associated with T1...                273, 274, 276-281\n",
-      "15          GLIS3  TF; regulator of islet development, insulin ge...  Neonatal diabetes syndrome associated with con...  rs7020673 G associated with T1D; rs7034200 A a...  78, 214, 289, 291, 292, 295-308\n"
+      "                                                  0                                1                                                  2                                                  3                                                  4                                                  5                                                  6                                                  7                                                  8                                                  9                                                  10                                                 11                                                 12                                                 13                                                 14                                                 15                                                 16          17                              18\n",
+      "0  SNP rs584438 IGFBP4 rs6662509 H6PD rs2362965 R...  Gene C T T C  A C  T A T T  A A  Effect Other allele allele A C A T  G T  C G C...  OR 0.98 1.00 0.95 1.03  1.09 1.08  1.27 1.09 1...  BMI tails   0.52 0.95 0.02 0.33  0.0001 0.0001...  P 1.02 1.11 0.97 1.06  1.11 1.125 5.41 X 10-5 ...  OR 0.64 0.07 0.25 0.11  0.0006 1.125 5.41 X 10...  Obesity class III  P 1.01 1.01 0.98 1.01  1.10...  OR 0.47 0.83 0.20 0.58  1.10 1.06 X 10-8 1.125...  Obesity class II  P 1.00 0.99 0.99 1.00  1.04 ...  OR 0.75 1.00 0.34 0.99 0.37 0.99 0.82 1.01  9....  Obesity class I  P 0.59 0.005 0.35 -0.006 0.21...  Overweight   OR 0.22 0.27 0.05 0.33  8.80 X 10...  Overweight class  P 1.18 1.23 1.12 1.15  1.00 ...  BMI (continuous)a  Effect P 5.22 X 10-12 3.19 ...  OR 0.025 9.43 X 10-11 0.031 7.76 X 10-12 0.017...  Height tails   0.025 9.43 X 10-11 0.031 7.76 X...  Height   P  Height (continuous)a  Effect P\n"
      ]
     }
    ],
@@ -254,6 +327,440 @@
     "print(dfc)"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>0</th>\n",
+       "      <th>1</th>\n",
+       "      <th>2</th>\n",
+       "      <th>3</th>\n",
+       "      <th>4</th>\n",
+       "      <th>5</th>\n",
+       "      <th>6</th>\n",
+       "      <th>7</th>\n",
+       "      <th>8</th>\n",
+       "      <th>9</th>\n",
+       "      <th>10</th>\n",
+       "      <th>11</th>\n",
+       "      <th>12</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>SNP</td>\n",
+       "      <td>Chr.</td>\n",
+       "      <td>Position\\nB36\\n(bp)</td>\n",
+       "      <td>Nearby\\ngenea</td>\n",
+       "      <td>Risk\\nalleleb</td>\n",
+       "      <td>Gene (transcript)</td>\n",
+       "      <td>Tissue</td>\n",
+       "      <td>Effect (s.e.m.)C</td>\n",
+       "      <td>P value</td>\n",
+       "      <td>P d\\nadj</td>\n",
+       "      <td>SNP(2f</td>\n",
+       "      <td>Pvalue</td>\n",
+       "      <td>P g\\nadj</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>Novel loci reported in this study</td>\n",
+       "      <td>Novel loci reported in this study</td>\n",
+       "      <td>Novel loci reported in this study</td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>rs4457053</td>\n",
+       "      <td>5</td>\n",
+       "      <td>76,460,705</td>\n",
+       "      <td>ZBED3</td>\n",
+       "      <td>G</td>\n",
+       "      <td>PDE8B(NM 003719)</td>\n",
+       "      <td>Adipose</td>\n",
+       "      <td>0.302 (0.070)</td>\n",
+       "      <td>2.8 X 10-5</td>\n",
+       "      <td>0.80</td>\n",
+       "      <td>rs6864250 (0.18)</td>\n",
+       "      <td>3.1 X 10-17</td>\n",
+       "      <td>5.8 X 10-13</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td>ZBED3(NM 032367)</td>\n",
+       "      <td>Adipose</td>\n",
+       "      <td>0.429 (0.068)</td>\n",
+       "      <td>1.0: x 10-9</td>\n",
+       "      <td>0.011</td>\n",
+       "      <td>rs4704389 (0.20)</td>\n",
+       "      <td>3.9 x 10-16</td>\n",
+       "      <td>6.0 X 10-9</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>rs972283</td>\n",
+       "      <td>7</td>\n",
+       "      <td>130,117,394</td>\n",
+       "      <td>KLF14</td>\n",
+       "      <td>G</td>\n",
+       "      <td>KLF14(NM_138693)</td>\n",
+       "      <td>Adipose</td>\n",
+       "      <td>-0.387 (0.058)</td>\n",
+       "      <td>8.1 X 10-11</td>\n",
+       "      <td>0.058</td>\n",
+       "      <td>rs738134 (0.30)</td>\n",
+       "      <td>2.2 X 10-12</td>\n",
+       "      <td>0.0014</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5</th>\n",
+       "      <td>rs896854</td>\n",
+       "      <td>8</td>\n",
+       "      <td>96,029,687</td>\n",
+       "      <td>TP53INPI</td>\n",
+       "      <td>T</td>\n",
+       "      <td>CCNE2 (NM 057749)</td>\n",
+       "      <td>Blood</td>\n",
+       "      <td>0.225 (0.053)</td>\n",
+       "      <td>3.8 X 10-5</td>\n",
+       "      <td>0.78</td>\n",
+       "      <td>rs4735339 (0.61)</td>\n",
+       "      <td>5.8 X 10-7</td>\n",
+       "      <td>0.0051</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>6</th>\n",
+       "      <td>rs1552224</td>\n",
+       "      <td>11</td>\n",
+       "      <td>72,110,746</td>\n",
+       "      <td>CENTD2</td>\n",
+       "      <td>A</td>\n",
+       "      <td>STARDIO(NM 006645)</td>\n",
+       "      <td>Blood</td>\n",
+       "      <td>0.337 (0.066)</td>\n",
+       "      <td>8.6 x 10-7</td>\n",
+       "      <td>0.026</td>\n",
+       "      <td>rs519790 (0.04)</td>\n",
+       "      <td>2.7x 10-24</td>\n",
+       "      <td>1.6 X 10-1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>7</th>\n",
+       "      <td>rs7957197</td>\n",
+       "      <td>12</td>\n",
+       "      <td>119,945,069</td>\n",
+       "      <td>HNFIA</td>\n",
+       "      <td>T</td>\n",
+       "      <td>ACADS (NM 000017)</td>\n",
+       "      <td>Adipose</td>\n",
+       "      <td>0.248 (0.067)</td>\n",
+       "      <td>3.7 x 10-4</td>\n",
+       "      <td>0.29</td>\n",
+       "      <td>rs9204\\n(0.02)</td>\n",
+       "      <td>1.3x 10-53</td>\n",
+       "      <td>5.9 X 10-50</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8</th>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td>PSMD9 (NM 002813)</td>\n",
+       "      <td>Blood</td>\n",
+       "      <td>0.240 (0.065)</td>\n",
+       "      <td>3.9 X 10-4</td>\n",
+       "      <td>0.0088</td>\n",
+       "      <td>rs3741593\\n(0.00)</td>\n",
+       "      <td>8.3x 10-8</td>\n",
+       "      <td>1.7 X 10-6</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>9</th>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td>OASL (NM_003733)</td>\n",
+       "      <td>Adipose</td>\n",
+       "      <td>0.318 (0.068)</td>\n",
+       "      <td>6.4 X 10-6</td>\n",
+       "      <td>0.13</td>\n",
+       "      <td>rs2259883\\n(0.19)</td>\n",
+       "      <td>1.1x1 10-7</td>\n",
+       "      <td>0.0018</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>10</th>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td>OASL (NM_ _003733)</td>\n",
+       "      <td>Blood</td>\n",
+       "      <td>0.319 (0.064)</td>\n",
+       "      <td>1.3 X 10-6</td>\n",
+       "      <td>0.37</td>\n",
+       "      <td>rs4556628\\n(0.21)</td>\n",
+       "      <td>4.4&gt; X 10-22</td>\n",
+       "      <td>1.4 X 10-16</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>11</th>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td>COQ5(NM_032314)</td>\n",
+       "      <td>Blood</td>\n",
+       "      <td>0.248 (0.065)</td>\n",
+       "      <td>2.1 x1 10-4</td>\n",
+       "      <td>0.92</td>\n",
+       "      <td>rs10774561\\n(0.02)</td>\n",
+       "      <td>8.7x 10-39</td>\n",
+       "      <td>4.9 X 10 -</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>12</th>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td>UNCI19B(NM 032661)</td>\n",
+       "      <td>Blood</td>\n",
+       "      <td>0.254 (0.064)</td>\n",
+       "      <td>1.4x 10-4</td>\n",
+       "      <td>0.048</td>\n",
+       "      <td>rs11065202\\n(0.09)</td>\n",
+       "      <td>7.8 x 10-12</td>\n",
+       "      <td>2.3 X 10-9</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>13</th>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td>CAMKK2 (NM 17 72215)</td>\n",
+       "      <td>Adipose</td>\n",
+       "      <td>0.497 (0.068)</td>\n",
+       "      <td>1.2 x 10-12</td>\n",
+       "      <td>0.18</td>\n",
+       "      <td>rs11065504\\n(0.08)</td>\n",
+       "      <td>2.7x 10-117</td>\n",
+       "      <td>3.8 X 10-98</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>14</th>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td>CAMKK2 (NM_ 1 172215)</td>\n",
+       "      <td>Blood</td>\n",
+       "      <td>0.360 (0.063)</td>\n",
+       "      <td>3.4 X 10-8</td>\n",
+       "      <td>0.68</td>\n",
+       "      <td>rs11065504\\n(0.08)</td>\n",
+       "      <td>7.0 X 10-105</td>\n",
+       "      <td>5.7 X 10-94</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>15</th>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td>P2RX4(NM 175568)</td>\n",
+       "      <td>Blood</td>\n",
+       "      <td>0.312 (0.065)</td>\n",
+       "      <td>3.4 x 10-6</td>\n",
+       "      <td>2.0 x 10-6</td>\n",
+       "      <td>rs25644\\n(0.03)</td>\n",
+       "      <td>3.4 x 10-17</td>\n",
+       "      <td>1.9 x 10-17</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>16</th>\n",
+       "      <td>rs8042680</td>\n",
+       "      <td>15</td>\n",
+       "      <td>89,322,341</td>\n",
+       "      <td>PRCI</td>\n",
+       "      <td>A</td>\n",
+       "      <td>VPS33B (NM_018668)</td>\n",
+       "      <td>Blood</td>\n",
+       "      <td>0.371 (0.057)</td>\n",
+       "      <td>2.9 x 10-10</td>\n",
+       "      <td>0.50</td>\n",
+       "      <td>rs12595616\\n(0.57)</td>\n",
+       "      <td>2.3 x 10-21</td>\n",
+       "      <td>4.5 X 10-1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>17</th>\n",
+       "      <td>Previously reported loci</td>\n",
+       "      <td>Previously reported loci</td>\n",
+       "      <td>Previously reported loci</td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>18</th>\n",
+       "      <td>rs7578326</td>\n",
+       "      <td>2</td>\n",
+       "      <td>226,728,897</td>\n",
+       "      <td>IRSI</td>\n",
+       "      <td>A</td>\n",
+       "      <td>IRS/(Contig50189RC)</td>\n",
+       "      <td>Adipose</td>\n",
+       "      <td>-0.251 (0.059)</td>\n",
+       "      <td>3.7 x 10-5</td>\n",
+       "      <td>0.89</td>\n",
+       "      <td>rs2943653 (0.93)</td>\n",
+       "      <td>3.4 X 10-5</td>\n",
+       "      <td>0.69</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>19</th>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td>IRSI(NM 005544)</td>\n",
+       "      <td>Adipose</td>\n",
+       "      <td>0.331 (0.059)</td>\n",
+       "      <td>5.7 X 10-8</td>\n",
+       "      <td>0.58</td>\n",
+       "      <td>rs2176040 (0.74)</td>\n",
+       "      <td>7.8 X 10-10</td>\n",
+       "      <td>0.0042</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>20</th>\n",
+       "      <td>rs13081389</td>\n",
+       "      <td>3</td>\n",
+       "      <td>12,264,800</td>\n",
+       "      <td>PPARG</td>\n",
+       "      <td>A</td>\n",
+       "      <td>IQSECI (NM 014869)</td>\n",
+       "      <td>Adipose</td>\n",
+       "      <td>-0.630(0.131)</td>\n",
+       "      <td>2.9 x 10-6</td>\n",
+       "      <td>1.4&gt; x 10-4</td>\n",
+       "      <td>rs9211\\n(0.01)</td>\n",
+       "      <td>1.1x 10-96</td>\n",
+       "      <td>7.4 X 10-94</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>21</th>\n",
+       "      <td>rs6795735</td>\n",
+       "      <td>3</td>\n",
+       "      <td>64,680,405</td>\n",
+       "      <td>ADAMTS9</td>\n",
+       "      <td>C</td>\n",
+       "      <td>BC040632(AK022320)</td>\n",
+       "      <td>Adipose</td>\n",
+       "      <td>0.229 (0.056)</td>\n",
+       "      <td>7.6 X 10-5</td>\n",
+       "      <td>0.28</td>\n",
+       "      <td>rs4521216\\n(0.02)</td>\n",
+       "      <td>3.0 X 10-13</td>\n",
+       "      <td>8.7 x 10-10</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                   0                                  1                                  2              3              4                      5        6                 7            8            9                   10            11           12\n",
+       "0                                 SNP                               Chr.                Position\\nB36\\n(bp)  Nearby\\ngenea  Risk\\nalleleb      Gene (transcript)   Tissue  Effect (s.e.m.)C      P value     P d\\nadj              SNP(2f        Pvalue     P g\\nadj\n",
+       "1   Novel loci reported in this study  Novel loci reported in this study  Novel loci reported in this study                                                                                                                                                         \n",
+       "2                           rs4457053                                  5                         76,460,705          ZBED3              G       PDE8B(NM 003719)  Adipose     0.302 (0.070)   2.8 X 10-5         0.80    rs6864250 (0.18)   3.1 X 10-17  5.8 X 10-13\n",
+       "3                                                                                                                                               ZBED3(NM 032367)  Adipose     0.429 (0.068)  1.0: x 10-9        0.011    rs4704389 (0.20)   3.9 x 10-16   6.0 X 10-9\n",
+       "4                            rs972283                                  7                        130,117,394          KLF14              G       KLF14(NM_138693)  Adipose    -0.387 (0.058)  8.1 X 10-11        0.058     rs738134 (0.30)   2.2 X 10-12       0.0014\n",
+       "5                            rs896854                                  8                         96,029,687       TP53INPI              T      CCNE2 (NM 057749)    Blood     0.225 (0.053)   3.8 X 10-5         0.78    rs4735339 (0.61)    5.8 X 10-7       0.0051\n",
+       "6                           rs1552224                                 11                         72,110,746         CENTD2              A     STARDIO(NM 006645)    Blood     0.337 (0.066)   8.6 x 10-7        0.026     rs519790 (0.04)    2.7x 10-24   1.6 X 10-1\n",
+       "7                           rs7957197                                 12                        119,945,069          HNFIA              T      ACADS (NM 000017)  Adipose     0.248 (0.067)   3.7 x 10-4         0.29      rs9204\\n(0.02)    1.3x 10-53  5.9 X 10-50\n",
+       "8                                                                                                                                              PSMD9 (NM 002813)    Blood     0.240 (0.065)   3.9 X 10-4       0.0088   rs3741593\\n(0.00)     8.3x 10-8   1.7 X 10-6\n",
+       "9                                                                                                                                               OASL (NM_003733)  Adipose     0.318 (0.068)   6.4 X 10-6         0.13   rs2259883\\n(0.19)    1.1x1 10-7       0.0018\n",
+       "10                                                                                                                                            OASL (NM_ _003733)    Blood     0.319 (0.064)   1.3 X 10-6         0.37   rs4556628\\n(0.21)  4.4> X 10-22  1.4 X 10-16\n",
+       "11                                                                                                                                               COQ5(NM_032314)    Blood     0.248 (0.065)  2.1 x1 10-4         0.92  rs10774561\\n(0.02)    8.7x 10-39   4.9 X 10 -\n",
+       "12                                                                                                                                            UNCI19B(NM 032661)    Blood     0.254 (0.064)    1.4x 10-4        0.048  rs11065202\\n(0.09)   7.8 x 10-12   2.3 X 10-9\n",
+       "13                                                                                                                                          CAMKK2 (NM 17 72215)  Adipose     0.497 (0.068)  1.2 x 10-12         0.18  rs11065504\\n(0.08)   2.7x 10-117  3.8 X 10-98\n",
+       "14                                                                                                                                         CAMKK2 (NM_ 1 172215)    Blood     0.360 (0.063)   3.4 X 10-8         0.68  rs11065504\\n(0.08)  7.0 X 10-105  5.7 X 10-94\n",
+       "15                                                                                                                                              P2RX4(NM 175568)    Blood     0.312 (0.065)   3.4 x 10-6   2.0 x 10-6     rs25644\\n(0.03)   3.4 x 10-17  1.9 x 10-17\n",
+       "16                          rs8042680                                 15                         89,322,341           PRCI              A     VPS33B (NM_018668)    Blood     0.371 (0.057)  2.9 x 10-10         0.50  rs12595616\\n(0.57)   2.3 x 10-21   4.5 X 10-1\n",
+       "17           Previously reported loci           Previously reported loci           Previously reported loci                                                                                                                                                         \n",
+       "18                          rs7578326                                  2                        226,728,897           IRSI              A    IRS/(Contig50189RC)  Adipose    -0.251 (0.059)   3.7 x 10-5         0.89    rs2943653 (0.93)    3.4 X 10-5         0.69\n",
+       "19                                                                                                                                               IRSI(NM 005544)  Adipose     0.331 (0.059)   5.7 X 10-8         0.58    rs2176040 (0.74)   7.8 X 10-10       0.0042\n",
+       "20                         rs13081389                                  3                         12,264,800          PPARG              A     IQSECI (NM 014869)  Adipose     -0.630(0.131)   2.9 x 10-6  1.4> x 10-4      rs9211\\n(0.01)    1.1x 10-96  7.4 X 10-94\n",
+       "21                          rs6795735                                  3                         64,680,405        ADAMTS9              C     BC040632(AK022320)  Adipose     0.229 (0.056)   7.6 X 10-5         0.28   rs4521216\\n(0.02)   3.0 X 10-13  8.7 x 10-10"
+      ]
+     },
+     "execution_count": 26,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "dfc = df.fillna('')\n",
+    "dfc"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -263,14 +770,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": 19,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "118 [('rs5393', 'GLUT2'), ('rs5404', 'SNPS'), ('rs757210', 'HNF1B'), ('rs884614', 'SNPS'), ('rs2144908', 'MODY'), ('rs2144908', 'CHI'), ('rs4684847', 'T1D'), ('rs1884613', 'MODY'), ('rs1884613', 'CHI'), ('rs5393', 'SNPS'), ('rs734312', 'SNPS'), ('rs5394', 'GLUT2'), ('rs757210', 'TS4430796'), ('rs7041847', 'T1D'), ('rs6446482', 'SNPS'), ('rs7020673', 'GLIS3'), ('rs4684847', 'TZDS'), ('rs757210', 'PNDM'), ('rs5400', 'GLUT2'), ('rs7020673', 'T2D'), ('rs3818247', 'HNF4A'), ('rs4810424', 'MODY'), ('rs4810424', 'CHI'), ('rs10229583', 'R133W'), ('rs1801262', 'R111L'), ('rs1801262', 'BETA2'), ('rs10010131', 'SNPS'), ('rs10229583', 'MODY9'), ('rs5400', 'SNPS'), ('rs1801282', 'T1D'), ('rs2144908', 'HNF4A'), ('rs5393', 'FBS'), ('rs757210', 'RCAD'), ('rs10229583', 'R121W'), ('rs1801262', 'INS'), ('rs10010131', 'R456'), ('rs4684847', 'SNP'), ('rs7034200', 'T2D'), ('rs5404', 'T1101'), ('rs4607517', 'MODY'), ('rs1799884', 'MODY'), ('rs1799884', 'CHI'), ('rs4607517', 'PNDM'), ('rs6446482', 'WFS1'), ('rs1799884', 'PNDM'), ('rs5404', 'SLC2A2'), ('rs1801282', 'TZDS'), ('rs5404', 'T198T'), ('rs884614', 'MODY1'), ('rs734312', 'DIDMOAD'), ('rs5394', 'FBS'), ('rs4810424', 'HNF4A'), ('rs7020673', 'T1D'), ('rs757210', 'TCF2'), ('rs5393', 'T1101'), ('rs6446482', 'DIDMOAD'), ('rs1801262', 'A45T'), ('rs5394', 'SNPS'), ('rs5393', 'SLC2A2'), ('rs884614', 'CHI'), ('rs884614', 'MODY'), ('rs5393', 'T198T'), ('rs5400', 'FBS'), ('rs3818247', 'SNPS'), ('rs757210', 'SNP'), ('rs10229583', 'R37W'), ('rs10229583', 'PAX4'), ('rs4684847', 'T2D'), ('rs1801282', 'SNP'), ('rs7034200', 'GLIS3'), ('rs1884613', 'HNF4A'), ('rs4607517', 'GCK'), ('rs757210', 'TS7501939'), ('rs1799884', 'GCK'), ('rs10010131', 'DIDMOAD'), ('rs734312', 'WFS1'), ('rs2144908', 'SNPS'), ('rs5394', 'T198T'), ('rs4684847', 'PPARG'), ('rs734312', 'H611'), ('rs1801262', 'MODY6'), ('rs4607517', 'CHI'), ('rs7041847', 'T2D'), ('rs5404', 'GLUT2'), ('rs5400', 'T1101'), ('rs4607517', 'UTR'), ('rs1799884', 'UTR'), ('rs5400', 'SLC2A2'), ('rs6446482', 'H611'), ('rs5400', 'T198T'), ('rs1799884', 'SNP'), ('rs884614', 'HNF4A'), ('rs4810424', 'SNPS'), ('rs10010131', 'WFS1'), ('rs1801282', 'T2D'), ('rs10010131', 'H611'), ('rs1801262', 'PNDM'), ('rs4607517', 'SNP'), ('rs5394', 'T1101'), ('rs757210', 'TNDM'), ('rs4810424', 'MODY1'), ('rs1801282', 'PPARG'), ('rs7034200', 'T1D'), ('rs7041847', 'GLIS3'), ('rs4607517', 'MODY2'), ('rs5394', 'SLC2A2'), ('rs3818247', 'MODY1'), ('rs1799884', 'MODY2'), ('rs1884613', 'SNPS'), ('rs757210', 'MODY5'), ('rs734312', 'R456'), ('rs3818247', 'MODY'), ('rs3818247', 'CHI'), ('rs6446482', 'R456'), ('rs5404', 'FBS'), ('rs1801262', 'NEUROD1'), ('rs2144908', 'MODY1'), ('rs1884613', 'MODY1')]\n"
+      "41 [('rs4607517', 'MODY2'), ('rs5400', 'T1101'), ('rs1799884', 'PNDM'), ('rs4607517', 'MODY'), ('rs5400', 'SNPS'), ('rs5394', 'SNPS'), ('rs2144908', 'MODY'), ('rs7020673', 'GLIS3'), ('rs5393', 'SLC2A2'), ('rs4684847', 'SNP'), ('rs5394', 'GLUT2'), ('rs7034200', 'T1D'), ('rs4607517', 'GCK'), ('rs3818247', 'SNPS'), ('rs2144908', 'CHI'), ('rs2144908', 'MODY1'), ('rs3818247', 'HNF4A'), ('rs6446482', 'DIDMOAD'), ('rs757210', 'MODY5'), ('rs1799884', 'MODY2'), ('rs757210', 'HNF1B'), ('rs5393', 'FBS'), ('rs757210', 'RCAD'), ('rs1799884', 'MODY'), ('rs757210', 'TS4430796'), ('rs5394', 'SLC2A2'), ('rs5404', 'T198T'), ('rs1799884', 'GCK'), ('rs4607517', 'PNDM'), ('rs3818247', 'MODY'), ('rs2144908', 'SNPS'), ('rs10010131', 'DIDMOAD'), ('rs5393', 'SNPS'), ('rs2144908', 'HNF4A'), ('rs5394', 'FBS'), ('rs3818247', 'CHI'), ('rs5393', 'GLUT2'), ('rs757210', 'SNP'), ('rs3818247', 'MODY1'), ('rs757210', 'TCF2'), ('rs1801282', 'PPARG')]\n"
      ]
     }
    ],