gegegege

.gitignore

@@ -0,0 +1,135 @@
+# Add any directories, files, or patterns you don't want to be tracked by version control
+
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+#*.py[cod]
+#*$py.class
+#*.txt
+#*.tsv
+#*.csv
+*.xlsx
+*.pdf
+*.nii
+#*.nii.gz
+*.DS_Store
+#*.png
+#*.pyn
+*.jpg
+*.nii.gz
+*.pkl
+*-checkpoint.ipynb
+*.pkls
+*.pth
+*.yaml
+*.ckpt
+# C extensions
+#*.so
+
+# Distribution / packaging
+#.Python
+#build/
+#develop-eggs/
+#dist/
+#downloads/
+#eggs/
+#.eggs/
+#lib/
+#lib64/
+#parts/
+#sdist/
+#var/
+#wheels/
+#*.egg-info/
+#.installed.cfg
+#*.egg
+#MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+#*.manifest
+#*.spec
+
+# Installer logs
+#pip-log.txt
+#pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+#htmlcov/
+#.tox/
+#.coverage
+#.coverage.*
+#.cache
+#nosetests.xml
+#coverage.xml
+#*.cover
+#.hypothesis/
+#.pytest_cache/
+
+# Translations
+#*.mo
+#*.pot
+
+# Django stuff:
+#*.log
+#.static_storage/
+#.media/
+#local_settings.py
+
+# Flask stuff:
+#instance/
+#.webassets-cache
+
+# Scrapy stuff:
+#.scrapy
+
+# Sphinx documentation
+#docs/_build/
+
+# PyBuilder
+#target/
+
+# Jupyter Notebook
+.ipynb_checkpoint/*
+
+# pyenv
+#.python-version
+
+# celery beat schedule file
+#celerybeat-schedule
+
+# SageMath parsed files
+#*.sage.py
+
+# Environments
+#.env
+#.venv
+#env/
+#venv/
+#ENV/
+#env.bak/
+#venv.bak/
+
+# Spyder project settings
+#.spyderproject
+#.spyproject
+
+# Rope project settings
+#.ropeproject
+
+# mkdocs documentation
+#/site
+/models/
+# mypy
+#.mypy_cache/
+#over 100MB
+
+# Add any directories, files, or patterns you don't want to be tracked by version control
+
+
+#deep settings
+*.h5
+
+.OpenAITools/chromedriver
+/OpenAITools/chromedriver

OpenAITools/.ipynb_checkpoints/ECarteTools-checkpoint.py

@@ -0,0 +1,73 @@
+import openai
+import time
+import wikipedia
+import random
+import re
+import requests
+from bs4 import BeautifulSoup
+import os
+import glob
+from natsort import natsorted
+import requests
+from bs4 import BeautifulSoup
+import xml.etree.ElementTree as ET
+import pandas as pd
+
+wikipedia.set_lang("ja")
+# APIキーの設定
+openai.api_key = os.environ['OPENAI_API_KEY']
+engine="gpt-3.5-turbo"
+
+
+def generate(system_template,prompt,engine="gpt-3.5-turbo"):
+    while True:  #OpenAI APIが落ちてる時に無限リトライするので注意
+        try:
+            response = openai.ChatCompletion.create(
+                model=engine,
+                messages=[
+                    {"role": "system", "content": system_template},
+                    {"role": "user", "content":prompt},
+                    ]
+            )
+            result=response["choices"][0]["message"]["content"]
+            return result
+        except:
+            print("リトライ")
+            time.sleep(30)
+            pass
+
+def generate_carte(prompt,engine="gpt-3.5-turbo"):
+    while True:  #OpenAI APIが落ちてる時に無限リトライするので注意
+        try:
+            response = openai.ChatCompletion.create(
+                model=engine,
+                messages=[
+                    {"role": "system", "content": "You are useful assistant"},
+                    {"role": "user", "content":"%s\n・・・という患者と医師の会話をSOAP形式のカルテとして日本語で端的にまとめて下さい。各セクションはS),O), A),P)として下さい "%prompt},
+                    ]
+            )
+            result=response["choices"][0]["message"]["content"]
+            return result
+        except:
+            print("リトライ")
+            time.sleep(30)
+            pass
+        
+def get_selected_fileds(texts):
+    input_name = texts.replace(' ' , "+")
+    corona_fields = ct.get_study_fields(
+    search_expr="%s SEARCH[Location](AREA[LocationCountry]Japan AND AREA[LocationStatus]Recruiting)"%(input_name),
+    fields=["NCTId", "Condition", "BriefTitle",'BriefSummary','EligibilityCriteria'],
+    max_studies=500,
+    fmt="csv")
+    return corona_fields
+
+def get_retriever_str(fields):
+    retriever_str=''
+    for i in range(1,len(fields)):
+        colnames = fields[0]
+        targetCol = fields[i]
+        for f in range(len(fields[0])):
+            retriever_str+=colnames[f] + ":" + targetCol[f] +"\n"
+        retriever_str+='\n'
+    return retriever_str

OpenAITools/.ipynb_checkpoints/ExpertTools-checkpoint.py

@@ -0,0 +1,245 @@
+import os
+import openai
+import time
+import wikipedia
+import random
+import re
+import requests
+from bs4 import BeautifulSoup
+import os
+import glob
+from natsort import natsorted
+import requests
+from bs4 import BeautifulSoup
+import xml.etree.ElementTree as ET
+from pytrials.client import ClinicalTrials
+from Bio import Entrez
+import pandas as pd
+import numpy as np
+import time
+#from langchain.agents import create_pandas_dataframe_agent
+from langchain_experimental.agents import create_pandas_dataframe_agent
+from langchain.llms import OpenAI
+
+# APIキーの設定
+openai.api_key = os.environ['OPENAI_API_KEY']
+gptengine="gpt-3.5-turbo"
+
+
+"""def get_selected_fileds(texts):
+    ct = ClinicalTrials()
+    input_name = texts.replace(' ' , "+")
+    corona_fields = ct.get_study_fields(
+    search_expr="%s SEARCH[Location](AREA[LocationCountry]Japan AND AREA[LocationStatus]Recruiting)"%(input_name),
+    fields=["NCTId", "Condition", "BriefTitle",'BriefSummary','EligibilityCriteria'],
+    max_studies=500,
+    fmt="csv")
+    return corona_fields"""
+
+def get_retriever_str(fields):
+    retriever_str=''
+    for i in range(1,len(fields)):
+        colnames = fields[0]
+        targetCol = fields[i]
+        for f in range(len(fields[0])):
+            retriever_str+=colnames[f] + ":" + targetCol[f] +"\n"
+        retriever_str+='\n'
+    return retriever_str
+
+def get_chanked_retriever(fields):
+    retriever_list =[]
+    for i in range(1,len(fields)):
+        retriever_str=''
+        colnames = fields[0]
+        targetCol = fields[i]
+        for f in range(len(fields[0])):
+            retriever_str+=colnames[f] + ":" + targetCol[f] +"\n"
+        retriever_list.append(retriever_str)
+    return retriever_list
+
+from pytrials.client import ClinicalTrials
+def get_selected_fields(texts, split_criteria=False, 
+                        split_word_number = False, split_number=700):
+    ct = ClinicalTrials()
+    input_name = texts.replace(' ', "+")
+    corona_fields = ct.get_study_fields(
+        search_expr="%s SEARCH[Location](AREA[LocationCountry]Japan AND AREA[LocationStatus]Recruiting)" % (input_name),
+        fields=["NCTId", "Condition", "BriefTitle", 'BriefSummary', 'EligibilityCriteria'],
+        max_studies=500,
+        fmt="csv")
+
+    if split_criteria:
+        new_fields = []
+        
+        # 検索対象の文字列
+        target_string1 = 'Exclusion Criteria'
+        target_string2 = 'Exclusion criteria'
+
+        # 各要素で検索対象の文字列を探し、直前で分割して新しいリストに格納
+        for corona_field in corona_fields:
+            new_list = []
+            for item in corona_field:
+                if target_string1 in item:
+                    split_position = item.index(target_string1)
+                    new_list.append(item[:split_position])
+                    new_list.append(item[split_position:])
+                elif target_string2 in item:
+                    split_position = item.index(target_string2)
+                    new_list.append(item[:split_position])
+                    new_list.append(item[split_position:])
+                else:
+                    new_list.append(item)
+            new_fields.append(new_list)
+    else:
+        new_fields = corona_fields
+    
+    if split_word_number:
+        split_fields = []
+        for new_field in new_fields:
+            new_list= []
+        
+             # 各要素を調べて、700文字以上であれば分割し、新しいリストに格納
+            for item in new_field:
+                item_length = len(item)
+                if item_length > split_number:
+                    num_parts = -(-item_length // split_number)  # 向上の除算を用いて分割数を計算
+                    for i in range(num_parts):
+                        start_index = i * split_number
+                        end_index = min((i + 1) * split_number, item_length)  # 文字列の終わりを超えないように調整
+                        new_list.append(item[start_index:end_index])
+                else:
+                    new_list.append(item)
+                    
+            split_fields.append(new_list)
+        new_fields = split_fields            
+        
+    return new_fields
+
+
+def print_agent_results(df, Ids, 
+                        interesteds = ['Condition', 'BriefTitle', 'BriefSummary', 'EligibilityCriteria'],
+                       translater=None):
+    results = ""
+    for Id in Ids:
+        print("%s\n"%Id)
+        sdf = df[df['NCTId'] == Id]
+        for interested in interesteds:
+            # 最初の要素を取得
+            results += '%s: \n %s \n' % (interested, sdf[interested].iloc[0])
+            #print('%s: \n %s \n' % (interested, sdf[interested].iloc[0]))
+        if translater:
+            to_be_printed = translater.translate(results)
+        else:
+            to_be_printed =results
+        print(to_be_printed)
+
+def search(query):
+    Entrez.email = os.getenv('MAIL_ADRESS')
+    #Entrez.email='sing.monotonyflower@gmail.com'
+    handle = Entrez.esearch(db='pubmed',
+                           sort = 'relevance',
+                           retmax = '20',
+                           retmode = 'xml',
+                           term = query)
+    results = Entrez.read(handle)
+    return results
+
+def fetch_details(id_list):
+    ids = ','.join(id_list)
+    Entrez.email = os.getenv('MAIL_ADRESS')
+    #Entrez.email = 'sing.monotonyflower@gmail.com' 
+    handle = Entrez.efetch(db = 'pubmed',
+                          retmode = 'xml',
+                          id = ids)
+    results = Entrez.read(handle)
+    return results
+'''def generate(prompt,engine=None):
+    if engine is None:
+        engine=gptengine
+    while True:  #OpenAI APIが落ちてる時に無限リトライするので注意
+        try:
+            response = openai.ChatCompletion.create(
+                model=engine,
+                messages=[
+                    {"role": "system", "content": "You are useful assistant"},
+                    {"role": "user", "content":prompt},
+                    ]
+            )
+            result=response["choices"][0]["message"]["content"]
+            return result
+        except Exception as e:
+            print(e)
+            print("リトライ")
+            time.sleep(30)
+            pass
+'''
+
+def generate(prompt,engine=None):
+    if engine is None:
+        engine=gptengine
+    while True:  #OpenAI APIが落ちてる時に無限リトライするので注意
+        try:
+            response = openai.chat.completions.create(
+                model=engine,
+                messages=[
+                    {"role": "system", "content": "You are useful assistant"},
+                    {"role": "user", "content":prompt},
+                    ]
+            )
+            #result=response["choices"][0]["message"]["content"]
+            result=response.choices[0].message.content
+            return result
+        except Exception as e:
+            print(e)
+            print("リトライ")
+            time.sleep(30)
+            pass
+
+def GetPubmedSummaryDf(studies):
+    title_list= []
+    abstract_list=[]
+    journal_list = []
+    language_list =[]
+    pubdate_year_list = []
+    pubdate_month_list = []
+    studiesIdList = studies['IdList']
+    chunk_size = 10000
+    for chunk_i in range(0, len(studiesIdList), chunk_size):
+        chunk = studiesIdList[chunk_i:chunk_i + chunk_size]
+        
+        try:
+            papers = fetch_details(chunk)
+            for i, paper in enumerate(papers['PubmedArticle']):
+                title_list.append(paper['MedlineCitation']['Article']['ArticleTitle'])
+                try:
+                    abstract_list.append(paper['MedlineCitation']['Article']['Abstract']['AbstractText'][0])
+                except:
+                    abstract_list.append('No Abstract')
+                journal_list.append(paper['MedlineCitation']['Article']['Journal']['Title'])
+                language_list.append(paper['MedlineCitation']['Article']['Language'][0])
+                try:
+                    pubdate_year_list.append(paper['MedlineCitation']['Article']['Journal']['JournalIssue']['PubDate']['Year'])
+                except:
+                    pubdate_year_list.append('No Data')
+                try:
+                    pubdate_month_list.append(paper['MedlineCitation']['Article']['Journal']['JournalIssue']['PubDate']['Month'])
+                except:
+                    pubdate_month_list.append('No Data')
+        except: # occasionally a chunk might annoy your parser
+            pass
+    df = pd.DataFrame(list(zip(
+        title_list, abstract_list, journal_list, language_list, pubdate_year_list,
+                          pubdate_month_list)),
+                      columns=['Title', 'Abstract', 'Journal', 'Language', 'Year','Month'])
+    return df, abstract_list
+
+def ClinicalAgent(fileds, verbose=False):
+    df = pd.DataFrame.from_records(fileds[1:], columns=fileds[0])
+    return create_pandas_dataframe_agent(OpenAI(temperature=0, model='gpt-3.5-turbo-16k'), df, verbose=verbose)
+
+def GetNCTID(results):
+    # NCTで始まる単語を検索する正規表現
+    pattern = r'\bNCT\d+\b'
+    # 正規表現を使って単語を抽出
+    nct_words = re.findall(pattern,results)
+    return nct_words

OpenAITools/.ipynb_checkpoints/FetchTools-checkpoint.py

@@ -0,0 +1,158 @@
+import os
+import pandas as pd
+#from llama_index.llms.replicate import Replicate
+import requests
+import re
+
+
+def extract_japan_cities(text):
+    # 正規表現を使用して " - Japan" で終わる都市名を抽出
+    pattern = r'(\b\w+\s*\w*\b) - Japan'
+    cities = re.findall(pattern, text)
+    unique_cities = list(set(cities))
+    # ユニークな都市名をソートしてカンマで区切られた文字列に変換
+    unique_cities.sort()
+    return ', '.join(unique_cities)
+
+def fetch_clinical_trials(cancer_name):
+    search_expr="%s SEARCH[Location](AREA[LocationCountry]Japan AND AREA[LocationStatus]Recruiting)" % (cancer_name)
+    # Initial URL for the first API call
+    base_url = "https://clinicaltrials.gov/api/v2/studies"
+    params = {
+        "query.titles": search_expr,
+        "pageSize": 100
+    }
+
+    # Initialize an empty list to store the data
+    data_list = []
+    # Loop until there is no nextPageToken
+    while True:
+        # Print the current URL (for debugging purposes)
+        print("Fetching data from:", base_url + '?' + '&'.join([f"{k}={v}" for k, v in params.items()]))
+        
+        # Send a GET request to the API
+        response = requests.get(base_url, params=params)
+
+        # Check if the request was successful
+        if response.status_code == 200:
+            data = response.json()  # Parse JSON response
+            studies = data.get('studies', [])  # Extract the list of studies
+
+            # Loop through each study and extract specific information
+            for study in studies:
+                # Safely access nested keys
+                nctId = study['protocolSection']['identificationModule'].get('nctId', 'Unknown')
+                startDate = study['protocolSection']['statusModule'].get('startDateStruct', {}).get('date', 'Unknown Date')
+                conditions = ', '.join(study['protocolSection']['conditionsModule'].get('conditions', ['No conditions listed']))
+                title = study['protocolSection']['identificationModule'].get('briefTitle', 'no title')
+                summary = study['protocolSection']['descriptionModule'].get('briefSummary', 'no summary')
+
+                # Extract locations safely
+                locations_list = study['protocolSection'].get('contactsLocationsModule', {}).get('locations', [])
+                locations = ', '.join([f"{location.get('city', 'No City')} - {location.get('country', 'No Country')}" for location in locations_list]) if locations_list else "No locations listed"
+
+                JapanesLocations = extract_japan_cities(locations)
+                # Extract dates and phases
+                primaryCompletionDate = study['protocolSection']['statusModule'].get('primaryCompletionDateStruct', {}).get('date', 'Unknown Date')
+
+                phases = ', '.join(study['protocolSection']['designModule'].get('phases', ['Not Available']))
+                eligibilityCriteria = study['protocolSection']['eligibilityModule'].get('eligibilityCriteria', 'Unknown')
+
+                # Append the data to the list as a dictionary
+                data_list.append({
+                    "NCTID": nctId,
+                    "Title": title,
+                    #"Start Date": startDate,
+                    "Primary Completion Date": primaryCompletionDate,
+                    #"Conditions": conditions,
+                    "Cancer": conditions,
+                    "Summary": summary,
+                    "Japanes Locations": JapanesLocations,
+                    #"Phases": phases,
+                    "Eligibility Criteria": eligibilityCriteria
+                })
+
+            # Check for nextPageToken and update the params or break the loop
+            nextPageToken = data.get('nextPageToken')
+            if nextPageToken:
+                params['pageToken'] = nextPageToken  # Set the pageToken for the next request
+            else:
+                break  # Exit the loop if no nextPageToken is present
+        else:
+            print("Failed to fetch data. Status code:", response.status_code)
+            break
+
+    # Create a DataFrame from the list of dictionaries
+    df = pd.DataFrame(data_list)
+    return df
+
+def fetch_clinical_trials_jp(cancer_name):
+    search_expr="%s SEARCH[Location](AREA[LocationCountry]Japan AND AREA[LocationStatus]Recruiting)" % (cancer_name)
+    # Initial URL for the first API call
+    base_url = "https://clinicaltrials.gov/api/v2/studies"
+    params = {
+        "query.titles": search_expr,
+        "pageSize": 100
+    }
+
+    # Initialize an empty list to store the data
+    data_list = []
+    # Loop until there is no nextPageToken
+    while True:
+        # Print the current URL (for debugging purposes)
+        print("Fetching data from:", base_url + '?' + '&'.join([f"{k}={v}" for k, v in params.items()]))
+        
+        # Send a GET request to the API
+        response = requests.get(base_url, params=params)
+
+        # Check if the request was successful
+        if response.status_code == 200:
+            data = response.json()  # Parse JSON response
+            studies = data.get('studies', [])  # Extract the list of studies
+
+            # Loop through each study and extract specific information
+            for study in studies:
+                # Safely access nested keys
+                nctId = study['protocolSection']['identificationModule'].get('nctId', 'Unknown')
+                startDate = study['protocolSection']['statusModule'].get('startDateStruct', {}).get('date', 'Unknown Date')
+                conditions = ', '.join(study['protocolSection']['conditionsModule'].get('conditions', ['No conditions listed']))
+                title = study['protocolSection']['identificationModule'].get('briefTitle', 'no title')
+                summary = study['protocolSection']['descriptionModule'].get('briefSummary', 'no summary')
+
+                # Extract locations safely
+                locations_list = study['protocolSection'].get('contactsLocationsModule', {}).get('locations', [])
+                locations = ', '.join([f"{location.get('city', 'No City')} - {location.get('country', 'No Country')}" for location in locations_list]) if locations_list else "No locations listed"
+
+                JapanesLocations = extract_japan_cities(locations)
+                # Extract dates and phases
+                primaryCompletionDate = study['protocolSection']['statusModule'].get('primaryCompletionDateStruct', {}).get('date', 'Unknown Date')
+
+                phases = ', '.join(study['protocolSection']['designModule'].get('phases', ['Not Available']))
+                eligibilityCriteria = study['protocolSection']['eligibilityModule'].get('eligibilityCriteria', 'Unknown')
+
+                # Append the data to the list as a dictionary
+                data_list.append({
+                    "NCTID": nctId,
+                    "タイトル": title,
+                    #"Start Date": startDate,
+                    #"Primary Completion Date": primaryCompletionDate,
+                    "対象となる癌": conditions,
+                    "サマリー": summary,
+                    "場所": JapanesLocations,
+                    #"Phases": phases,
+                    "クライテリア": eligibilityCriteria
+                })
+
+            # Check for nextPageToken and update the params or break the loop
+            nextPageToken = data.get('nextPageToken')
+            if nextPageToken:
+                params['pageToken'] = nextPageToken  # Set the pageToken for the next request
+            else:
+                break  # Exit the loop if no nextPageToken is present
+        else:
+            print("Failed to fetch data. Status code:", response.status_code)
+            break
+
+    # Create a DataFrame from the list of dictionaries
+    df = pd.DataFrame(data_list)
+    return df

OpenAITools/.ipynb_checkpoints/scrapeThisData-checkpoint.py

@@ -0,0 +1,237 @@
+from selenium import webdriver
+from selenium.webdriver.support.ui import Select
+from selenium.webdriver.common.by import By
+
+import requests
+from bs4 import BeautifulSoup
+import re
+
+import os
+import time
+
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.common.action_chains import ActionChains
+import chromedriver_autoinstaller
+
+class ScrapeThatData:
+    
+    def __init__(self, time_threshold = 10):
+        
+        try:
+            chrome_options = webdriver.ChromeOptions()
+            chrome_options.add_argument('--no-sandbox')
+            self.driver = webdriver.Chrome(options=chrome_options)
+            
+        except:
+            chromedriver_autoinstaller.install()
+            chrome_options = webdriver.ChromeOptions()
+            chrome_options.add_argument('--no-sandbox')
+            self.driver = webdriver.Chrome(options=chrome_options)
+            
+            
+
+        self.wait = WebDriverWait(self.driver,time_threshold)
+        self.attribute_dict = {'status':1 ,'conditions':2, 'interventions': 3, 'study type':4, 
+                   'phase':5, 'sponsor':6, 'funder type':7 , 'study design': 8,
+                   'outcome measures':9, 'number enrolled':10, 'sex':11, 'age':12,
+                   'nct number': 13, 'other ids':14, 'title acronym': 15 , 'study start': 16,
+                   'primary completion': 17, 'study completion': 18 , 'first posted': 19, 
+                   'last update posted': 20 , 'results first posted': 21 , 'locations':22, 'study documents': 23}
+
+        self.status_dict =     {'not yet recruiting' : 'notYetRecrCB',
+         'recruiting' : 'recruitingCB',
+         'enrolling by invitation':'enrollingByInvCB',
+         'active, not recruiting': 'activeCB',
+         'suspended': 'suspendedCB',
+         'terminated':'terminatedCB',
+         'completed':'completedCB',
+         'withdrawn': 'withdrawnCB',
+         'unknown status': 'unknownCB'}
+    
+    def clicking_show_hide_cols(self, driver):
+        columns = driver.find_element(By.XPATH,'//*[@id="theDataTable_wrapper"]/div[3]/button')
+        action_chain = ActionChains(driver)
+        action_chain.move_to_element(columns).click()
+        action_chain.perform()
+    
+    def select_attributes_to_show(self, listed_attributes, attribute_dict):
+        ll = [value.lower() for value in listed_attributes if value.lower() in ['status', 'conditions', 'interventions', 'locations']]
+        if ll:
+            to_show = [value.lower() for  value in listed_attributes if value.lower() not in ll]
+            to_hide = [value for value in ['status', 'conditions', 'interventions', 'locations'] if value not in ll]
+            to_click = to_hide + to_show
+            for att in to_click:
+                self.clicking_show_hide_cols(self.driver)
+                time.sleep(1)
+                self.wait.until(EC.presence_of_element_located((By.XPATH,'//*[@id="theDataTable_wrapper"]/div[3]/div[2]/button['+ str(attribute_dict[att]) + ']'))).click()
+                time.sleep(1)
+        else:
+            for att in listed_attributes:
+                self.clicking_show_hide_cols(self.driver)
+                time.sleep(1)
+                self.wait.until(EC.presence_of_element_located((By.XPATH,'//*[@id="theDataTable_wrapper"]/div[3]/div[2]/button['+ str(attribute_dict[att.lower()]) + ']'))).click()
+                time.sleep(1)
+
+    def select_by_status(self, listed_states, status_dict):
+        if listed_states:
+            for status in listed_states:
+                self.driver.find_element(By.ID,status_dict[status.lower()]).click()
+
+            self.driver.find_element(By.XPATH,'//*[@id="FiltersBody"]/div[1]/input[1]').click()
+            time.sleep(3)
+
+
+        select = Select(self.driver.find_element_by_name('theDataTable_length'))
+        select.select_by_value('100')
+
+    def collect_data_search_page(self,l_ordered, amount_of_data = None):
+
+        class_name = ''
+        page_index = 1
+        
+        elements = [l_ordered]
+
+        while 'disabled' not in class_name :
+           
+
+
+            time.sleep(10)
+
+            print('Getting data from page {}'.format(page_index))
+
+            #Counting how many rows of the table appear
+            table = self.driver.find_element(By.ID,'theDataTable')
+            row_count = len(table.find_elements(By.TAG_NAME,"tr")) 
+
+            #Looping table page
+            for index in range(1, row_count):
+                row = []
+                if 'status' in l_ordered:
+                    self.wait.until(EC.presence_of_element_located((By.CSS_SELECTOR,'#theDataTable > tbody > tr:nth-child('+str(index)+') > td:nth-child(3)')))
+                    status_element = self.driver.find_elements(By.CLASS_NAME,'#theDataTable > tbody > tr:nth-child('+str(index)+') > td:nth-child(3) > span')
+                    row.append(status_element.text.strip())
+                    for i, val  in enumerate(l_ordered):
+                        if val == 'status':
+                            continue
+                        
+                        self.wait.until(EC.presence_of_element_located((By.CSS_SELECTOR,'#theDataTable > tbody > tr:nth-child('+str(index)+') > td:nth-child('+str(4+i)+')')))
+                        element = self.driver.find_elements(By.CLASS_NAME,'#theDataTable > tbody > tr:nth-child('+str(index)+') > td:nth-child('+str(4+i)+')')
+                        try:
+                            row.append(element.text.strip())
+                        except:
+                            print(i, element)
+                else:
+                    for i, val  in enumerate(l_ordered):
+                        self.wait.until(EC.presence_of_element_located((By.CSS_SELECTOR,'#theDataTable > tbody > tr:nth-child('+str(index)+') > td:nth-child('+str(3+i)+')')))
+                        element = self.driver.find_elements(By.CLASS_NAME,'#theDataTable > tbody > tr:nth-child('+str(index)+') > td:nth-child('+str(3+i)+')')
+                        try:
+                            row.append(element.text.strip())
+                        except:
+                            print(i, element)
+                elements.append(row)
+
+
+
+
+            #Getting next page button
+            next_page= self.driver.find_element(By.ID,"theDataTable_next")
+
+            #Getting the class attribute of the next page button
+            class_name = next_page.get_attribute('class')
+
+            #Going to the next page
+            next_page.click()
+            page_index += 1
+            
+            if amount_of_data:
+                if len(elements) >= amount_of_data or row_count < amount_of_data :
+                    break
+                else:
+                    continue
+
+        return elements
+
+    def get_criteria(self, NCTnumber):
+    
+        url = 'https://clinicaltrials.gov/ct2/show/' + NCTnumber
+        ClinicalTrialpage = requests.get(url)
+        soup = BeautifulSoup(ClinicalTrialpage.text, 'html.parser')
+
+        wrapping_crit_class = soup.find_all("div", {"class": "tr-indent2"})
+        list_elements = wrapping_crit_class[1].find_all(re.compile("(ul|ol)"))
+        inclusion, exclusion  = ('','')
+
+
+        if not list_elements:
+            print ("WARNING: Study number " + NCTnumber + " doesn't have eligibility criteria or HTML tag format is not a list")
+        else:
+
+            if len(list_elements) == 1: 
+                try:
+                    if wrapping_crit_class[1].find(text = 'Inclusion Criteria:'):
+                        inclusion = list_elements[0].find_all("li")
+
+                    elif wrapping_crit_class[1].find(text = 'Exclusion Criteria:'):
+                        exclusion = list_elements[0].find_all("li")
+                except:
+                    print('criteria doesnt exist')
+            else:
+                inclusion = list_elements[0].find_all("li")
+                exclusion = list_elements[1].find_all("li")
+
+
+        inclusion = ' '.join([t.text.strip() for t in inclusion ])
+        exclusion = ' '.join([t.text.strip() for t in exclusion ])
+        
+        return(inclusion, exclusion)
+
+#function that gets number of patients enrolled in a study
+    def get_enrollment (self, NCTnumber):
+        url = 'https://clinicaltrials.gov/ct2/show/' + NCTnumber
+        ClinicalTrialpage = requests.get(url)
+        soup = BeautifulSoup(ClinicalTrialpage.text, 'html.parser')
+        enrollment = ''
+        wrapping_enrol_class = soup.find_all('td', {'headers':'studyInfoColData','style':"padding-left:1em"})
+        if not wrapping_enrol_class:
+            print('WARNING: Number of Participants in Study number '+ NCTnumber +' is unavailable')
+        else:
+            enrollment = wrapping_enrol_class[1]
+            enrollment = enrollment.text.split()[0]
+            if enrollment.isdigit() == False:
+                print ('WARNING: Number of Participants in Study number '+ NCTnumber +' is unavailable')
+            else:
+                return(enrollment)
+
+    
+        
+    def __call__(self, condition, listed_attributes, listed_states, amount_of_data):
+        
+        self.driver.get('https://clinicaltrials.gov/ct2/results?cond=' + condition + '&rank=1&view=record#rowId0')
+        self.select_attributes_to_show(listed_attributes, self.attribute_dict)
+     
+        try:
+            self.select_by_status(listed_states, self.status_dict)
+        except:
+            print('select by status is a problem')
+        n = []
+        for i in listed_attributes: 
+            n.append(self.attribute_dict[i.lower()])
+        attribute_ordered = [list(self.attribute_dict.keys())[list(self.attribute_dict.values()).index(i)]for i in sorted(n)]
+
+        search_data = self.collect_data_search_page(attribute_ordered, amount_of_data=amount_of_data)
+        nct_numbers = [e[search_data[0].index('nct number')] for e in search_data[1:]]
+        search_data[0].extend(['inclusion', 'exclusion', 'enrollment'])
+        for index, nct in enumerate(nct_numbers):
+            if index % 100 == 0 and index!= 0:
+                print("Collected Data from {} Studies: ".format(index))
+
+            inc, exc = self.get_criteria(nct)
+            enrol = self.get_enrollment(nct)
+            search_data[index + 1].extend([inc, exc, enrol])
+        return search_data
+#     except: 
+#         print('no data available with the specified status')
+
+        

OpenAITools/ECarteTools.py

@@ -0,0 +1,73 @@
+import openai
+import time
+import wikipedia
+import random
+import re
+import requests
+from bs4 import BeautifulSoup
+import os
+import glob
+from natsort import natsorted
+import requests
+from bs4 import BeautifulSoup
+import xml.etree.ElementTree as ET
+import pandas as pd
+
+wikipedia.set_lang("ja")
+# APIキーの設定
+openai.api_key = os.environ['OPENAI_API_KEY']
+engine="gpt-3.5-turbo"
+
+
+def generate(system_template,prompt,engine="gpt-3.5-turbo"):
+    while True:  #OpenAI APIが落ちてる時に無限リトライするので注意
+        try:
+            response = openai.ChatCompletion.create(
+                model=engine,
+                messages=[
+                    {"role": "system", "content": system_template},
+                    {"role": "user", "content":prompt},
+                    ]
+            )
+            result=response["choices"][0]["message"]["content"]
+            return result
+        except:
+            print("リトライ")
+            time.sleep(30)
+            pass
+
+def generate_carte(prompt,engine="gpt-3.5-turbo"):
+    while True:  #OpenAI APIが落ちてる時に無限リトライするので注意
+        try:
+            response = openai.ChatCompletion.create(
+                model=engine,
+                messages=[
+                    {"role": "system", "content": "You are useful assistant"},
+                    {"role": "user", "content":"%s\n・・・という患者と医師の会話をSOAP形式のカルテとして日本語で端的にまとめて下さい。各セクションはS),O), A),P)として下さい "%prompt},
+                    ]
+            )
+            result=response["choices"][0]["message"]["content"]
+            return result
+        except:
+            print("リトライ")
+            time.sleep(30)
+            pass
+        
+def get_selected_fileds(texts):
+    input_name = texts.replace(' ' , "+")
+    corona_fields = ct.get_study_fields(
+    search_expr="%s SEARCH[Location](AREA[LocationCountry]Japan AND AREA[LocationStatus]Recruiting)"%(input_name),
+    fields=["NCTId", "Condition", "BriefTitle",'BriefSummary','EligibilityCriteria'],
+    max_studies=500,
+    fmt="csv")
+    return corona_fields
+
+def get_retriever_str(fields):
+    retriever_str=''
+    for i in range(1,len(fields)):
+        colnames = fields[0]
+        targetCol = fields[i]
+        for f in range(len(fields[0])):
+            retriever_str+=colnames[f] + ":" + targetCol[f] +"\n"
+        retriever_str+='\n'
+    return retriever_str

OpenAITools/ExpertTools.py

@@ -0,0 +1,246 @@
+import os
+import openai
+import time
+import wikipedia
+import random
+import re
+import requests
+from bs4 import BeautifulSoup
+import os
+import glob
+from natsort import natsorted
+import requests
+from bs4 import BeautifulSoup
+import xml.etree.ElementTree as ET
+from pytrials.client import ClinicalTrials
+from Bio import Entrez
+import pandas as pd
+import numpy as np
+import time
+#from langchain.agents import create_pandas_dataframe_agent
+from langchain_experimental.agents import create_pandas_dataframe_agent
+#from langchain.llms import OpenAI
+from langchain_community.llms import OpenAI
+
+# APIキーの設定
+openai.api_key = os.environ['OPENAI_API_KEY']
+gptengine="gpt-3.5-turbo"
+
+
+"""def get_selected_fileds(texts):
+    ct = ClinicalTrials()
+    input_name = texts.replace(' ' , "+")
+    corona_fields = ct.get_study_fields(
+    search_expr="%s SEARCH[Location](AREA[LocationCountry]Japan AND AREA[LocationStatus]Recruiting)"%(input_name),
+    fields=["NCTId", "Condition", "BriefTitle",'BriefSummary','EligibilityCriteria'],
+    max_studies=500,
+    fmt="csv")
+    return corona_fields"""
+
+def get_retriever_str(fields):
+    retriever_str=''
+    for i in range(1,len(fields)):
+        colnames = fields[0]
+        targetCol = fields[i]
+        for f in range(len(fields[0])):
+            retriever_str+=colnames[f] + ":" + targetCol[f] +"\n"
+        retriever_str+='\n'
+    return retriever_str
+
+def get_chanked_retriever(fields):
+    retriever_list =[]
+    for i in range(1,len(fields)):
+        retriever_str=''
+        colnames = fields[0]
+        targetCol = fields[i]
+        for f in range(len(fields[0])):
+            retriever_str+=colnames[f] + ":" + targetCol[f] +"\n"
+        retriever_list.append(retriever_str)
+    return retriever_list
+
+from pytrials.client import ClinicalTrials
+def get_selected_fields(texts, split_criteria=False, 
+                        split_word_number = False, split_number=700):
+    ct = ClinicalTrials()
+    input_name = texts.replace(' ', "+")
+    corona_fields = ct.get_study_fields(
+        search_expr="%s SEARCH[Location](AREA[LocationCountry]Japan AND AREA[LocationStatus]Recruiting)" % (input_name),
+        fields=["NCTId", "Condition", "BriefTitle", 'BriefSummary', 'EligibilityCriteria'],
+        max_studies=500,
+        fmt="csv")
+
+    if split_criteria:
+        new_fields = []
+        
+        # 検索対象の文字列
+        target_string1 = 'Exclusion Criteria'
+        target_string2 = 'Exclusion criteria'
+
+        # 各要素で検索対象の文字列を探し、直前で分割して新しいリストに格納
+        for corona_field in corona_fields:
+            new_list = []
+            for item in corona_field:
+                if target_string1 in item:
+                    split_position = item.index(target_string1)
+                    new_list.append(item[:split_position])
+                    new_list.append(item[split_position:])
+                elif target_string2 in item:
+                    split_position = item.index(target_string2)
+                    new_list.append(item[:split_position])
+                    new_list.append(item[split_position:])
+                else:
+                    new_list.append(item)
+            new_fields.append(new_list)
+    else:
+        new_fields = corona_fields
+    
+    if split_word_number:
+        split_fields = []
+        for new_field in new_fields:
+            new_list= []
+        
+             # 各要素を調べて、700文字以上であれば分割し、新しいリストに格納
+            for item in new_field:
+                item_length = len(item)
+                if item_length > split_number:
+                    num_parts = -(-item_length // split_number)  # 向上の除算を用いて分割数を計算
+                    for i in range(num_parts):
+                        start_index = i * split_number
+                        end_index = min((i + 1) * split_number, item_length)  # 文字列の終わりを超えないように調整
+                        new_list.append(item[start_index:end_index])
+                else:
+                    new_list.append(item)
+                    
+            split_fields.append(new_list)
+        new_fields = split_fields            
+        
+    return new_fields
+
+
+def print_agent_results(df, Ids, 
+                        interesteds = ['Condition', 'BriefTitle', 'BriefSummary', 'EligibilityCriteria'],
+                       translater=None):
+    results = ""
+    for Id in Ids:
+        print("%s\n"%Id)
+        sdf = df[df['NCTId'] == Id]
+        for interested in interesteds:
+            # 最初の要素を取得
+            results += '%s: \n %s \n' % (interested, sdf[interested].iloc[0])
+            #print('%s: \n %s \n' % (interested, sdf[interested].iloc[0]))
+        if translater:
+            to_be_printed = translater.translate(results)
+        else:
+            to_be_printed =results
+        print(to_be_printed)
+
+def search(query):
+    Entrez.email = os.getenv('MAIL_ADRESS')
+    #Entrez.email='sing.monotonyflower@gmail.com'
+    handle = Entrez.esearch(db='pubmed',
+                           sort = 'relevance',
+                           retmax = '20',
+                           retmode = 'xml',
+                           term = query)
+    results = Entrez.read(handle)
+    return results
+
+def fetch_details(id_list):
+    ids = ','.join(id_list)
+    Entrez.email = os.getenv('MAIL_ADRESS')
+    #Entrez.email = 'sing.monotonyflower@gmail.com' 
+    handle = Entrez.efetch(db = 'pubmed',
+                          retmode = 'xml',
+                          id = ids)
+    results = Entrez.read(handle)
+    return results
+'''def generate(prompt,engine=None):
+    if engine is None:
+        engine=gptengine
+    while True:  #OpenAI APIが落ちてる時に無限リトライするので注意
+        try:
+            response = openai.ChatCompletion.create(
+                model=engine,
+                messages=[
+                    {"role": "system", "content": "You are useful assistant"},
+                    {"role": "user", "content":prompt},
+                    ]
+            )
+            result=response["choices"][0]["message"]["content"]
+            return result
+        except Exception as e:
+            print(e)
+            print("リトライ")
+            time.sleep(30)
+            pass
+'''
+
+def generate(prompt,engine=None):
+    if engine is None:
+        engine=gptengine
+    while True:  #OpenAI APIが落ちてる時に無限リトライするので注意
+        try:
+            response = openai.chat.completions.create(
+                model=engine,
+                messages=[
+                    {"role": "system", "content": "You are useful assistant"},
+                    {"role": "user", "content":prompt},
+                    ]
+            )
+            #result=response["choices"][0]["message"]["content"]
+            result=response.choices[0].message.content
+            return result
+        except Exception as e:
+            print(e)
+            print("リトライ")
+            time.sleep(30)
+            pass
+
+def GetPubmedSummaryDf(studies):
+    title_list= []
+    abstract_list=[]
+    journal_list = []
+    language_list =[]
+    pubdate_year_list = []
+    pubdate_month_list = []
+    studiesIdList = studies['IdList']
+    chunk_size = 10000
+    for chunk_i in range(0, len(studiesIdList), chunk_size):
+        chunk = studiesIdList[chunk_i:chunk_i + chunk_size]
+        
+        try:
+            papers = fetch_details(chunk)
+            for i, paper in enumerate(papers['PubmedArticle']):
+                title_list.append(paper['MedlineCitation']['Article']['ArticleTitle'])
+                try:
+                    abstract_list.append(paper['MedlineCitation']['Article']['Abstract']['AbstractText'][0])
+                except:
+                    abstract_list.append('No Abstract')
+                journal_list.append(paper['MedlineCitation']['Article']['Journal']['Title'])
+                language_list.append(paper['MedlineCitation']['Article']['Language'][0])
+                try:
+                    pubdate_year_list.append(paper['MedlineCitation']['Article']['Journal']['JournalIssue']['PubDate']['Year'])
+                except:
+                    pubdate_year_list.append('No Data')
+                try:
+                    pubdate_month_list.append(paper['MedlineCitation']['Article']['Journal']['JournalIssue']['PubDate']['Month'])
+                except:
+                    pubdate_month_list.append('No Data')
+        except: # occasionally a chunk might annoy your parser
+            pass
+    df = pd.DataFrame(list(zip(
+        title_list, abstract_list, journal_list, language_list, pubdate_year_list,
+                          pubdate_month_list)),
+                      columns=['Title', 'Abstract', 'Journal', 'Language', 'Year','Month'])
+    return df, abstract_list
+
+def ClinicalAgent(fileds, verbose=False):
+    df = pd.DataFrame.from_records(fileds[1:], columns=fileds[0])
+    return create_pandas_dataframe_agent(OpenAI(temperature=0, model='gpt-3.5-turbo-16k'), df, verbose=verbose)
+
+def GetNCTID(results):
+    # NCTで始まる単語を検索する正規表現
+    pattern = r'\bNCT\d+\b'
+    # 正規表現を使って単語を抽出
+    nct_words = re.findall(pattern,results)
+    return nct_words

OpenAITools/FetchTools.py

@@ -0,0 +1,158 @@
+import os
+import pandas as pd
+#from llama_index.llms.replicate import Replicate
+import requests
+import re
+
+
+def extract_japan_cities(text):
+    # 正規表現を使用して " - Japan" で終わる都市名を抽出
+    pattern = r'(\b\w+\s*\w*\b) - Japan'
+    cities = re.findall(pattern, text)
+    unique_cities = list(set(cities))
+    # ユニークな都市名をソートしてカンマで区切られた文字列に変換
+    unique_cities.sort()
+    return ', '.join(unique_cities)
+
+def fetch_clinical_trials(cancer_name):
+    search_expr="%s SEARCH[Location](AREA[LocationCountry]Japan AND AREA[LocationStatus]Recruiting)" % (cancer_name)
+    # Initial URL for the first API call
+    base_url = "https://clinicaltrials.gov/api/v2/studies"
+    params = {
+        "query.titles": search_expr,
+        "pageSize": 100
+    }
+
+    # Initialize an empty list to store the data
+    data_list = []
+    # Loop until there is no nextPageToken
+    while True:
+        # Print the current URL (for debugging purposes)
+        print("Fetching data from:", base_url + '?' + '&'.join([f"{k}={v}" for k, v in params.items()]))
+        
+        # Send a GET request to the API
+        response = requests.get(base_url, params=params)
+
+        # Check if the request was successful
+        if response.status_code == 200:
+            data = response.json()  # Parse JSON response
+            studies = data.get('studies', [])  # Extract the list of studies
+
+            # Loop through each study and extract specific information
+            for study in studies:
+                # Safely access nested keys
+                nctId = study['protocolSection']['identificationModule'].get('nctId', 'Unknown')
+                startDate = study['protocolSection']['statusModule'].get('startDateStruct', {}).get('date', 'Unknown Date')
+                conditions = ', '.join(study['protocolSection']['conditionsModule'].get('conditions', ['No conditions listed']))
+                title = study['protocolSection']['identificationModule'].get('briefTitle', 'no title')
+                summary = study['protocolSection']['descriptionModule'].get('briefSummary', 'no summary')
+
+                # Extract locations safely
+                locations_list = study['protocolSection'].get('contactsLocationsModule', {}).get('locations', [])
+                locations = ', '.join([f"{location.get('city', 'No City')} - {location.get('country', 'No Country')}" for location in locations_list]) if locations_list else "No locations listed"
+
+                JapanesLocations = extract_japan_cities(locations)
+                # Extract dates and phases
+                primaryCompletionDate = study['protocolSection']['statusModule'].get('primaryCompletionDateStruct', {}).get('date', 'Unknown Date')
+
+                phases = ', '.join(study['protocolSection']['designModule'].get('phases', ['Not Available']))
+                eligibilityCriteria = study['protocolSection']['eligibilityModule'].get('eligibilityCriteria', 'Unknown')
+
+                # Append the data to the list as a dictionary
+                data_list.append({
+                    "NCTID": nctId,
+                    "Title": title,
+                    #"Start Date": startDate,
+                    "Primary Completion Date": primaryCompletionDate,
+                    #"Conditions": conditions,
+                    "Cancer": conditions,
+                    "Summary": summary,
+                    "Japanes Locations": JapanesLocations,
+                    #"Phases": phases,
+                    "Eligibility Criteria": eligibilityCriteria
+                })
+
+            # Check for nextPageToken and update the params or break the loop
+            nextPageToken = data.get('nextPageToken')
+            if nextPageToken:
+                params['pageToken'] = nextPageToken  # Set the pageToken for the next request
+            else:
+                break  # Exit the loop if no nextPageToken is present
+        else:
+            print("Failed to fetch data. Status code:", response.status_code)
+            break
+
+    # Create a DataFrame from the list of dictionaries
+    df = pd.DataFrame(data_list)
+    return df
+
+def fetch_clinical_trials_jp(cancer_name):
+    search_expr="%s SEARCH[Location](AREA[LocationCountry]Japan AND AREA[LocationStatus]Recruiting)" % (cancer_name)
+    # Initial URL for the first API call
+    base_url = "https://clinicaltrials.gov/api/v2/studies"
+    params = {
+        "query.titles": search_expr,
+        "pageSize": 100
+    }
+
+    # Initialize an empty list to store the data
+    data_list = []
+    # Loop until there is no nextPageToken
+    while True:
+        # Print the current URL (for debugging purposes)
+        print("Fetching data from:", base_url + '?' + '&'.join([f"{k}={v}" for k, v in params.items()]))
+        
+        # Send a GET request to the API
+        response = requests.get(base_url, params=params)
+
+        # Check if the request was successful
+        if response.status_code == 200:
+            data = response.json()  # Parse JSON response
+            studies = data.get('studies', [])  # Extract the list of studies
+
+            # Loop through each study and extract specific information
+            for study in studies:
+                # Safely access nested keys
+                nctId = study['protocolSection']['identificationModule'].get('nctId', 'Unknown')
+                startDate = study['protocolSection']['statusModule'].get('startDateStruct', {}).get('date', 'Unknown Date')
+                conditions = ', '.join(study['protocolSection']['conditionsModule'].get('conditions', ['No conditions listed']))
+                title = study['protocolSection']['identificationModule'].get('briefTitle', 'no title')
+                summary = study['protocolSection']['descriptionModule'].get('briefSummary', 'no summary')
+
+                # Extract locations safely
+                locations_list = study['protocolSection'].get('contactsLocationsModule', {}).get('locations', [])
+                locations = ', '.join([f"{location.get('city', 'No City')} - {location.get('country', 'No Country')}" for location in locations_list]) if locations_list else "No locations listed"
+
+                JapanesLocations = extract_japan_cities(locations)
+                # Extract dates and phases
+                primaryCompletionDate = study['protocolSection']['statusModule'].get('primaryCompletionDateStruct', {}).get('date', 'Unknown Date')
+
+                phases = ', '.join(study['protocolSection']['designModule'].get('phases', ['Not Available']))
+                eligibilityCriteria = study['protocolSection']['eligibilityModule'].get('eligibilityCriteria', 'Unknown')
+
+                # Append the data to the list as a dictionary
+                data_list.append({
+                    "NCTID": nctId,
+                    "タイトル": title,
+                    #"Start Date": startDate,
+                    #"Primary Completion Date": primaryCompletionDate,
+                    "対象となる癌": conditions,
+                    "サマリー": summary,
+                    "場所": JapanesLocations,
+                    #"Phases": phases,
+                    "クライテリア": eligibilityCriteria
+                })
+
+            # Check for nextPageToken and update the params or break the loop
+            nextPageToken = data.get('nextPageToken')
+            if nextPageToken:
+                params['pageToken'] = nextPageToken  # Set the pageToken for the next request
+            else:
+                break  # Exit the loop if no nextPageToken is present
+        else:
+            print("Failed to fetch data. Status code:", response.status_code)
+            break
+
+    # Create a DataFrame from the list of dictionaries
+    df = pd.DataFrame(data_list)
+    return df

OpenAITools/ReviewPaperTools.py

@@ -0,0 +1,42 @@
+import re
+import pandas as pd
+
+def parse_text_file(text):
+    # セクションを分割するための正規表現パターンを定義
+    # \d+ は1つ以上の数字にマッチします
+    pattern = re.compile(r'\n\n\n\d+\.')
+
+    # テキストをセクションごとに分割
+    sections = pattern.split(text)[1:]  # 最初の空のセクションを除外
+
+    # 各セクションの前後の空白を削除
+    sections = [section.strip() for section in sections]
+
+    return sections
+
+def split_sections(text):
+    contents = text.split('\n\n')
+    contents = [section.strip() for section in contents if section.strip()]
+    if len(contents) == 8 :
+        keys = ['PublishInfo', 'Title', 'AuthorName', 'AuthorInfo', 'Abstract', 'Copyrights', 'DOI', 'COI']
+    elif len(contents) == 7 :
+        keys = ['PublishInfo', 'Title', 'AuthorName', 'AuthorInfo', 'Abstract', 'Copyrights', 'DOI']
+    elif len(contents) == 6:
+        keys = ['PublishInfo', 'Title', 'AuthorName', 'AuthorInfo', 'Abstract', 'DOI']
+    elif len(contents) == 5:
+        keys = ['PublishInfo', 'Title', 'AuthorName', 'Abstract', 'DOI']
+    
+    # 辞書を作成し、キーが存在しない場合は空の文字列を設定
+    section_dict = {key: contents[i] if i < len(contents) else "" for i, key in enumerate(keys)}
+    return section_dict
+
+
+def GetSummaryDf(textdir):
+    with open(textdir, 'r', encoding='utf-8') as f:
+        content = f.read()
+    sections = parse_text_file(content)
+    dicts = []
+    for section in sections:
+        splited_dic = split_sections(section)
+        dicts.append(splited_dic)
+    return pd.DataFrame(dicts)

OpenAITools/scrapeThisData.py

@@ -0,0 +1,237 @@
+from selenium import webdriver
+from selenium.webdriver.support.ui import Select
+from selenium.webdriver.common.by import By
+
+import requests
+from bs4 import BeautifulSoup
+import re
+
+import os
+import time
+
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.common.action_chains import ActionChains
+import chromedriver_autoinstaller
+
+class ScrapeThatData:
+    
+    def __init__(self, time_threshold = 10):
+        
+        try:
+            chrome_options = webdriver.ChromeOptions()
+            chrome_options.add_argument('--no-sandbox')
+            self.driver = webdriver.Chrome(options=chrome_options)
+            
+        except:
+            chromedriver_autoinstaller.install()
+            chrome_options = webdriver.ChromeOptions()
+            chrome_options.add_argument('--no-sandbox')
+            self.driver = webdriver.Chrome(options=chrome_options)
+            
+            
+
+        self.wait = WebDriverWait(self.driver,time_threshold)
+        self.attribute_dict = {'status':1 ,'conditions':2, 'interventions': 3, 'study type':4, 
+                   'phase':5, 'sponsor':6, 'funder type':7 , 'study design': 8,
+                   'outcome measures':9, 'number enrolled':10, 'sex':11, 'age':12,
+                   'nct number': 13, 'other ids':14, 'title acronym': 15 , 'study start': 16,
+                   'primary completion': 17, 'study completion': 18 , 'first posted': 19, 
+                   'last update posted': 20 , 'results first posted': 21 , 'locations':22, 'study documents': 23}
+
+        self.status_dict =     {'not yet recruiting' : 'notYetRecrCB',
+         'recruiting' : 'recruitingCB',
+         'enrolling by invitation':'enrollingByInvCB',
+         'active, not recruiting': 'activeCB',
+         'suspended': 'suspendedCB',
+         'terminated':'terminatedCB',
+         'completed':'completedCB',
+         'withdrawn': 'withdrawnCB',
+         'unknown status': 'unknownCB'}
+    
+    def clicking_show_hide_cols(self, driver):
+        columns = driver.find_element(By.XPATH,'//*[@id="theDataTable_wrapper"]/div[3]/button')
+        action_chain = ActionChains(driver)
+        action_chain.move_to_element(columns).click()
+        action_chain.perform()
+    
+    def select_attributes_to_show(self, listed_attributes, attribute_dict):
+        ll = [value.lower() for value in listed_attributes if value.lower() in ['status', 'conditions', 'interventions', 'locations']]
+        if ll:
+            to_show = [value.lower() for  value in listed_attributes if value.lower() not in ll]
+            to_hide = [value for value in ['status', 'conditions', 'interventions', 'locations'] if value not in ll]
+            to_click = to_hide + to_show
+            for att in to_click:
+                self.clicking_show_hide_cols(self.driver)
+                time.sleep(1)
+                self.wait.until(EC.presence_of_element_located((By.XPATH,'//*[@id="theDataTable_wrapper"]/div[3]/div[2]/button['+ str(attribute_dict[att]) + ']'))).click()
+                time.sleep(1)
+        else:
+            for att in listed_attributes:
+                self.clicking_show_hide_cols(self.driver)
+                time.sleep(1)
+                self.wait.until(EC.presence_of_element_located((By.XPATH,'//*[@id="theDataTable_wrapper"]/div[3]/div[2]/button['+ str(attribute_dict[att.lower()]) + ']'))).click()
+                time.sleep(1)
+
+    def select_by_status(self, listed_states, status_dict):
+        if listed_states:
+            for status in listed_states:
+                self.driver.find_element(By.ID,status_dict[status.lower()]).click()
+
+            self.driver.find_element(By.XPATH,'//*[@id="FiltersBody"]/div[1]/input[1]').click()
+            time.sleep(3)
+
+
+        select = Select(self.driver.find_element_by_name('theDataTable_length'))
+        select.select_by_value('100')
+
+    def collect_data_search_page(self,l_ordered, amount_of_data = None):
+
+        class_name = ''
+        page_index = 1
+        
+        elements = [l_ordered]
+
+        while 'disabled' not in class_name :
+           
+
+
+            time.sleep(10)
+
+            print('Getting data from page {}'.format(page_index))
+
+            #Counting how many rows of the table appear
+            table = self.driver.find_element(By.ID,'theDataTable')
+            row_count = len(table.find_elements(By.TAG_NAME,"tr")) 
+
+            #Looping table page
+            for index in range(1, row_count):
+                row = []
+                if 'status' in l_ordered:
+                    self.wait.until(EC.presence_of_element_located((By.CSS_SELECTOR,'#theDataTable > tbody > tr:nth-child('+str(index)+') > td:nth-child(3)')))
+                    status_element = self.driver.find_elements(By.CLASS_NAME,'#theDataTable > tbody > tr:nth-child('+str(index)+') > td:nth-child(3) > span')
+                    row.append(status_element.text.strip())
+                    for i, val  in enumerate(l_ordered):
+                        if val == 'status':
+                            continue
+                        
+                        self.wait.until(EC.presence_of_element_located((By.CSS_SELECTOR,'#theDataTable > tbody > tr:nth-child('+str(index)+') > td:nth-child('+str(4+i)+')')))
+                        element = self.driver.find_elements(By.CLASS_NAME,'#theDataTable > tbody > tr:nth-child('+str(index)+') > td:nth-child('+str(4+i)+')')
+                        try:
+                            row.append(element.text.strip())
+                        except:
+                            print(i, element)
+                else:
+                    for i, val  in enumerate(l_ordered):
+                        self.wait.until(EC.presence_of_element_located((By.CSS_SELECTOR,'#theDataTable > tbody > tr:nth-child('+str(index)+') > td:nth-child('+str(3+i)+')')))
+                        element = self.driver.find_elements(By.CLASS_NAME,'#theDataTable > tbody > tr:nth-child('+str(index)+') > td:nth-child('+str(3+i)+')')
+                        try:
+                            row.append(element.text.strip())
+                        except:
+                            print(i, element)
+                elements.append(row)
+
+
+
+
+            #Getting next page button
+            next_page= self.driver.find_element(By.ID,"theDataTable_next")
+
+            #Getting the class attribute of the next page button
+            class_name = next_page.get_attribute('class')
+
+            #Going to the next page
+            next_page.click()
+            page_index += 1
+            
+            if amount_of_data:
+                if len(elements) >= amount_of_data or row_count < amount_of_data :
+                    break
+                else:
+                    continue
+
+        return elements
+
+    def get_criteria(self, NCTnumber):
+    
+        url = 'https://clinicaltrials.gov/ct2/show/' + NCTnumber
+        ClinicalTrialpage = requests.get(url)
+        soup = BeautifulSoup(ClinicalTrialpage.text, 'html.parser')
+
+        wrapping_crit_class = soup.find_all("div", {"class": "tr-indent2"})
+        list_elements = wrapping_crit_class[1].find_all(re.compile("(ul|ol)"))
+        inclusion, exclusion  = ('','')
+
+
+        if not list_elements:
+            print ("WARNING: Study number " + NCTnumber + " doesn't have eligibility criteria or HTML tag format is not a list")
+        else:
+
+            if len(list_elements) == 1: 
+                try:
+                    if wrapping_crit_class[1].find(text = 'Inclusion Criteria:'):
+                        inclusion = list_elements[0].find_all("li")
+
+                    elif wrapping_crit_class[1].find(text = 'Exclusion Criteria:'):
+                        exclusion = list_elements[0].find_all("li")
+                except:
+                    print('criteria doesnt exist')
+            else:
+                inclusion = list_elements[0].find_all("li")
+                exclusion = list_elements[1].find_all("li")
+
+
+        inclusion = ' '.join([t.text.strip() for t in inclusion ])
+        exclusion = ' '.join([t.text.strip() for t in exclusion ])
+        
+        return(inclusion, exclusion)
+
+#function that gets number of patients enrolled in a study
+    def get_enrollment (self, NCTnumber):
+        url = 'https://clinicaltrials.gov/ct2/show/' + NCTnumber
+        ClinicalTrialpage = requests.get(url)
+        soup = BeautifulSoup(ClinicalTrialpage.text, 'html.parser')
+        enrollment = ''
+        wrapping_enrol_class = soup.find_all('td', {'headers':'studyInfoColData','style':"padding-left:1em"})
+        if not wrapping_enrol_class:
+            print('WARNING: Number of Participants in Study number '+ NCTnumber +' is unavailable')
+        else:
+            enrollment = wrapping_enrol_class[1]
+            enrollment = enrollment.text.split()[0]
+            if enrollment.isdigit() == False:
+                print ('WARNING: Number of Participants in Study number '+ NCTnumber +' is unavailable')
+            else:
+                return(enrollment)
+
+    
+        
+    def __call__(self, condition, listed_attributes, listed_states, amount_of_data):
+        
+        self.driver.get('https://clinicaltrials.gov/ct2/results?cond=' + condition + '&rank=1&view=record#rowId0')
+        self.select_attributes_to_show(listed_attributes, self.attribute_dict)
+     
+        try:
+            self.select_by_status(listed_states, self.status_dict)
+        except:
+            print('select by status is a problem')
+        n = []
+        for i in listed_attributes: 
+            n.append(self.attribute_dict[i.lower()])
+        attribute_ordered = [list(self.attribute_dict.keys())[list(self.attribute_dict.values()).index(i)]for i in sorted(n)]
+
+        search_data = self.collect_data_search_page(attribute_ordered, amount_of_data=amount_of_data)
+        nct_numbers = [e[search_data[0].index('nct number')] for e in search_data[1:]]
+        search_data[0].extend(['inclusion', 'exclusion', 'enrollment'])
+        for index, nct in enumerate(nct_numbers):
+            if index % 100 == 0 and index!= 0:
+                print("Collected Data from {} Studies: ".format(index))
+
+            inc, exc = self.get_criteria(nct)
+            enrol = self.get_enrollment(nct)
+            search_data[index + 1].extend([inc, exc, enrol])
+        return search_data
+#     except: 
+#         print('no data available with the specified status')
+
+        

app.py

@@ -1,63 +1,66 @@
+import os
+from llama_index.core.query_engine import NLSQLTableQueryEngine
+import pickle
+import pandas as pd
+import sqlalchemy as sa
+#from llama_index.llms.replicate import Replicate
+from llama_index.llms.groq import Groq
+from llama_index.llms.llama_cpp.llama_utils import messages_to_prompt, completion_to_prompt
+from llama_index.core import SQLDatabase
+import requests
+import re
+from llama_index.llms.openai import OpenAI
+from OpenAITools.FetchTools import fetch_clinical_trials, fetch_clinical_trials_jp
 import gradio as gr
-from huggingface_hub import InferenceClient
-
-"""
-For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
-"""
-client = InferenceClient("HuggingFaceH4/zephyr-7b-beta")
-
-
-def respond(
-    message,
-    history: list[tuple[str, str]],
-    system_message,
-    max_tokens,
-    temperature,
-    top_p,
-):
-    messages = [{"role": "system", "content": system_message}]
-
-    for val in history:
-        if val[0]:
-            messages.append({"role": "user", "content": val[0]})
-        if val[1]:
-            messages.append({"role": "assistant", "content": val[1]})
-
-    messages.append({"role": "user", "content": message})
-
-    response = ""
-
-    for message in client.chat_completion(
-        messages,
-        max_tokens=max_tokens,
-        stream=True,
-        temperature=temperature,
-        top_p=top_p,
-    ):
-        token = message.choices[0].delta.content
-
-        response += token
-        yield response
-
-"""
-For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface
-"""
-demo = gr.ChatInterface(
-    respond,
-    additional_inputs=[
-        gr.Textbox(value="You are a friendly Chatbot.", label="System message"),
-        gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),
-        gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
-        gr.Slider(
-            minimum=0.1,
-            maximum=1.0,
-            value=0.95,
-            step=0.05,
-            label="Top-p (nucleus sampling)",
+
+
+def custom_completion_to_prompt(completion:str) ->str:
+    return completion_to_prompt(
+        completion, system_prompt=(
+            "You are a sophisticated AI model designed to assist users in finding relevant clinical trials efficiently.  **Accurate Retrieval**: Utilize the RAG mechanism to retrieve the most relevant and current clinical trial information from reliable databases. Ensure the information is accurate and cite the sources when necessary. Make decisions based solely on the information provided by the retriever."
         ),
-    ],
-)
+    )
+
+def RetriviralClinicalTrrial_multi(message,history,cancer_name):
+    df = fetch_clinical_trials(cancer_name)
+    engine = sa.create_engine("sqlite:///:memory:")
+    sql_database = df.to_sql(name='clinical_study', con=engine, if_exists='replace', index=False)
+    database = SQLDatabase(engine)
+    # The replicate endpoint
+    LLAMA3_8B = "Llama3-8b-8192"
+    LLAMA3_70b = "Llama3-70b-8192"
+    Mixtral  = "mixtral-8x7b-32768" 
+
+    '''llm = OpenAI(
+    model='gpt-3.5-turbo',
+    temperature=0.01,
+    context_window=4096,
+    completion_to_prompt=custom_completion_to_prompt,
+    messages_to_prompt=messages_to_prompt,)'''
+
+    llm = Groq(
+        model=LLAMA3_70b,
+        temperature=0.01,
+        context_window=4096,
+        completion_to_prompt=custom_completion_to_prompt,
+        messages_to_prompt=messages_to_prompt,
+    )
+    query_engine = NLSQLTableQueryEngine(
+    sql_database = database,
+    tables=["clinical_study"],llm=llm)
+    response = query_engine.query(message)
+    return response.response
+
+with gr.Blocks() as demo:
+    cancer_name = gr.Textbox("lung cancer", label="cancer name")
 
+    gr.ChatInterface(
+        RetriviralClinicalTrrial_multi, additional_inputs=[cancer_name],
+        title="Assistant for Clinical traial Search",
+        description='リアルタイムで日本で行われているClinical trialの情報を入手して治験を探すお手伝いをします。対象となるがんの名前を上に入力して、まずNCTIDを聞いてください。その後にそれぞれのNCTIDのを聴いてください。その後興味のある内容　Title,Cancer,Summary,Japanes Locations,Eligibility Criteriaなどを聞いてみてください（スペルの打ち間違いに注意）例えば「NCT04270591のTitleを教えて』のように尋ねてください',
+        theme="soft",
+        examples=[["NCTIDを全て教えて"], ["のTitleを教えて"], ["の対象Cancerを教えて"], ["のSummaryを教えて"], ["のJapanes Locationsを教えて"], ["のEligibility Criteriaを教えて"], ["のPrimary Completion Dateを教えて"]]
+    )
 
 if __name__ == "__main__":
     demo.launch()

requirements.txt

@@ -1 +1,72 @@
-huggingface_hub==0.22.2
+aiohttp==3.9.5
+aiosignal==1.3.1
+bio==1.7.1
+biopython==1.83
+biothings-client==0.3.1
+dataclasses-json==0.6.6
+Deprecated==1.2.14
+dirtyjson==1.0.8
+diskcache==5.6.3
+distro==1.9.0
+frozenlist==1.4.1
+gprofiler-official==1.0.0
+greenlet==3.0.3
+hpack==4.0.0
+jsonpatch==1.33
+langchain==0.2.2
+langchain-community==0.2.4
+langchain-core==0.2.4
+langchain-experimental==0.0.60
+langchain-openai==0.1.8
+langchain-text-splitters==0.2.1
+langsmith==0.1.71
+llama-index==0.10.43
+llama-index-agent-openai==0.2.7
+llama-index-cli==0.1.12
+llama-index-core==0.10.43
+llama-index-embeddings-openai==0.1.10
+llama-index-indices-managed-llama-cloud==0.1.6
+llama-index-legacy==0.9.48
+llama-index-llms-groq==0.1.4
+llama-index-llms-llama-cpp==0.1.3
+llama-index-llms-openai==0.1.22
+llama-index-llms-openai-like==0.1.3
+llama-index-llms-replicate==0.1.3
+llama-index-multi-modal-llms-openai==0.1.6
+llama-index-program-openai==0.1.6
+llama-index-question-gen-openai==0.1.3
+llama-index-readers-file==0.1.23
+llama-index-readers-llama-parse==0.1.4
+llama-parse==0.4.4
+llama_cpp_python==0.2.77
+llamaindex-py-client==0.1.19
+marshmallow==3.21.2
+multidict==6.0.5
+munkres==1.1.4
+mygene==3.2.2
+mypy-extensions==1.0.0
+natsort==8.4.0
+networkx==3.3
+nltk
+openai
+packaging==23.2
+pooch==1.8.1
+pypdf==4.2.0
+pytrials==1.0.0
+regex==2024.5.15
+replicate==0.26.0
+safetensors
+setuptools==70.0.0
+SQLAlchemy==2.0.30
+striprtf==0.0.26
+tenacity==8.3.0
+tiktoken==0.7.0
+tokenizers==0.19.1
+transformers==4.41.2
+typer==0.12.3
+typer-slim==0.12.3
+typing-inspect==0.9.0
+wheel==0.43.0
+wikipedia==1.4.0
+wrapt==1.16.0
+yarl==1.9.4