Major update. Support for 15 LLMs, World Flora Online taxonomy validation, geolocation, 2 OCR methods, significant UI changes, stability improvements, consistent JSON parsing

app.py

@@ -407,11 +407,14 @@ class JSONReport:
             return json.dumps(json_obj, indent=4, sort_keys=False)
 
     
+
 def does_private_file_exist():
     dir_home = os.path.dirname(os.path.dirname(__file__))
     path_cfg_private = os.path.join(dir_home, 'PRIVATE_DATA.yaml')
     return os.path.exists(path_cfg_private)
 
+
+
 def setup_streamlit_config(dir_home):
     # Define the directory path and filename
     dir_path = os.path.join(dir_home, ".streamlit")
@@ -436,6 +439,8 @@ def setup_streamlit_config(dir_home):
     with open(file_path, "w") as f:
         f.write(config_content.strip())
 
+
+
 def display_scrollable_results(JSON_results, test_results, OPT2, OPT3):
     """
     Display the results from JSON_results in a scrollable container.
@@ -479,6 +484,7 @@ def display_scrollable_results(JSON_results, test_results, OPT2, OPT3):
         st.markdown(results_html, unsafe_allow_html=True)
 
 
+
 def display_test_results(test_results, JSON_results, llm_version):
     if llm_version == 'gpt':
         OPT1, OPT2, OPT3 = TestOptionsGPT.get_options()
@@ -543,9 +549,13 @@ def display_test_results(test_results, JSON_results, llm_version):
     # proportional_rain("🥇", success_count, "💔", failure_count, font_size=72, falling_speed=5, animation_length="infinite")
     rain_emojis(test_results)
 
+
+
 def add_emoji_delay():
     time.sleep(0.3)
 
+
+
 def rain_emojis(test_results):
     # test_results = {
     #     'test1': True,   # Test passed
@@ -585,17 +595,23 @@ def rain_emojis(test_results):
         )
         add_emoji_delay()
 
+
+
 def format_json(json_obj):
     try:
         return json.dumps(json.loads(json_obj), indent=4, sort_keys=False)
     except:
         return json.dumps(json_obj, indent=4, sort_keys=False)
     
+
+
 def get_prompt_versions(LLM_version):
     yaml_files = [f for f in os.listdir(os.path.join(st.session_state.dir_home, 'custom_prompts')) if f.endswith('.yaml')]
 
     return yaml_files
 
+
+
 def get_private_file():
     dir_home = os.path.dirname(os.path.dirname(__file__))
     path_cfg_private = os.path.join(dir_home, 'PRIVATE_DATA.yaml')
@@ -603,17 +619,7 @@ def get_private_file():
 
 
 
-# def create_private_file():
-#     st.session_state.proceed_to_main = False
-
-#     if st.session_state.private_file:
-#         cfg_private = get_private_file()
-#         create_private_file_0(cfg_private)
-#     else:
-#         st.title("VoucherVision")
-#         create_private_file_0()
-
-def create_private_file():
+def create_private_file(): #################################################################################### UPDATE
     st.session_state.proceed_to_main = False
     st.title("VoucherVision")
     col_private,_= st.columns([12,2])
@@ -823,7 +829,7 @@ def test_API(api, message_loc, cfg_private,openai_api_key,azure_openai_api_versi
        
 
 def save_changes_to_API_keys(cfg_private,openai_api_key,azure_openai_api_version,azure_openai_api_key,
-                             azure_openai_api_base,azure_openai_organization,azure_openai_api_type,google_vision,google_palm):
+                             azure_openai_api_base,azure_openai_organization,azure_openai_api_type,google_vision,google_palm): ########################################  UPDATE
     # Update the configuration dictionary with the new values
     cfg_private['openai']['OPENAI_API_KEY'] = openai_api_key 
 
@@ -833,9 +839,9 @@ def save_changes_to_API_keys(cfg_private,openai_api_key,azure_openai_api_version
     cfg_private['openai_azure']['openai_organization'] = azure_openai_organization
     cfg_private['openai_azure']['openai_api_type'] = azure_openai_api_type
 
-    cfg_private['google_cloud']['path_json_file'] = google_vision
+    cfg_private['google']['GOOGLE_APPLICATION_CREDENTIALS'] = google_vision
 
-    cfg_private['google_palm']['google_palm_api'] = google_palm
+    cfg_private['google']['GOOGLE_PALM_API'] = google_palm
     # Call the function to write the updated configuration to the YAML file
     write_config_file(cfg_private, st.session_state.dir_home, filename="PRIVATE_DATA.yaml")
     st.session_state.private_file = does_private_file_exist()
@@ -1885,12 +1891,7 @@ def content_api_check():
 
 def content_collage_overlay():
     st.write("---")
-    st.header('LeafMachine2 Label Collage')    
-    col_cropped_1, col_cropped_2 = st.columns([4,4])   
-
-    st.write("---")
-    st.header('OCR Overlay Image')    
-    col_ocr_1, col_ocr_2 = st.columns([4,4])        
+    col_collage, col_overlay = st.columns([4,4])   
     
     demo_text_h = f"Google_OCR_Handwriting:\nHERBARIUM OF MARCUS W. LYON , JR . Tracaulon sagittatum Indiana : Porter Co. incal Springs edge wet subdunal woods 1927 TX 11 Ilowers pink UNIVERSITE HERBARIUM MICH University of Michigan Herbarium 1439649 copyright reserved PERSICARIA FEB 2 6 1965 cm "
     demo_text_tr = f"trOCR:\nherbarium of marcus w. lyon jr. : : : tracaulon sagittatum indiana porter co. incal springs TX 11 Ilowers pink  1439649 copyright reserved D H U Q "
@@ -1900,7 +1901,8 @@ def content_collage_overlay():
     demo_text_trh = demo_text_h + '\n' + demo_text_tr
     demo_text_trp = demo_text_p + '\n' + demo_text_tr
 
-    with col_cropped_1:
+    with col_collage:
+        st.header('LeafMachine2 Label Collage')    
         default_crops = st.session_state.config['leafmachine']['cropped_components']['save_cropped_annotations']
         st.write("Prior to transcription, use LeafMachine2 to crop all labels from input images to create label collages for each specimen image. (Requires GPU)")
         st.session_state.config['leafmachine']['use_RGB_label_images'] = st.checkbox("Use LeafMachine2 label collage for transcriptions", st.session_state.config['leafmachine'].get('use_RGB_label_images', False))
@@ -1910,7 +1912,6 @@ def content_collage_overlay():
                 options=['ruler', 'barcode','label', 'colorcard','map','envelope','photo','attached_item','weights',
                 'leaf_whole', 'leaf_partial', 'leaflet', 'seed_fruit_one', 'seed_fruit_many', 'flower_one', 'flower_many', 'bud','specimen','roots','wood'],default=default_crops)
         st.session_state.config['leafmachine']['cropped_components']['save_cropped_annotations'] = option_selected_crops
-    with col_cropped_2:
         # Load the image only if it's not already in the session state
         if "demo_collage" not in st.session_state:
             # ba = os.path.join(st.session_state.dir_home, 'demo', 'ba', 'ba2.png')
@@ -1918,12 +1919,14 @@ def content_collage_overlay():
             st.session_state["demo_collage"] = Image.open(ba)
 
         # Display the image
-        st.image(st.session_state["demo_collage"], caption='LeafMachine2 Collage', output_format="PNG", width=500)
-        # st.image(st.session_state["demo_collage"], caption='LeafMachine2 Collage', output_format="JPEG")
+        with st.expander(":frame_with_picture: View an example of the LeafMachine2 collage image"):
+            st.image(st.session_state["demo_collage"], caption='LeafMachine2 Collage', output_format="PNG")
+            # st.image(st.session_state["demo_collage"], caption='LeafMachine2 Collage', output_format="JPEG")
         
 
 
-    with col_ocr_1:
+    with col_overlay:
+        st.header('OCR Overlay Image')    
         options = [":rainbow[Printed + Handwritten]", "Printed", "Use both models"]
         captions = [
             "Works well for both printed and handwritten text", 
@@ -1933,7 +1936,7 @@ def content_collage_overlay():
 
         st.write('This will plot bounding boxes around all text that Google Vision was able to detect. If there are no boxes around text, then the OCR failed, so that missing text will not be seen by the LLM when it is creating the JSON object. The created image will be viewable in the VoucherVisionEditor.')
         
-        do_create_OCR_helper_image = st.checkbox("Create image showing an overlay of the OCR detections",value=st.session_state.config['leafmachine']['do_create_OCR_helper_image'])
+        do_create_OCR_helper_image = st.checkbox("Create image showing an overlay of the OCR detections",value=st.session_state.config['leafmachine']['do_create_OCR_helper_image'],disabled=True)
         st.session_state.config['leafmachine']['do_create_OCR_helper_image'] = do_create_OCR_helper_image
         
         
@@ -1985,14 +1988,15 @@ def content_collage_overlay():
         elif (OCR_option == 'hand') and do_use_trOCR:
             st.text_area(label='Handwritten/Printed + trOCR',placeholder=demo_text_trh,disabled=True, label_visibility='visible', height=150)
         
-    with col_ocr_2:
+
         if "demo_overlay" not in st.session_state:
             # ocr = os.path.join(st.session_state.dir_home,'demo', 'ba','ocr.png')
-            ocr = os.path.join(st.session_state.dir_home,'demo', 'ba','ocr2_low.png')
+            ocr = os.path.join(st.session_state.dir_home,'demo', 'ba','ocr2.png')
             st.session_state["demo_overlay"] = Image.open(ocr)
         
-        st.image(st.session_state["demo_overlay"], caption='OCR Overlay Images', output_format = "PNG", width=500)
-        # st.image(st.session_state["demo_overlay"], caption='OCR Overlay Images', output_format = "JPEG")
+        with st.expander(":frame_with_picture: View an example of the OCR overlay image"):
+            st.image(st.session_state["demo_overlay"], caption='OCR Overlay Images', output_format = "PNG")
+            # st.image(st.session_state["demo_overlay"], caption='OCR Overlay Images', output_format = "JPEG")
 
 
 
@@ -2299,7 +2303,7 @@ st.set_page_config(layout="wide", page_icon='img/icon.ico', page_title='VoucherV
 
 # Parse the 'is_hf' argument and set it in session state
 if 'is_hf' not in st.session_state:
-    st.session_state['is_hf'] = True
+    st.session_state['is_hf'] = False
   
 
 #################################################################################################################################################

vouchervision/API_validation.py

@@ -56,10 +56,10 @@ class APIvalidation:
                 # Initialize the Azure OpenAI client
                 model = AzureChatOpenAI(
                     deployment_name = 'gpt-35-turbo',#'gpt-35-turbo',
-                    openai_api_version = self.cfg_private['openai_azure']['api_version'],
-                    openai_api_key = self.cfg_private['openai_azure']['openai_api_key'],
-                    azure_endpoint = self.cfg_private['openai_azure']['openai_api_base'],
-                    openai_organization = self.cfg_private['openai_azure']['openai_organization'],
+                    openai_api_version = self.cfg_private['openai_azure']['OPENAI_API_VERSION'],
+                    openai_api_key = self.cfg_private['openai_azure']['OPENAI_API_KEY'],
+                    azure_endpoint = self.cfg_private['openai_azure']['OPENAI_API_BASE'],
+                    openai_organization = self.cfg_private['openai_azure']['OPENAI_ORGANIZATION'],
                 )
                 msg = HumanMessage(content="hello")
                 # self.llm_object.temperature = self.config.get('temperature')
@@ -103,7 +103,7 @@ class APIvalidation:
     def check_mistral_api_key(self):
         try:
             if not self.is_hf:
-                client = MistralClient(api_key=self.cfg_private['mistral']['mistral_key'])
+                client = MistralClient(api_key=self.cfg_private['mistral']['MISTRAL_API_KEY'])
             else:
                 client = MistralClient(api_key=os.getenv('MISTRAL_API_KEY'))
 
@@ -184,7 +184,8 @@ class APIvalidation:
                 print(f"palm2 fail [{test_response_palm}]")
 
         except Exception as e:
-            print(f"palm2 fail2 [{test_response_palm}]")
+            # print(f"palm2 fail2 [{e}]")
+            print(f"palm2 fail2")
             
 
         try:
@@ -198,7 +199,8 @@ class APIvalidation:
                 print(f"gemini fail [{test_response_gemini}]")
 
         except Exception as e:
-            print(f"palm2 fail2 [{test_response_gemini}]")
+            # print(f"palm2 fail2 [{e}]")
+            print(f"palm2 fail2")
 
         return results
 
@@ -210,12 +212,12 @@ class APIvalidation:
             credentials = service_account.Credentials.from_service_account_info(json.loads(creds_json_str))
             return credentials
         else:
-            with open(self.cfg_private['google_cloud']['GOOGLE_APPLICATION_CREDENTIALS'], 'r') as file:
+            with open(self.cfg_private['google']['GOOGLE_APPLICATION_CREDENTIALS'], 'r') as file:
                 data = json.load(file)
-                creds_json_str = json.dumps(data)
-                credentials = service_account.Credentials.from_service_account_info(json.loads(creds_json_str))
-                os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = creds_json_str
-                return credentials
+            creds_json_str = json.dumps(data)
+            credentials = service_account.Credentials.from_service_account_info(json.loads(creds_json_str))
+            os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = creds_json_str
+            return credentials
 
 
 
@@ -223,28 +225,30 @@ class APIvalidation:
         missing_keys = []
         present_keys = []
 
-        if not self.is_hf:
-            k_OPENAI_API_KEY = self.cfg_private['openai']['OPENAI_API_KEY']
-            k_openai_azure = self.cfg_private['openai_azure']['api_version']
-            # k_google_palm_api = self.cfg_private['google_palm']['google_palm_api']
-
-            k_project_id = self.cfg_private['google_palm']['project_id']
-            k_location = self.cfg_private['google_palm']['location']
-            k_google_application_credentials = self.cfg_private['google_cloud']['GOOGLE_APPLICATION_CREDENTIALS']
-            
-            k_mistral = self.cfg_private['mistral']['mistral_key']
-            k_here = self.cfg_private['here']['api_key']
-            k_opencage = self.cfg_private['open_cage_geocode']['api_key']
-        else:
+        if self.is_hf:
             k_OPENAI_API_KEY = os.getenv('OPENAI_API_KEY')
             k_openai_azure = os.getenv('AZURE_API_VERSION')
-            # k_google_palm_api = os.getenv('PALM_API_KEY')
 
+            k_google_application_credentials = os.getenv('GOOGLE_APPLICATION_CREDENTIALS')
             k_project_id = os.getenv('GOOGLE_PROJECT_ID')
             k_location = os.getenv('GOOGLE_LOCATION')
+            
             k_mistral = os.getenv('MISTRAL_API_KEY')
             k_here = os.getenv('HERE_API_KEY')
-            k_opencage = os.getenv('open_cage_geocode')
+            k_opencage = os.getenv('OPENCAGE_API_KEY')
+        else:
+            k_OPENAI_API_KEY = self.cfg_private['openai']['OPENAI_API_KEY']
+            k_openai_azure = self.cfg_private['openai_azure']['OPENAI_API_VERSION']
+
+            k_project_id = self.cfg_private['google']['GOOGLE_PROJECT_ID']
+            k_location = self.cfg_private['google']['GOOGLE_LOCATION']
+            k_google_application_credentials = self.cfg_private['google']['GOOGLE_APPLICATION_CREDENTIALS']
+            
+            k_mistral = self.cfg_private['mistral']['MISTRAL_API_KEY']
+            k_here = self.cfg_private['here']['API_KEY']
+            k_opencage = self.cfg_private['open_cage_geocode']['API_KEY']
+
+            
 
 
         # Check each key and add to the respective list
@@ -285,7 +289,7 @@ class APIvalidation:
 
         # Google PALM2/Gemini key check
         if self.has_API_key(k_google_application_credentials) and self.has_API_key(k_project_id) and self.has_API_key(k_location): ##################
-            vertexai.init(project=os.getenv('GOOGLE_PROJECT_ID'), location=os.getenv('GOOGLE_LOCATION'), credentials=self.get_google_credentials())
+            vertexai.init(project=k_project_id, location=k_location, credentials=self.get_google_credentials())
             google_results = self.check_google_vertex_genai_api_key()
             if google_results['palm2']:
                 present_keys.append('Palm2 (Valid)')

vouchervision/VoucherVision_Config_Builder.py

@@ -594,13 +594,14 @@ def check_if_usable(is_hf): ####################################################
 
         has_key_openai = has_API_key(cfg_private['openai']['OPENAI_API_KEY'])
 
-        has_key_azure_openai = has_API_key(cfg_private['openai_azure']['api_version']) 
-
-        has_key_palm2 = has_API_key(cfg_private['google_palm']['google_palm_api'])
+        has_key_azure_openai = has_API_key(cfg_private['openai_azure']['OPENAI_API_VERSION']) 
         
-        has_key_google_OCR = has_API_key(cfg_private['google_cloud']['path_json_file'])
+        has_key_google_OCR = has_API_key(cfg_private['google']['GOOGLE_APPLICATION_CREDENTIALS'])
+
+        has_key_MISTRAL = has_API_key(cfg_private['mistral']['MISTRAL_API_KEY'])
 
-        if has_key_google_OCR and (has_key_azure_openai or has_key_openai or has_key_palm2):
+
+        if has_key_google_OCR and (has_key_azure_openai or has_key_openai or has_key_MISTRAL):
             return True
         else:
             return False
@@ -614,13 +615,11 @@ def check_API_key(dir_home, api_version):
 
     has_key_azure_openai = has_API_key(cfg_private['openai_azure']['api_version']) 
 
-    has_key_palm2 = has_API_key(cfg_private['google_palm']['google_palm_api'])
+    # has_key_palm2 = has_API_key(cfg_private['google_palm']['google_palm_api'])
 
-    has_key_google_OCR = has_API_key(cfg_private['google_cloud']['path_json_file'])
+    has_key_google_OCR = has_API_key(cfg_private['google']['GOOGLE_APPLICATION_CREDENTIALS'])
 
-    if api_version == 'palm' and has_key_palm2:
-        return True
-    elif api_version in ['gpt','openai'] and has_key_openai:
+    if api_version in ['gpt','openai'] and has_key_openai:
         return True
     elif api_version in ['gpt-azure', 'azure_openai'] and has_key_azure_openai:
         return True

vouchervision/utils_VoucherVision.py

@@ -417,7 +417,7 @@ class VoucherVision():
             credentials = service_account.Credentials.from_service_account_info(json.loads(creds_json_str))
             return credentials
         else:
-            with open(self.cfg_private['google_cloud']['GOOGLE_APPLICATION_CREDENTIALS'], 'r') as file:
+            with open(self.cfg_private['google']['GOOGLE_APPLICATION_CREDENTIALS'], 'r') as file:
                 data = json.load(file)
                 creds_json_str = json.dumps(data)
                 credentials = service_account.Credentials.from_service_account_info(json.loads(creds_json_str))
@@ -432,15 +432,15 @@ class VoucherVision():
             self.cfg_private = get_cfg_from_full_path(self.path_cfg_private)
 
             k_openai = self.cfg_private['openai']['OPENAI_API_KEY']
-            k_openai_azure = self.cfg_private['openai_azure']['api_version']
+            k_openai_azure = self.cfg_private['openai_azure']['OPENAI_API_VERSION']
 
-            k_google_project_id = self.cfg_private['google_palm']['project_id']
-            k_google_location = self.cfg_private['google_palm']['location']
-            k_google_application_credentials = self.cfg_private['google_cloud']['GOOGLE_APPLICATION_CREDENTIALS']
+            k_google_project_id = self.cfg_private['google']['GOOGLE_PROJECT_ID']
+            k_google_location = self.cfg_private['google']['GOOGLE_LOCATION']
+            k_google_application_credentials = self.cfg_private['google']['GOOGLE_APPLICATION_CREDENTIALS']
             
-            k_mistral = self.cfg_private['mistral']['mistral_key']
-            k_here = self.cfg_private['here']['api_key']
-            k_opencage = self.cfg_private['open_cage_geocode']['api_key']
+            k_mistral = self.cfg_private['mistral']['MISTRAL_API_KEY']
+            k_here = self.cfg_private['here']['API_KEY']
+            k_opencage = self.cfg_private['open_cage_geocode']['API_KEY']
         else:
             self.dir_home = os.path.dirname(os.path.dirname(__file__))
             self.path_cfg_private = None
@@ -471,10 +471,11 @@ class VoucherVision():
 
         
         ### Google - OCR, Palm2, Gemini
-        if self.is_hf:
-            if self.has_key_google_application_credentials and self.has_key_google_project_id and self.has_key_google_location:
+        if self.has_key_google_application_credentials and self.has_key_google_project_id and self.has_key_google_location:
+            if self.is_hf:
                 vertexai.init(project=os.getenv('GOOGLE_PROJECT_ID'), location=os.getenv('GOOGLE_LOCATION'), credentials=self.get_google_credentials())
-        
+            else:
+                vertexai.init(project=k_google_project_id, location=k_google_location, credentials=self.get_google_credentials())
 
         ### OpenAI
         if self.has_key_openai:
@@ -502,10 +503,10 @@ class VoucherVision():
                 # Initialize the Azure OpenAI client
                 self.llm = AzureChatOpenAI(
                     deployment_name = 'gpt-35-turbo',#'gpt-35-turbo',
-                    openai_api_version = self.cfg_private['openai_azure']['api_version'],
-                    openai_api_key = self.cfg_private['openai_azure']['openai_api_key'],
-                    azure_endpoint = self.cfg_private['openai_azure']['openai_api_base'],
-                    openai_organization = self.cfg_private['openai_azure']['openai_organization'],
+                    openai_api_version = self.cfg_private['openai_azure']['OPENAI_API_VERSION'],
+                    openai_api_key = self.cfg_private['openai_azure']['OPENAI_API_KEY'],
+                    azure_endpoint = self.cfg_private['openai_azure']['OPENAI_API_BASE'],
+                    openai_organization = self.cfg_private['openai_azure']['OPENAI_ORGANIZATION'],
                 )
                 self.has_key_azure_openai = True
                 
@@ -515,7 +516,7 @@ class VoucherVision():
             if self.is_hf:
                 pass # Already set
             else:
-                os.environ['MISTRAL_API_KEY'] = self.cfg_private['mistral']['mistral_key']
+                os.environ['MISTRAL_API_KEY'] = self.cfg_private['mistral']['MISTRAL_API_KEY']
 
 
         ### HERE
@@ -523,8 +524,8 @@ class VoucherVision():
             if self.is_hf:
                 pass # Already set
             else:
-                os.environ['HERE_APP_ID'] = self.cfg_private['here']['app_id']
-                os.environ['HERE_API_KEY'] = self.cfg_private['here']['api_key']
+                os.environ['HERE_APP_ID'] = self.cfg_private['here']['APP_ID']
+                os.environ['HERE_API_KEY'] = self.cfg_private['here']['API_KEY']
 
 
         ### HERE
@@ -532,7 +533,7 @@ class VoucherVision():
             if self.is_hf:
                 pass # Already set
             else:
-                os.environ['OPENCAGE_API_KEY'] = self.cfg_private['open_cage_geocode']['api_key']
+                os.environ['OPENCAGE_API_KEY'] = self.cfg_private['open_cage_geocode']['API_KEY']