Upload 3 files

App_Function_Libraries/Gradio_UI/Embeddings_tab.py

@@ -7,14 +7,17 @@ import logging
 #
 # External Imports
 import gradio as gr
+from tqdm import tqdm
 
-from App_Function_Libraries.Chunk_Lib import improved_chunking_process, determine_chunk_position
+from App_Function_Libraries.Chunk_Lib import improved_chunking_process, chunk_for_embedding
 #
 # Local Imports
 from App_Function_Libraries.DB.DB_Manager import get_all_content_from_database
 from App_Function_Libraries.RAG.ChromaDB_Library import chroma_client, \
-    store_in_chroma
-from App_Function_Libraries.RAG.Embeddings_Create import create_embedding
+    store_in_chroma, situate_context
+from App_Function_Libraries.RAG.Embeddings_Create import create_embedding, create_embeddings_batch
+
+
 #
 ########################################################################################################################
 #
@@ -174,17 +177,23 @@ def create_view_embeddings_tab():
                     value="words"
                 )
                 max_chunk_size = gr.Slider(
-                    minimum=1, maximum=8000, step=1, value=500,
+                    minimum=1, maximum=8000, step=5, value=500,
                     label="Max Chunk Size"
                 )
                 chunk_overlap = gr.Slider(
-                    minimum=0, maximum=5000, step=1, value=200,
+                    minimum=0, maximum=5000, step=5, value=200,
                     label="Chunk Overlap"
                 )
                 adaptive_chunking = gr.Checkbox(
                     label="Use Adaptive Chunking",
                     value=False
                 )
+                contextual_api_choice = gr.Dropdown(
+                    choices=["Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek", "Mistral", "OpenRouter", "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM", "ollama", "HuggingFace"],
+                    label="Select API for Contextualized Embeddings",
+                    value="OpenAI"
+                )
+                contextual_api_key = gr.Textbox(label="API Key", lines=1)
 
         def get_items_with_embedding_status():
             try:
@@ -242,7 +251,7 @@ def create_view_embeddings_tab():
                 logging.error(f"Error in check_embedding_status: {str(e)}")
                 return f"Error processing item: {selected_item}. Details: {str(e)}", "", ""
 
-        def create_new_embedding_for_item(selected_item, provider, model, api_url, method, max_size, overlap, adaptive, item_mapping):
+        def create_new_embedding_for_item(selected_item, provider, model, api_url, method, max_size, overlap, adaptive, item_mapping, contextual_api_choice=None):
             if not selected_item:
                 return "Please select an item", "", ""
 
@@ -263,31 +272,30 @@ def create_view_embeddings_tab():
                     'adaptive': adaptive
                 }
 
-                chunks = improved_chunking_process(item['content'], chunk_options)
+                logging.info(f"Chunking content for item: {item['title']} (ID: {item_id})")
+                chunks = chunk_for_embedding(item['content'], item['title'], chunk_options)
                 collection_name = "all_content_embeddings"
                 collection = chroma_client.get_or_create_collection(name=collection_name)
 
                 # Delete existing embeddings for this item
                 existing_ids = [f"doc_{item_id}_chunk_{i}" for i in range(len(chunks))]
                 collection.delete(ids=existing_ids)
+                logging.info(f"Deleted {len(existing_ids)} existing embeddings for item {item_id}")
 
-                for i, chunk in enumerate(chunks):
+                texts, ids, metadatas = [], [], []
+                chunk_count = 0
+                logging.info("Generating contextual summaries and preparing chunks for embedding")
+                for i, chunk in tqdm(enumerate(chunks), total=len(chunks), desc="Processing chunks"):
                     chunk_text = chunk['text']
                     chunk_metadata = chunk['metadata']
-                    chunk_position = determine_chunk_position(chunk_metadata['relative_position'])
-
-                    chunk_header = f"""
-                    Original Document: {item['title']}
-                    Chunk: {i + 1} of {len(chunks)}
-                    Position: {chunk_position}
-                    Header: {chunk_metadata.get('header_text', 'N/A')}
-        
-                    --- Chunk Content ---
-                    """
-
-                    full_chunk_text = chunk_header + chunk_text
+                    if chunk_count == 0:
+                        chunk_count = 1
+                    # Generate contextual summary
+                    logging.debug(f"Generating contextual summary for chunk {chunk_count}")
+                    context = situate_context(contextual_api_choice, item['content'], chunk_text)
+                    contextualized_text = f"{chunk_text}\n\nContextual Summary: {context}"
+
                     chunk_id = f"doc_{item_id}_chunk_{i}"
-                    embedding = create_embedding(full_chunk_text, provider, model, api_url)
                     metadata = {
                         "media_id": str(item_id),
                         "chunk_index": i,
@@ -298,13 +306,26 @@ def create_view_embeddings_tab():
                         "adaptive_chunking": adaptive,
                         "embedding_model": model,
                         "embedding_provider": provider,
+                        "original_text": chunk_text,
+                        "contextual_summary": context,
                         **chunk_metadata
                     }
-                    store_in_chroma(collection_name, [full_chunk_text], [embedding], [chunk_id], [metadata])
 
-                embedding_preview = str(embedding[:50])
-                status = f"New embeddings created and stored for item: {item['title']} (ID: {item_id})"
-                return status, f"First 50 elements of new embedding:\n{embedding_preview}", json.dumps(metadata, indent=2)
+                    texts.append(contextualized_text)
+                    ids.append(chunk_id)
+                    metadatas.append(metadata)
+                    chunk_count = chunk_count+1
+
+                # Create embeddings in batch
+                logging.info(f"Creating embeddings for {len(texts)} chunks")
+                embeddings = create_embeddings_batch(texts, provider, model, api_url)
+
+                # Store in Chroma
+                store_in_chroma(collection_name, texts, embeddings, ids, metadatas)
+
+                embedding_preview = str(embeddings[0][:50]) if embeddings else "No embeddings created"
+                status = f"New contextual embeddings created and stored for item: {item['title']} (ID: {item_id})"
+                return status, f"First 50 elements of new embedding:\n{embedding_preview}", json.dumps(metadatas[0], indent=2)
             except Exception as e:
                 logging.error(f"Error in create_new_embedding_for_item: {str(e)}")
                 return f"Error creating embedding: {str(e)}", "", ""
@@ -321,7 +342,7 @@ def create_view_embeddings_tab():
         create_new_embedding_button.click(
             create_new_embedding_for_item,
             inputs=[item_dropdown, embedding_provider, embedding_model, embedding_api_url,
-                    chunking_method, max_chunk_size, chunk_overlap, adaptive_chunking, item_mapping],
+                    chunking_method, max_chunk_size, chunk_overlap, adaptive_chunking, item_mapping, contextual_api_choice],
             outputs=[embedding_status, embedding_preview, embedding_metadata]
         )
         embedding_provider.change(

App_Function_Libraries/Gradio_UI/Search_Tab.py

@@ -66,6 +66,8 @@ def update_content_for_version(selected_item, item_mapping, selected_version):
     return "", "", ""
 
 def format_as_html(content, title):
+    if content is None:
+        content = "No content available"
     escaped_content = html.escape(content)
     formatted_content = escaped_content.replace('\n', '<br>')
     return f"""
@@ -79,9 +81,9 @@ def format_as_html(content, title):
 
 def create_search_tab():
     with gr.TabItem("Search / Detailed View"):
+        gr.Markdown("# Search across all ingested items in the Database")
         with gr.Row():
             with gr.Column(scale=1):
-                gr.Markdown("# Search across all ingested items in the Database")
                 gr.Markdown("by Title / URL / Keyword / or Content via SQLite Full-Text-Search")
                 search_query_input = gr.Textbox(label="Search Query", placeholder="Enter your search query here...")
                 search_type_input = gr.Radio(choices=["Title", "URL", "Keyword", "Content"], value="Title",

App_Function_Libraries/Gradio_UI/View_DB_Items_tab.py

@@ -10,7 +10,8 @@ import gradio as gr
 #
 # Local Imports
 from App_Function_Libraries.DB.DB_Manager import view_database, get_all_document_versions, \
-    fetch_paginated_data, fetch_item_details, get_latest_transcription
+    fetch_paginated_data, fetch_item_details, get_latest_transcription, list_prompts, fetch_prompt_details, \
+    load_preset_prompts
 from App_Function_Libraries.DB.SQLite_DB import get_document_version
 from App_Function_Libraries.Utils.Utils import get_database_path, format_text_with_line_breaks
 #
@@ -30,72 +31,40 @@ def create_prompt_view_tab():
                 next_page_button = gr.Button("Next Page")
                 previous_page_button = gr.Button("Previous Page")
                 pagination_info = gr.Textbox(label="Pagination Info", interactive=False)
+                prompt_selector = gr.Dropdown(label="Select Prompt to View", choices=[])
             with gr.Column():
-                results_display = gr.HTML()
-
-        # FIXME - SQL functions to be moved to DB_Manager
+                results_table = gr.HTML()
+                selected_prompt_display = gr.HTML()
 
         def view_database(page, entries_per_page):
-            offset = (page - 1) * entries_per_page
             try:
-                with sqlite3.connect(get_database_path('prompts.db')) as conn:
-                    cursor = conn.cursor()
-                    cursor.execute('''
-                        SELECT p.name, p.details, p.system, p.user, GROUP_CONCAT(k.keyword, ', ') as keywords
-                        FROM Prompts p
-                        LEFT JOIN PromptKeywords pk ON p.id = pk.prompt_id
-                        LEFT JOIN Keywords k ON pk.keyword_id = k.id
-                        GROUP BY p.id
-                        ORDER BY p.name
-                        LIMIT ? OFFSET ?
-                    ''', (entries_per_page, offset))
-                    prompts = cursor.fetchall()
-
-                    cursor.execute('SELECT COUNT(*) FROM Prompts')
-                    total_prompts = cursor.fetchone()[0]
-
-                results = ""
-                for prompt in prompts:
-                    # Escape HTML special characters and replace newlines with <br> tags
-                    title = html.escape(prompt[0]).replace('\n', '<br>')
-                    details = html.escape(prompt[1] or '').replace('\n', '<br>')
-                    system_prompt = html.escape(prompt[2] or '')
-                    user_prompt = html.escape(prompt[3] or '')
-                    keywords = html.escape(prompt[4] or '').replace('\n', '<br>')
-
-                    results += f"""
-                    <div style="border: 1px solid #ddd; padding: 10px; margin-bottom: 20px;">
-                        <div style="display: grid; grid-template-columns: 1fr 1fr; gap: 10px;">
-                            <div><strong>Title:</strong> {title}</div>
-                            <div><strong>Details:</strong> {details}</div>
-                        </div>
-                        <div style="margin-top: 10px;">
-                            <strong>User Prompt:</strong>
-                            <pre style="white-space: pre-wrap; word-wrap: break-word;">{user_prompt}</pre>
-                        </div>
-                        <div style="margin-top: 10px;">
-                            <strong>System Prompt:</strong>
-                            <pre style="white-space: pre-wrap; word-wrap: break-word;">{system_prompt}</pre>
-                        </div>
-                        <div style="margin-top: 10px;">
-                            <strong>Keywords:</strong> {keywords}
-                        </div>
-                    </div>
-                    """
-
-                total_pages = (total_prompts + entries_per_page - 1) // entries_per_page
-                pagination = f"Page {page} of {total_pages} (Total prompts: {total_prompts})"
-
-                return results, pagination, total_pages
-            except sqlite3.Error as e:
-                return f"<p>Error fetching prompts: {e}</p>", "Error", 0
+                prompts, total_pages, current_page = list_prompts(page, entries_per_page)
+
+                table_html = "<table style='width:100%; border-collapse: collapse;'>"
+                table_html += "<tr><th style='border: 1px solid black; padding: 8px;'>Title</th><th style='border: 1px solid black; padding: 8px;'>Author</th></tr>"
+                prompt_choices = []
+                for prompt_name in prompts:
+                    details = fetch_prompt_details(prompt_name)
+                    if details:
+                        title, _, _, _, _ = details
+                        author = "Unknown"  # Assuming author is not stored in the current schema
+                        table_html += f"<tr><td style='border: 1px solid black; padding: 8px;'>{html.escape(title)}</td><td style='border: 1px solid black; padding: 8px;'>{html.escape(author)}</td></tr>"
+                        prompt_choices.append((title, title))  # Using title as both label and value
+                table_html += "</table>"
+
+                total_prompts = len(load_preset_prompts())  # This might be inefficient for large datasets
+                pagination = f"Page {current_page} of {total_pages} (Total prompts: {total_prompts})"
+
+                return table_html, pagination, total_pages, prompt_choices
+            except Exception as e:
+                return f"<p>Error fetching prompts: {e}</p>", "Error", 0, []
 
         def update_page(page, entries_per_page):
-            results, pagination, total_pages = view_database(page, entries_per_page)
+            results, pagination, total_pages, prompt_choices = view_database(page, entries_per_page)
             next_disabled = page >= total_pages
             prev_disabled = page <= 1
             return results, pagination, page, gr.update(interactive=not next_disabled), gr.update(
-                interactive=not prev_disabled)
+                interactive=not prev_disabled), gr.update(choices=prompt_choices)
 
         def go_to_next_page(current_page, entries_per_page):
             next_page = current_page + 1
@@ -105,23 +74,158 @@ def create_prompt_view_tab():
             previous_page = max(1, current_page - 1)
             return update_page(previous_page, entries_per_page)
 
+        def display_selected_prompt(prompt_name):
+            details = fetch_prompt_details(prompt_name)
+            if details:
+                title, description, system_prompt, user_prompt, keywords = details
+                html_content = f"""
+                <div style="border: 1px solid #ddd; padding: 10px; margin-bottom: 20px;">
+                    <h3>{html.escape(title)}</h3>
+                    <p><strong>Description:</strong> {html.escape(description or '')}</p>
+                    <div style="margin-top: 10px;">
+                        <strong>System Prompt:</strong>
+                        <pre style="white-space: pre-wrap; word-wrap: break-word;">{html.escape(system_prompt or '')}</pre>
+                    </div>
+                    <div style="margin-top: 10px;">
+                        <strong>User Prompt:</strong>
+                        <pre style="white-space: pre-wrap; word-wrap: break-word;">{html.escape(user_prompt or '')}</pre>
+                    </div>
+                    <p><strong>Keywords:</strong> {html.escape(keywords or '')}</p>
+                </div>
+                """
+                return html_content
+            else:
+                return "<p>Prompt not found.</p>"
+
         view_button.click(
             fn=update_page,
             inputs=[page_number, entries_per_page],
-            outputs=[results_display, pagination_info, page_number, next_page_button, previous_page_button]
+            outputs=[results_table, pagination_info, page_number, next_page_button, previous_page_button,
+                     prompt_selector]
         )
 
         next_page_button.click(
             fn=go_to_next_page,
             inputs=[page_number, entries_per_page],
-            outputs=[results_display, pagination_info, page_number, next_page_button, previous_page_button]
+            outputs=[results_table, pagination_info, page_number, next_page_button, previous_page_button,
+                     prompt_selector]
         )
 
         previous_page_button.click(
             fn=go_to_previous_page,
             inputs=[page_number, entries_per_page],
-            outputs=[results_display, pagination_info, page_number, next_page_button, previous_page_button]
+            outputs=[results_table, pagination_info, page_number, next_page_button, previous_page_button,
+                     prompt_selector]
+        )
+
+        prompt_selector.change(
+            fn=display_selected_prompt,
+            inputs=[prompt_selector],
+            outputs=[selected_prompt_display]
         )
+# def create_prompt_view_tab():
+#     with gr.TabItem("View Prompt Database"):
+#         gr.Markdown("# View Prompt Database Entries")
+#         with gr.Row():
+#             with gr.Column():
+#                 entries_per_page = gr.Dropdown(choices=[10, 20, 50, 100], label="Entries per Page", value=10)
+#                 page_number = gr.Number(value=1, label="Page Number", precision=0)
+#                 view_button = gr.Button("View Page")
+#                 next_page_button = gr.Button("Next Page")
+#                 previous_page_button = gr.Button("Previous Page")
+#                 pagination_info = gr.Textbox(label="Pagination Info", interactive=False)
+#             with gr.Column():
+#                 results_display = gr.HTML()
+#
+#         # FIXME - SQL functions to be moved to DB_Manager
+#
+#         def view_database(page, entries_per_page):
+#             offset = (page - 1) * entries_per_page
+#             try:
+#                 with sqlite3.connect(get_database_path('prompts.db')) as conn:
+#                     cursor = conn.cursor()
+#                     cursor.execute('''
+#                         SELECT p.name, p.details, p.system, p.user, GROUP_CONCAT(k.keyword, ', ') as keywords
+#                         FROM Prompts p
+#                         LEFT JOIN PromptKeywords pk ON p.id = pk.prompt_id
+#                         LEFT JOIN Keywords k ON pk.keyword_id = k.id
+#                         GROUP BY p.id
+#                         ORDER BY p.name
+#                         LIMIT ? OFFSET ?
+#                     ''', (entries_per_page, offset))
+#                     prompts = cursor.fetchall()
+#
+#                     cursor.execute('SELECT COUNT(*) FROM Prompts')
+#                     total_prompts = cursor.fetchone()[0]
+#
+#                 results = ""
+#                 for prompt in prompts:
+#                     # Escape HTML special characters and replace newlines with <br> tags
+#                     title = html.escape(prompt[0]).replace('\n', '<br>')
+#                     details = html.escape(prompt[1] or '').replace('\n', '<br>')
+#                     system_prompt = html.escape(prompt[2] or '')
+#                     user_prompt = html.escape(prompt[3] or '')
+#                     keywords = html.escape(prompt[4] or '').replace('\n', '<br>')
+#
+#                     results += f"""
+#                     <div style="border: 1px solid #ddd; padding: 10px; margin-bottom: 20px;">
+#                         <div style="display: grid; grid-template-columns: 1fr 1fr; gap: 10px;">
+#                             <div><strong>Title:</strong> {title}</div>
+#                             <div><strong>Details:</strong> {details}</div>
+#                         </div>
+#                         <div style="margin-top: 10px;">
+#                             <strong>User Prompt:</strong>
+#                             <pre style="white-space: pre-wrap; word-wrap: break-word;">{user_prompt}</pre>
+#                         </div>
+#                         <div style="margin-top: 10px;">
+#                             <strong>System Prompt:</strong>
+#                             <pre style="white-space: pre-wrap; word-wrap: break-word;">{system_prompt}</pre>
+#                         </div>
+#                         <div style="margin-top: 10px;">
+#                             <strong>Keywords:</strong> {keywords}
+#                         </div>
+#                     </div>
+#                     """
+#
+#                 total_pages = (total_prompts + entries_per_page - 1) // entries_per_page
+#                 pagination = f"Page {page} of {total_pages} (Total prompts: {total_prompts})"
+#
+#                 return results, pagination, total_pages
+#             except sqlite3.Error as e:
+#                 return f"<p>Error fetching prompts: {e}</p>", "Error", 0
+#
+#         def update_page(page, entries_per_page):
+#             results, pagination, total_pages = view_database(page, entries_per_page)
+#             next_disabled = page >= total_pages
+#             prev_disabled = page <= 1
+#             return results, pagination, page, gr.update(interactive=not next_disabled), gr.update(
+#                 interactive=not prev_disabled)
+#
+#         def go_to_next_page(current_page, entries_per_page):
+#             next_page = current_page + 1
+#             return update_page(next_page, entries_per_page)
+#
+#         def go_to_previous_page(current_page, entries_per_page):
+#             previous_page = max(1, current_page - 1)
+#             return update_page(previous_page, entries_per_page)
+#
+#         view_button.click(
+#             fn=update_page,
+#             inputs=[page_number, entries_per_page],
+#             outputs=[results_display, pagination_info, page_number, next_page_button, previous_page_button]
+#         )
+#
+#         next_page_button.click(
+#             fn=go_to_next_page,
+#             inputs=[page_number, entries_per_page],
+#             outputs=[results_display, pagination_info, page_number, next_page_button, previous_page_button]
+#         )
+#
+#         previous_page_button.click(
+#             fn=go_to_previous_page,
+#             inputs=[page_number, entries_per_page],
+#             outputs=[results_display, pagination_info, page_number, next_page_button, previous_page_button]
+#         )
 
 
 def format_as_html(content, title):