Upload 28 files

App_Function_Libraries/Books/Book_Ingestion_Lib.py

@@ -0,0 +1,170 @@
+# Book_Ingestion_Lib.py
+#########################################
+# Library to hold functions for ingesting book files.#
+#
+####################
+# Function List
+#
+# 1. ingest_text_file(file_path, title=None, author=None, keywords=None):
+# 2.
+#
+#
+####################
+#
+# Import necessary libraries
+import os
+import re
+from datetime import datetime
+import logging
+
+import ebooklib
+from bs4 import BeautifulSoup
+from ebooklib import epub
+#
+# Import Local
+from App_Function_Libraries.DB.DB_Manager import add_media_with_keywords
+#
+#######################################################################################################################
+# Function Definitions
+#
+
+
+
+def read_epub(file_path):
+    """Read and extract text from an EPUB file."""
+    book = epub.read_epub(file_path)
+    chapters = []
+    for item in book.get_items():
+        if item.get_type() == ebooklib.ITEM_DOCUMENT:
+            chapters.append(item.get_content())
+
+    text = ""
+    for html_content in chapters:
+        soup = BeautifulSoup(html_content, 'html.parser')
+        text += soup.get_text() + "\n\n"
+    return text
+
+
+# Ingest a text file into the database with Title/Author/Keywords
+def extract_epub_metadata(content):
+    title_match = re.search(r'Title:\s*(.*?)\n', content)
+    author_match = re.search(r'Author:\s*(.*?)\n', content)
+
+    title = title_match.group(1) if title_match else None
+    author = author_match.group(1) if author_match else None
+
+    return title, author
+
+
+def ingest_text_file(file_path, title=None, author=None, keywords=None):
+    try:
+        with open(file_path, 'r', encoding='utf-8') as file:
+            content = file.read()
+
+        # Check if it's a converted epub and extract metadata if so
+        if 'epub_converted' in (keywords or ''):
+            extracted_title, extracted_author = extract_epub_metadata(content)
+            title = title or extracted_title
+            author = author or extracted_author
+
+        # If title is still not provided, use the filename without extension
+        if not title:
+            title = os.path.splitext(os.path.basename(file_path))[0]
+
+        # If author is still not provided, set it to 'Unknown'
+        if not author:
+            author = 'Unknown'
+
+        # If keywords are not provided, use a default keyword
+        if not keywords:
+            keywords = 'text_file,epub_converted'
+        else:
+            keywords = f'text_file,epub_converted,{keywords}'
+
+        # Add the text file to the database
+        add_media_with_keywords(
+            url=file_path,
+            title=title,
+            media_type='document',
+            content=content,
+            keywords=keywords,
+            prompt='No prompt for text files',
+            summary='No summary for text files',
+            transcription_model='None',
+            author=author,
+            ingestion_date=datetime.now().strftime('%Y-%m-%d')
+        )
+
+        return f"Text file '{title}' by {author} ingested successfully."
+    except Exception as e:
+        logging.error(f"Error ingesting text file: {str(e)}")
+        return f"Error ingesting text file: {str(e)}"
+
+
+def ingest_folder(folder_path, keywords=None):
+    results = []
+    for filename in os.listdir(folder_path):
+        if filename.lower().endswith('.txt'):
+            file_path = os.path.join(folder_path, filename)
+            result = ingest_text_file(file_path, keywords=keywords)
+            results.append(result)
+
+
+def epub_to_markdown(epub_path):
+    book = epub.read_epub(epub_path)
+    markdown_content = "# Table of Contents\n\n"
+    chapters = []
+
+    # Extract and format the table of contents
+    toc = book.toc
+    for item in toc:
+        if isinstance(item, tuple):
+            section, children = item
+            level = 1
+            markdown_content += format_toc_item(section, level)
+            for child in children:
+                markdown_content += format_toc_item(child, level + 1)
+        else:
+            markdown_content += format_toc_item(item, 1)
+
+    markdown_content += "\n---\n\n"
+
+    # Process each chapter
+    for item in book.get_items():
+        if item.get_type() == ebooklib.ITEM_DOCUMENT:
+            chapter_content = item.get_content().decode('utf-8')
+            soup = BeautifulSoup(chapter_content, 'html.parser')
+
+            # Extract chapter title
+            title = soup.find(['h1', 'h2', 'h3'])
+            if title:
+                chapter_title = title.get_text()
+                markdown_content += f"# {chapter_title}\n\n"
+
+            # Process chapter content
+            for elem in soup.find_all(['p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'ul', 'ol']):
+                if elem.name.startswith('h'):
+                    level = int(elem.name[1])
+                    markdown_content += f"{'#' * level} {elem.get_text()}\n\n"
+                elif elem.name == 'p':
+                    markdown_content += f"{elem.get_text()}\n\n"
+                elif elem.name in ['ul', 'ol']:
+                    for li in elem.find_all('li'):
+                        markdown_content += f"- {li.get_text()}\n"
+                    markdown_content += "\n"
+
+            markdown_content += "---\n\n"
+
+    return markdown_content
+
+
+def format_toc_item(item, level):
+    return f"{'  ' * (level - 1)}- [{item.title}](#{slugify(item.title)})\n"
+
+
+def slugify(text):
+    return re.sub(r'[\W_]+', '-', text.lower())
+
+#
+# End of Function Definitions
+#######################################################################################################################

App_Function_Libraries/DB/DB_Manager.py

@@ -5,8 +5,8 @@
 import configparser
 import os
 import logging
-from typing import Tuple, List, Union, Dict
 import time
+from typing import Tuple, List, Union, Dict
 #
 # 3rd-Party Libraries
 from elasticsearch import Elasticsearch
@@ -64,7 +64,11 @@ from App_Function_Libraries.DB.SQLite_DB import (
     get_paginated_files as sqlite_get_paginated_files, get_media_title as sqlite_get_media_title, \
     get_all_content_from_database as sqlite_get_all_content_from_database,
     get_next_media_id as sqlite_get_next_media_id, \
-    batch_insert_chunks as sqlite_batch_insert_chunks, Database,
+    batch_insert_chunks as sqlite_batch_insert_chunks, Database, save_workflow_chat_to_db as sqlite_save_workflow_chat_to_db, \
+    get_workflow_chat as sqlite_get_workflow_chat, update_media_content_with_version as sqlite_update_media_content_with_version, \
+    check_existing_media as sqlite_check_existing_media, get_all_document_versions as sqlite_get_all_document_versions, \
+    fetch_paginated_data as sqlite_fetch_paginated_data, get_latest_transcription as sqlite_get_latest_transcription, \
+
 )
 #
 # Local Imports
@@ -320,6 +324,19 @@ def add_media_to_database(*args, **kwargs):
         # Implement Elasticsearch version
         raise NotImplementedError("Elasticsearch version of add_media_to_database not yet implemented")
 
+def check_existing_media(*args, **kwargs):
+    if db_type == 'sqlite':
+        return sqlite_check_existing_media(*args, **kwargs)
+    elif db_type == 'elasticsearch':
+        # Implement Elasticsearch version
+        raise NotImplementedError("Elasticsearch version of check_existing_media not yet implemented")
+
+def update_media_content_with_version(*args, **kwargs):
+    if db_type == 'sqlite':
+        return sqlite_update_media_content_with_version(*args, **kwargs)
+    elif db_type == 'elasticsearch':
+        # Implement Elasticsearch version
+        raise NotImplementedError("Elasticsearch version of update_media_content not yet implemented")
 
 def import_obsidian_note_to_db(*args, **kwargs):
     if db_type == 'sqlite':
@@ -501,6 +518,24 @@ def mark_as_trash(media_id: int) -> None:
     else:
         raise ValueError(f"Unsupported database type: {db_type}")
 
+
+def get_latest_transcription(*args, **kwargs):
+    if db_type == 'sqlite':
+        return sqlite_get_latest_transcription(*args, **kwargs)
+    elif db_type == 'elasticsearch':
+        # Implement Elasticsearch version
+        raise NotImplementedError("Elasticsearch version of get_latest_transcription not yet implemented")
+
+def fetch_paginated_data(*args, **kwargs):
+    if db_type == 'sqlite':
+        return sqlite_fetch_paginated_data(*args, **kwargs)
+    elif db_type == 'elasticsearch':
+        # Implement Elasticsearch version
+        raise NotImplementedError("Elasticsearch version of fetch_paginated_data not yet implemented")
+    else:
+        raise ValueError(f"Unsupported database type: {db_type}")
+
+        
 def get_media_content(media_id: int) -> str:
     if db_type == 'sqlite':
         return sqlite_get_media_content(media_id)
@@ -541,6 +576,29 @@ def get_specific_summary(summary_id: int) -> Dict:
     else:
         raise ValueError(f"Unsupported database type: {db_type}")
 
+def fetch_item_details_single(*args, **kwargs):
+    if db_type == 'sqlite':
+        return sqlite_fetch_item_details(*args, **kwargs)
+    elif db_type == 'elasticsearch':
+        # Implement Elasticsearch version
+        raise NotImplementedError("Elasticsearch version of fetch_item_details not yet implemented")
+    else:
+        raise ValueError(f"Unsupported database type: {db_type}")
+
+def get_all_document_versions(*args, **kwargs):
+    if db_type == 'sqlite':
+        return sqlite_get_all_document_versions(*args, **kwargs)
+    elif db_type == 'elasticsearch':
+        # Implement Elasticsearch version
+        raise NotImplementedError("Elasticsearch version of get_all_document_versions not yet implemented")
+    else:
+        raise ValueError(f"Unsupported database type: {db_type}")
+#
+#
+############################################################################################################
+#
+# Prompt Functions:
+
 def get_media_prompts(media_id: int) -> List[Dict]:
     if db_type == 'sqlite':
         return sqlite_get_media_prompts(media_id)
@@ -790,9 +848,35 @@ def get_document_version(*args, **kwargs):
 # End of Document Versioning Functions
 ############################################################################################################
 
-def close_connection():
+
+############################################################################################################
+#
+# Workflow Functions
+
+def get_workflow_chat(*args, **kwargs):
+    if db_type == 'sqlite':
+        return sqlite_get_workflow_chat(*args, **kwargs)
+    elif db_type == 'elasticsearch':
+        # Implement Elasticsearch version
+        raise NotImplementedError("Elasticsearch version of get_workflow_chat not yet implemented")
+
+
+def save_workflow_chat_to_db(*args, **kwargs):
     if db_type == 'sqlite':
-        db.get_connection().close()
+        # FIXME
+        return sqlite_save_workflow_chat_to_db(*args, **kwargs)
+    elif db_type == 'elasticsearch':
+        # Implement Elasticsearch version
+        raise NotImplementedError("Elasticsearch version of save_workflow_chat_to_db not yet implemented")
+
+#
+# End of Workflow Functions
+############################################################################################################
+
+# Dead code FIXME
+# def close_connection():
+#     if db_type == 'sqlite':
+#         db.get_connection().close()
 
 #
 # End of file

App_Function_Libraries/DB/SQLite_DB.py

@@ -46,6 +46,7 @@ import configparser
 #
 # Import necessary libraries
 import csv
+import hashlib
 import html
 import logging
 import os
@@ -53,9 +54,13 @@ import queue
 import re
 import shutil
 import sqlite3
+import threading
 import traceback
+from contextlib import contextmanager
 from datetime import datetime, timedelta
 from typing import List, Tuple, Dict, Any, Optional
+from urllib.parse import quote
+
 # Local Libraries
 from App_Function_Libraries.Utils.Utils import get_project_relative_path, get_database_path, \
     get_database_dir
@@ -219,39 +224,62 @@ class DatabaseError(Exception):
 class InputError(Exception):
     pass
 
+
 class Database:
     def __init__(self, db_name='media_summary.db'):
         self.db_path = get_database_path(db_name)
-        self.timeout = 60.0  # 60 seconds timeout
+        self.timeout = 10.0
+        self._local = threading.local()
 
+    @contextmanager
     def get_connection(self):
-        return sqlite3.connect(self.db_path, timeout=self.timeout)
-
-    def execute_query(self, query: str, params: Tuple = ()) -> None:
+        if not hasattr(self._local, 'connection') or self._local.connection is None:
+            self._local.connection = sqlite3.connect(self.db_path, timeout=self.timeout)
+            self._local.connection.isolation_level = None  # This enables autocommit mode
+        yield self._local.connection
+
+    def close_connection(self):
+        if hasattr(self._local, 'connection') and self._local.connection:
+            self._local.connection.close()
+            self._local.connection = None
+
+    @contextmanager
+    def transaction(self):
         with self.get_connection() as conn:
             try:
-                cursor = conn.cursor()
-                cursor.execute(query, params)
-                conn.commit()
-            except sqlite3.Error as e:
-                logging.error(f"Database error: {e}, Query: {query}")
-                raise DatabaseError(f"Database error: {e}, Query: {query}")
+                conn.execute("BEGIN")
+                yield conn
+                conn.execute("COMMIT")
+            except Exception:
+                conn.execute("ROLLBACK")
+                raise
+
+    def execute_query(self, query: str, params: Tuple = ()) -> Any:
+        with self.get_connection() as conn:
+            cursor = conn.cursor()
+            cursor.execute(query, params)
+            if query.strip().upper().startswith("SELECT"):
+                return cursor.fetchall()
+            else:
+                return cursor.rowcount
 
     def execute_many(self, query: str, params_list: List[Tuple]) -> None:
         with self.get_connection() as conn:
-            try:
-                cursor = conn.cursor()
-                cursor.executemany(query, params_list)
-                conn.commit()
-            except sqlite3.Error as e:
-                logging.error(f"Database error: {e}, Query: {query}")
-                raise DatabaseError(f"Database error: {e}, Query: {query}")
+            cursor = conn.cursor()
+            cursor.executemany(query, params_list)
+
+    def table_exists(self, table_name: str) -> bool:
+        query = 'SELECT name FROM sqlite_master WHERE type="table" AND name=?'
+        result = self.execute_query(query, (table_name,))
+        return bool(result)
 
 db = Database()
 
-def instantiate_sqlite_db():
-    global sqlite_db
-    sqlite_db = Database()
+# Usage example:
+if db.table_exists('DocumentVersions'):
+    logging.info("DocumentVersions table exists")
+else:
+    logging.error("DocumentVersions table does not exist")
 
 
 # Function to create tables with the new media schema
@@ -399,7 +427,6 @@ def create_tables(db) -> None:
         'CREATE INDEX IF NOT EXISTS idx_unvectorized_media_chunks_media_id ON UnvectorizedMediaChunks(media_id)',
         'CREATE INDEX IF NOT EXISTS idx_unvectorized_media_chunks_is_processed ON UnvectorizedMediaChunks(is_processed)',
         'CREATE INDEX IF NOT EXISTS idx_unvectorized_media_chunks_chunk_type ON UnvectorizedMediaChunks(chunk_type)',
-        # CREATE UNIQUE INDEX statements
         'CREATE UNIQUE INDEX IF NOT EXISTS idx_unique_media_url ON Media(url)',
         'CREATE UNIQUE INDEX IF NOT EXISTS idx_unique_media_keyword ON MediaKeywords(media_id, keyword_id)',
         'CREATE INDEX IF NOT EXISTS idx_document_versions_media_id ON DocumentVersions(media_id)',
@@ -426,6 +453,14 @@ def create_tables(db) -> None:
 
 create_tables(db)
 
+#
+# End of DB Setup Functions
+#######################################################################################################################
+
+
+#######################################################################################################################
+#
+# Media-related Functions
 
 def check_media_exists(title: str, url: str) -> Optional[int]:
     try:
@@ -560,11 +595,20 @@ def add_keyword(keyword: str) -> int:
     with db.get_connection() as conn:
         cursor = conn.cursor()
         try:
+            # Insert into Keywords table
             cursor.execute('INSERT OR IGNORE INTO Keywords (keyword) VALUES (?)', (keyword,))
+
+            # Get the keyword_id (whether it was just inserted or already existed)
             cursor.execute('SELECT id FROM Keywords WHERE keyword = ?', (keyword,))
             keyword_id = cursor.fetchone()[0]
-            cursor.execute('INSERT OR IGNORE INTO keyword_fts (rowid, keyword) VALUES (?, ?)', (keyword_id, keyword))
-            logging.info(f"Keyword '{keyword}' added to keyword_fts with ID: {keyword_id}")
+
+            # Check if the keyword exists in keyword_fts
+            cursor.execute('SELECT rowid FROM keyword_fts WHERE rowid = ?', (keyword_id,))
+            if not cursor.fetchone():
+                # If it doesn't exist in keyword_fts, insert it
+                cursor.execute('INSERT OR IGNORE INTO keyword_fts (rowid, keyword) VALUES (?, ?)', (keyword_id, keyword))
+
+            logging.info(f"Keyword '{keyword}' added or updated with ID: {keyword_id}")
             conn.commit()
             return keyword_id
         except sqlite3.IntegrityError as e:
@@ -575,6 +619,7 @@ def add_keyword(keyword: str) -> int:
             raise DatabaseError(f"Error adding keyword: {e}")
 
 
+
 # Function to delete a keyword
 def delete_keyword(keyword: str) -> str:
     keyword = keyword.strip().lower()
@@ -881,10 +926,12 @@ def browse_items(search_query, search_type):
 
 
 # Function to fetch item details
+
 def fetch_item_details(media_id: int):
     try:
         with db.get_connection() as conn:
             cursor = conn.cursor()
+            # Fetch the latest prompt and summary from MediaModifications
             cursor.execute("""
                 SELECT prompt, summary 
                 FROM MediaModifications 
@@ -893,18 +940,19 @@ def fetch_item_details(media_id: int):
                 LIMIT 1
             """, (media_id,))
             prompt_summary_result = cursor.fetchone()
+
+            # Fetch the latest transcription
             cursor.execute("SELECT content FROM Media WHERE id = ?", (media_id,))
             content_result = cursor.fetchone()
 
-            prompt = prompt_summary_result[0] if prompt_summary_result else ""
-            summary = prompt_summary_result[1] if prompt_summary_result else ""
-            content = content_result[0] if content_result else ""
+            prompt = prompt_summary_result[0] if prompt_summary_result else "No prompt available."
+            summary = prompt_summary_result[1] if prompt_summary_result else "No summary available."
+            content = content_result[0] if content_result else "No content available."
 
-            return content, prompt, summary
+            return prompt, summary, content
     except sqlite3.Error as e:
         logging.error(f"Error fetching item details: {e}")
-        # Return empty strings if there's an error
-        return "", "", ""
+        return "Error fetching prompt.", "Error fetching summary.", "Error fetching media."
 
 #
 #  End of Media-related Functions
@@ -916,7 +964,6 @@ def fetch_item_details(media_id: int):
 # Media-related Functions
 
 
-
 # Function to add a version of a prompt and summary
 def add_media_version(conn, media_id: int, prompt: str, summary: str) -> None:
     try:
@@ -1113,12 +1160,22 @@ def is_valid_date(date_string: str) -> bool:
     except ValueError:
         return False
 
-def add_media_to_database(url, info_dict, segments, summary, keywords, custom_prompt_input, whisper_model, media_type='video'):
+
+
+
+def add_media_to_database(url, info_dict, segments, summary, keywords, custom_prompt_input, whisper_model, media_type='video', overwrite=False):
     db = Database()
     try:
         with db.get_connection() as conn:
             cursor = conn.cursor()
 
+            # Generate URL if not provided
+            if not url:
+                title = info_dict.get('title', 'Untitled')
+                url_hash = hashlib.md5(f"{title}{media_type}".encode()).hexdigest()
+                url = f"https://No-URL-Submitted.com/{media_type}/{quote(title)}-{url_hash}"
+
+
             # Extract content from segments
             if isinstance(segments, list):
                 content = ' '.join([segment.get('Text', '') for segment in segments if 'Text' in segment])
@@ -1140,13 +1197,14 @@ def add_media_to_database(url, info_dict, segments, summary, keywords, custom_pr
             existing_media = cursor.fetchone()
 
             if existing_media:
-                media_id = existing_media[0]
-                cursor.execute('''
-                UPDATE Media 
-                SET content = ?, transcription_model = ?, title = ?, type = ?, author = ?, ingestion_date = ?, chunking_status = ?
-                WHERE id = ?
-                ''', (content, whisper_model, info_dict.get('title', 'Untitled'), media_type,
-                      info_dict.get('uploader', 'Unknown'), datetime.now().strftime('%Y-%m-%d'), 'pending', media_id))
+                if overwrite:
+                    media_id = existing_media[0]
+                    cursor.execute('''
+                    UPDATE Media 
+                    SET content = ?, transcription_model = ?, title = ?, type = ?, author = ?, ingestion_date = ?, chunking_status = ?
+                    WHERE id = ?
+                    ''', (content, whisper_model, info_dict.get('title', 'Untitled'), media_type,
+                          info_dict.get('uploader', 'Unknown'), datetime.now().strftime('%Y-%m-%d'), 'pending', media_id))
             else:
                 cursor.execute('''
                 INSERT INTO Media (url, title, type, content, author, ingestion_date, transcription_model, chunking_status)
@@ -1186,7 +1244,8 @@ def add_media_to_database(url, info_dict, segments, summary, keywords, custom_pr
         # Schedule chunking
         schedule_chunking(media_id, content, info_dict.get('title', 'Untitled'))
 
-        return f"Media '{info_dict.get('title', 'Untitled')}' added/updated successfully with keywords: {', '.join(keyword_list)}. Chunking scheduled."
+        action = "updated" if existing_media and overwrite else "added"
+        return f"Media '{info_dict.get('title', 'Untitled')}' {action} successfully with URL: {url} and keywords: {', '.join(keyword_list)}. Chunking scheduled."
 
     except DatabaseError as e:
         logging.error(f"Database error: {e}")
@@ -1196,6 +1255,66 @@ def add_media_to_database(url, info_dict, segments, summary, keywords, custom_pr
         raise DatabaseError(f"Unexpected error: {e}")
 
 
+def check_existing_media(url):
+    db = Database()
+    try:
+        with db.get_connection() as conn:
+            cursor = conn.cursor()
+            cursor.execute('SELECT id FROM Media WHERE url = ?', (url,))
+            result = cursor.fetchone()
+            return {'id': result[0]} if result else None
+    except Exception as e:
+        logging.error(f"Error checking existing media: {e}")
+        return None
+
+
+# Modified update_media_content function to create a new version
+def update_media_content_with_version(media_id, info_dict, content_input, prompt_input, summary_input, whisper_model):
+    db = Database()
+    try:
+        with db.get_connection() as conn:
+            cursor = conn.cursor()
+
+            # Create new document version
+            cursor.execute('SELECT MAX(version) FROM MediaVersion WHERE media_id = ?', (media_id,))
+            current_version = cursor.fetchone()[0] or 0
+            new_version = current_version + 1
+
+            # Insert new version
+            cursor.execute('''
+            INSERT INTO MediaVersion (media_id, version, prompt, summary, created_at)
+            VALUES (?, ?, ?, ?, ?)
+            ''', (media_id, new_version, prompt_input, summary_input, datetime.now().strftime('%Y-%m-%d %H:%M:%S')))
+
+            # Update the main content in the Media table
+            cursor.execute('''
+            UPDATE Media 
+            SET content = ?, transcription_model = ?, title = ?, author = ?, ingestion_date = ?, chunking_status = ?
+            WHERE id = ?
+            ''', (content_input, whisper_model, info_dict.get('title', 'Untitled'),
+                  info_dict.get('uploader', 'Unknown'), datetime.now().strftime('%Y-%m-%d'), 'pending', media_id))
+
+            # Update or insert into MediaModifications
+            cursor.execute('''
+            INSERT OR REPLACE INTO MediaModifications (media_id, prompt, summary, modification_date)
+            VALUES (?, ?, ?, ?)
+            ''', (media_id, prompt_input, summary_input, datetime.now().strftime('%Y-%m-%d')))
+
+            # Update full-text search index
+            cursor.execute('INSERT OR REPLACE INTO media_fts (rowid, title, content) VALUES (?, ?, ?)',
+                           (media_id, info_dict.get('title', 'Untitled'), content_input))
+
+            conn.commit()
+
+        # Schedule chunking
+        schedule_chunking(media_id, content_input, info_dict.get('title', 'Untitled'))
+
+        return f"Content updated successfully for media ID: {media_id}. New version: {new_version}"
+    except Exception as e:
+        logging.error(f"Error updating media content: {e}")
+        return f"Error updating content: {str(e)}"
+
+
 # FIXME: This function is not complete and needs to be implemented
 def schedule_chunking(media_id: int, content: str, media_name: str):
     try:
@@ -1622,13 +1741,14 @@ def fetch_item_details_single(media_id: int):
             cursor.execute("SELECT content FROM Media WHERE id = ?", (media_id,))
             content_result = cursor.fetchone()
 
-            prompt = prompt_summary_result[0] if prompt_summary_result else ""
-            summary = prompt_summary_result[1] if prompt_summary_result else ""
-            content = content_result[0] if content_result else ""
+            prompt = prompt_summary_result[0] if prompt_summary_result else "No prompt available."
+            summary = prompt_summary_result[1] if prompt_summary_result else "No summary available."
+            content = content_result[0] if content_result else "No content available."
 
             return prompt, summary, content
     except sqlite3.Error as e:
-        raise Exception(f"Error fetching item details: {e}")
+        logging.error(f"Error fetching item details: {e}")
+        return "Error fetching prompt.", "Error fetching summary.", "Error fetching content."
 
 
 
@@ -2045,6 +2165,22 @@ def get_transcripts(media_id):
         logging.error(f"Error in get_transcripts: {str(e)}")
         return []
 
+def get_latest_transcription(media_id: int):
+    try:
+        with db.get_connection() as conn:
+            cursor = conn.cursor()
+            cursor.execute("""
+                SELECT transcription
+                FROM Transcripts
+                WHERE media_id = ?
+                ORDER BY created_at DESC
+                LIMIT 1
+            """, (media_id,))
+            result = cursor.fetchone()
+            return result[0] if result else "No transcription available."
+    except sqlite3.Error as e:
+        logging.error(f"Error fetching latest transcription: {e}")
+        return "Error fetching transcription."
 
 #
 # End of Functions to Compare Transcripts
@@ -2436,32 +2572,57 @@ def get_paginated_files(page: int = 1, results_per_page: int = 50) -> Tuple[List
 #
 # Functions to manage document versions
 
+
 def create_document_version(media_id: int, content: str) -> int:
+    logging.info(f"Attempting to create document version for media_id: {media_id}")
     try:
         with db.get_connection() as conn:
             cursor = conn.cursor()
+            
+            # Start a transaction
+            cursor.execute("BEGIN EXCLUSIVE TRANSACTION")
 
-            # Get the latest version number
-            cursor.execute('''
-                SELECT MAX(version_number) 
-                FROM DocumentVersions 
-                WHERE media_id = ?
-            ''', (media_id,))
+            try:
+                # Verify media_id exists and get the latest version in one query
+                cursor.execute('''
+                    SELECT m.id, COALESCE(MAX(dv.version_number), 0)
+                    FROM Media m
+                    LEFT JOIN DocumentVersions dv ON m.id = dv.media_id
+                    WHERE m.id = ?
+                    GROUP BY m.id
+                ''', (media_id,))
+                result = cursor.fetchone()
 
-            latest_version = cursor.fetchone()[0] or 0
-            new_version = latest_version + 1
+                if not result:
+                    raise ValueError(f"No Media entry found for id: {media_id}")
 
-            # Insert new version
-            cursor.execute('''
-                INSERT INTO DocumentVersions (media_id, version_number, content)
-                VALUES (?, ?, ?)
-            ''', (media_id, new_version, content))
+                _, latest_version = result
+                new_version = latest_version + 1
 
-            conn.commit()
-            return new_version
+                logging.debug(f"Inserting new version {new_version} for media_id: {media_id}")
+
+                # Insert new version
+                cursor.execute('''
+                    INSERT INTO DocumentVersions (media_id, version_number, content)
+                    VALUES (?, ?, ?)
+                ''', (media_id, new_version, content))
+
+                # Commit the transaction
+                conn.commit()
+                logging.info(f"Successfully created document version {new_version} for media_id: {media_id}")
+                return new_version
+            except Exception as e:
+                # If any error occurs, roll back the transaction
+                conn.rollback()
+                raise e
     except sqlite3.Error as e:
-        logging.error(f"Error creating document version: {e}")
-        raise DatabaseError(f"Error creating document version: {e}")
+        logging.error(f"Database error creating document version: {e}")
+        logging.error(f"Error details - media_id: {media_id}, content length: {len(content)}")
+        raise DatabaseError(f"Failed to create document version: {e}")
+    except Exception as e:
+        logging.error(f"Unexpected error creating document version: {e}")
+        logging.error(f"Error details - media_id: {media_id}, content length: {len(content)}")
+        raise
 
 
 def get_document_version(media_id: int, version_number: int = None) -> Dict[str, Any]:
@@ -2501,6 +2662,91 @@ def get_document_version(media_id: int, version_number: int = None) -> Dict[str,
         logging.error(error_message)
         return {'error': error_message}
 
+def get_all_document_versions(media_id: int) -> List[Dict[str, Any]]:
+    try:
+        with db.get_connection() as conn:
+            cursor = conn.cursor()
+            cursor.execute('''
+                SELECT id, version_number, content, created_at
+                FROM DocumentVersions
+                WHERE media_id = ?
+                ORDER BY version_number DESC
+            ''', (media_id,))
+            results = cursor.fetchall()
+
+            if results:
+                return [
+                    {
+                        'id': row[0],
+                        'version_number': row[1],
+                        'content': row[2],
+                        'created_at': row[3]
+                    }
+                    for row in results
+                ]
+            else:
+                return []
+    except sqlite3.Error as e:
+        error_message = f"Error retrieving all document versions: {e}"
+        logging.error(error_message)
+        return [{'error': error_message}]
+
+def delete_document_version(media_id: int, version_number: int) -> Dict[str, Any]:
+    try:
+        with db.get_connection() as conn:
+            cursor = conn.cursor()
+            cursor.execute('''
+                DELETE FROM DocumentVersions
+                WHERE media_id = ? AND version_number = ?
+            ''', (media_id, version_number))
+            conn.commit()
+
+            if cursor.rowcount > 0:
+                return {'success': f"Document version {version_number} for media_id {media_id} deleted successfully"}
+            else:
+                return {'error': f"No document version found for media_id {media_id} and version_number {version_number}"}
+    except sqlite3.Error as e:
+        error_message = f"Error deleting document version: {e}"
+        logging.error(error_message)
+        return {'error': error_message}
+
+def rollback_to_version(media_id: int, version_number: int) -> Dict[str, Any]:
+    try:
+        with db.get_connection() as conn:
+            cursor = conn.cursor()
+            
+            # Get the content of the version to rollback to
+            cursor.execute('''
+                SELECT content
+                FROM DocumentVersions
+                WHERE media_id = ? AND version_number = ?
+            ''', (media_id, version_number))
+            result = cursor.fetchone()
+            
+            if not result:
+                return {'error': f"No document version found for media_id {media_id} and version_number {version_number}"}
+            
+            rollback_content = result[0]
+            
+            # Create a new version with the content of the version to rollback to
+            cursor.execute('''
+                INSERT INTO DocumentVersions (media_id, version_number, content)
+                VALUES (?, (SELECT COALESCE(MAX(version_number), 0) + 1 FROM DocumentVersions WHERE media_id = ?), ?)
+            ''', (media_id, media_id, rollback_content))
+            
+            new_version_number = cursor.lastrowid
+            
+            conn.commit()
+            
+            return {
+                'success': f"Rolled back to version {version_number} for media_id {media_id}",
+                'new_version_number': new_version_number
+            }
+    except sqlite3.Error as e:
+        error_message = f"Error rolling back to document version: {e}"
+        logging.error(error_message)
+        return {'error': error_message}
+
 #
 # End of Functions to manage document versions
 #######################################################################################################################
@@ -2653,3 +2899,102 @@ def update_media_table(db):
 #
 # End of Functions to manage media chunks
 #######################################################################################################################
+
+
+#######################################################################################################################
+#
+# Workflow Functions
+
+def save_workflow_chat_to_db(chat_history, workflow_name, conversation_id=None):
+    try:
+        with db.get_connection() as conn:
+            cursor = conn.cursor()
+
+            if conversation_id is None:
+                # Create a new conversation
+                conversation_name = f"{workflow_name}_Workflow_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
+                cursor.execute('''
+                    INSERT INTO ChatConversations (media_id, media_name, conversation_name, created_at, updated_at)
+                    VALUES (NULL, ?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
+                ''', (workflow_name, conversation_name))
+                conversation_id = cursor.lastrowid
+            else:
+                # Update existing conversation
+                cursor.execute('''
+                    UPDATE ChatConversations
+                    SET updated_at = CURRENT_TIMESTAMP
+                    WHERE id = ?
+                ''', (conversation_id,))
+
+            # Save messages
+            for user_msg, ai_msg in chat_history:
+                if user_msg:
+                    cursor.execute('''
+                        INSERT INTO ChatMessages (conversation_id, sender, message, timestamp)
+                        VALUES (?, 'user', ?, CURRENT_TIMESTAMP)
+                    ''', (conversation_id, user_msg))
+                if ai_msg:
+                    cursor.execute('''
+                        INSERT INTO ChatMessages (conversation_id, sender, message, timestamp)
+                        VALUES (?, 'ai', ?, CURRENT_TIMESTAMP)
+                    ''', (conversation_id, ai_msg))
+
+            conn.commit()
+
+        return conversation_id, f"Chat saved successfully! Conversation ID: {conversation_id}"
+    except Exception as e:
+        logging.error(f"Error saving workflow chat to database: {str(e)}")
+        return None, f"Error saving chat to database: {str(e)}"
+
+
+def get_workflow_chat(conversation_id):
+    """
+    Retrieve a workflow chat from the database.
+
+    Args:
+    conversation_id: ID of the conversation to retrieve
+
+    Returns:
+    tuple: (chat_history, workflow_name, status_message)
+    """
+    try:
+        with db.get_connection() as conn:
+            cursor = conn.cursor()
+
+            # Get conversation details
+            cursor.execute('''
+                SELECT media_name, conversation_name FROM ChatConversations
+                WHERE id = ?
+            ''', (conversation_id,))
+            result = cursor.fetchone()
+            if not result:
+                return None, None, "Conversation not found"
+
+            workflow_name, conversation_name = result
+
+            # Get chat messages
+            cursor.execute('''
+                SELECT sender, message FROM ChatMessages
+                WHERE conversation_id = ?
+                ORDER BY timestamp
+            ''', (conversation_id,))
+            messages = cursor.fetchall()
+
+            chat_history = []
+            for sender, message in messages:
+                if sender == 'user':
+                    chat_history.append((message, None))
+                else:
+                    if chat_history and chat_history[-1][1] is None:
+                        chat_history[-1] = (chat_history[-1][0], message)
+                    else:
+                        chat_history.append((None, message))
+
+        return chat_history, workflow_name, f"Chat retrieved successfully"
+    except Exception as e:
+        logging.error(f"Error retrieving workflow chat from database: {str(e)}")
+        return None, None, f"Error retrieving chat from database: {str(e)}"
+
+#
+# End of Workflow Functions
+#######################################################################################################################

App_Function_Libraries/DB/Test_SQLite_DB.py

@@ -0,0 +1,202 @@
+# Test_SQLite_DB.py
+# Description:  Test file for SQLite_DB.py
+#
+# Usage: python -m unittest test_sqlite_db.py
+#
+# Imports
+import unittest
+import sqlite3
+import threading
+import time
+from unittest.mock import patch
+#
+# Local Imports
+from App_Function_Libraries.DB.SQLite_DB import Database, add_media_with_keywords, add_media_version, DatabaseError
+#
+#######################################################################################################################
+#
+# Functions:
+
+class TestDatabase(unittest.TestCase):
+    def setUp(self):
+        self.db = Database(':memory:')  # Use in-memory database for testing
+
+    def test_connection_management(self):
+        with self.db.get_connection() as conn:
+            self.assertIsInstance(conn, sqlite3.Connection)
+        self.assertEqual(len(self.db.pool), 1)
+
+    def test_execute_query(self):
+        self.db.execute_query("CREATE TABLE test (id INTEGER PRIMARY KEY, name TEXT)")
+        self.db.execute_query("INSERT INTO test (name) VALUES (?)", ("test_name",))
+        with self.db.get_connection() as conn:
+            cursor = conn.cursor()
+            cursor.execute("SELECT name FROM test")
+            result = cursor.fetchone()
+        self.assertEqual(result[0], "test_name")
+
+    def test_execute_many(self):
+        self.db.execute_query("CREATE TABLE test (id INTEGER PRIMARY KEY, name TEXT)")
+        data = [("name1",), ("name2",), ("name3",)]
+        self.db.execute_many("INSERT INTO test (name) VALUES (?)", data)
+        with self.db.get_connection() as conn:
+            cursor = conn.cursor()
+            cursor.execute("SELECT COUNT(*) FROM test")
+            count = cursor.fetchone()[0]
+        self.assertEqual(count, 3)
+
+    def test_connection_retry(self):
+        def lock_database():
+            with self.db.get_connection() as conn:
+                cursor = conn.cursor()
+                cursor.execute("BEGIN EXCLUSIVE TRANSACTION")
+                time.sleep(2)  # Hold the lock for 2 seconds
+
+        thread = threading.Thread(target=lock_database)
+        thread.start()
+        time.sleep(0.1)  # Give the thread time to acquire the lock
+
+        with self.assertRaises(DatabaseError):
+            self.db.execute_query("SELECT 1")  # This should retry and eventually fail
+
+        thread.join()
+
+class TestAddMediaWithKeywords(unittest.TestCase):
+    def setUp(self):
+        self.db = Database(':memory:')
+        self.db.execute_query("""
+            CREATE TABLE Media (
+                id INTEGER PRIMARY KEY,
+                url TEXT,
+                title TEXT NOT NULL,
+                type TEXT NOT NULL,
+                content TEXT,
+                author TEXT,
+                ingestion_date TEXT,
+                transcription_model TEXT
+            )
+        """)
+        self.db.execute_query("CREATE TABLE Keywords (id INTEGER PRIMARY KEY, keyword TEXT NOT NULL UNIQUE)")
+        self.db.execute_query("""
+            CREATE TABLE MediaKeywords (
+                id INTEGER PRIMARY KEY,
+                media_id INTEGER NOT NULL,
+                keyword_id INTEGER NOT NULL,
+                FOREIGN KEY (media_id) REFERENCES Media(id),
+                FOREIGN KEY (keyword_id) REFERENCES Keywords(id)
+            )
+        """)
+        self.db.execute_query("""
+            CREATE TABLE MediaModifications (
+                id INTEGER PRIMARY KEY,
+                media_id INTEGER NOT NULL,
+                prompt TEXT,
+                summary TEXT,
+                modification_date TEXT,
+                FOREIGN KEY (media_id) REFERENCES Media(id)
+            )
+        """)
+        self.db.execute_query("""
+            CREATE TABLE MediaVersion (
+                id INTEGER PRIMARY KEY,
+                media_id INTEGER NOT NULL,
+                version INTEGER NOT NULL,
+                prompt TEXT,
+                summary TEXT,
+                created_at TEXT NOT NULL,
+                FOREIGN KEY (media_id) REFERENCES Media(id)
+            )
+        """)
+        self.db.execute_query("CREATE VIRTUAL TABLE media_fts USING fts5(title, content)")
+
+    @patch('App_Function_Libraries.DB.SQLite_DB.db', new_callable=lambda: Database(':memory:'))
+    def test_add_new_media(self, mock_db):
+        mock_db.get_connection = self.db.get_connection
+        result = add_media_with_keywords(
+            url="http://example.com",
+            title="Test Title",
+            media_type="article",
+            content="Test content",
+            keywords="test,keyword",
+            prompt="Test prompt",
+            summary="Test summary",
+            transcription_model="Test model",
+            author="Test Author",
+            ingestion_date="2023-01-01"
+        )
+        self.assertIn("added/updated successfully", result)
+
+        with self.db.get_connection() as conn:
+            cursor = conn.cursor()
+            cursor.execute("SELECT COUNT(*) FROM Media")
+            self.assertEqual(cursor.fetchone()[0], 1)
+
+            cursor.execute("SELECT COUNT(*) FROM Keywords")
+            self.assertEqual(cursor.fetchone()[0], 2)
+
+            cursor.execute("SELECT COUNT(*) FROM MediaKeywords")
+            self.assertEqual(cursor.fetchone()[0], 2)
+
+            cursor.execute("SELECT COUNT(*) FROM MediaModifications")
+            self.assertEqual(cursor.fetchone()[0], 1)
+
+            cursor.execute("SELECT COUNT(*) FROM MediaVersion")
+            self.assertEqual(cursor.fetchone()[0], 1)
+
+    @patch('App_Function_Libraries.DB.SQLite_DB.db', new_callable=lambda: Database(':memory:'))
+    def test_update_existing_media(self, mock_db):
+        mock_db.get_connection = self.db.get_connection
+        add_media_with_keywords(
+            url="http://example.com",
+            title="Test Title",
+            media_type="article",
+            content="Test content",
+            keywords="test,keyword",
+            prompt="Test prompt",
+            summary="Test summary",
+            transcription_model="Test model",
+            author="Test Author",
+            ingestion_date="2023-01-01"
+        )
+
+        result = add_media_with_keywords(
+            url="http://example.com",
+            title="Updated Title",
+            media_type="article",
+            content="Updated content",
+            keywords="test,new",
+            prompt="Updated prompt",
+            summary="Updated summary",
+            transcription_model="Updated model",
+            author="Updated Author",
+            ingestion_date="2023-01-02"
+        )
+
+        self.assertIn("added/updated successfully", result)
+
+        with self.db.get_connection() as conn:
+            cursor = conn.cursor()
+            cursor.execute("SELECT COUNT(*) FROM Media")
+            self.assertEqual(cursor.fetchone()[0], 1)
+
+            cursor.execute("SELECT title FROM Media")
+            self.assertEqual(cursor.fetchone()[0], "Updated Title")
+
+            cursor.execute("SELECT COUNT(*) FROM Keywords")
+            self.assertEqual(cursor.fetchone()[0], 3)
+
+            cursor.execute("SELECT COUNT(*) FROM MediaKeywords")
+            self.assertEqual(cursor.fetchone()[0], 3)
+
+            cursor.execute("SELECT COUNT(*) FROM MediaModifications")
+            self.assertEqual(cursor.fetchone()[0], 2)
+
+            cursor.execute("SELECT COUNT(*) FROM MediaVersion")
+            self.assertEqual(cursor.fetchone()[0], 2)
+
+if __name__ == '__main__':
+    unittest.main()
+
+#
+# End of File
+#######################################################################################################################

App_Function_Libraries/Gradio_Related.py

@@ -1,375 +1,383 @@
-# Gradio_Related.py
-#########################################
-# Gradio UI Functions Library
-# I fucking hate Gradio.
-# Yea, fuck Gradio. https://github.com/gradio-app/gradio/pull/8263 & https://github.com/gradio-app/gradio/issues/7968
-#
-#########################################
-#
-# Built-In Imports
-import logging
-import os
-#
-# Import 3rd-Party Libraries
-import gradio as gr
-#
-# Local Imports
-from App_Function_Libraries.DB.DB_Manager import get_db_config
-from App_Function_Libraries.Gradio_UI.Audio_ingestion_tab import create_audio_processing_tab
-from App_Function_Libraries.Gradio_UI.Character_Interaction_tab import create_character_card_interaction_tab, \
-    create_multiple_character_chat_tab, create_narrator_controlled_conversation_tab
-from App_Function_Libraries.Gradio_UI.Chat_ui import create_chat_management_tab, \
-    create_chat_interface_four, create_chat_interface_multi_api, create_chat_interface_stacked, create_chat_interface
-from App_Function_Libraries.Gradio_UI.Config_tab import create_config_editor_tab
-from App_Function_Libraries.Gradio_UI.Explain_summarize_tab import create_summarize_explain_tab
-from App_Function_Libraries.Gradio_UI.Export_Functionality import create_export_tab
-from App_Function_Libraries.Gradio_UI.Backup_Functionality import create_backup_tab, create_view_backups_tab, \
-    create_restore_backup_tab
-from App_Function_Libraries.Gradio_UI.Import_Functionality import create_import_single_prompt_tab, \
-    create_import_obsidian_vault_tab, create_import_item_tab, create_import_book_tab, create_import_multiple_prompts_tab
-from App_Function_Libraries.Gradio_UI.Introduction_tab import create_introduction_tab
-from App_Function_Libraries.Gradio_UI.Keywords import create_view_keywords_tab, create_add_keyword_tab, \
-    create_delete_keyword_tab, create_export_keywords_tab
-#from App_Function_Libraries.Gradio_UI.Live_Recording import create_live_recording_tab
-from App_Function_Libraries.Gradio_UI.Llamafile_tab import create_chat_with_llamafile_tab
-from App_Function_Libraries.Gradio_UI.Media_edit import create_prompt_clone_tab, create_prompt_edit_tab, \
-    create_media_edit_and_clone_tab, create_media_edit_tab
-from App_Function_Libraries.Gradio_UI.Media_wiki_tab import create_mediawiki_import_tab, create_mediawiki_config_tab
-from App_Function_Libraries.Gradio_UI.PDF_ingestion_tab import create_pdf_ingestion_tab, create_pdf_ingestion_test_tab
-from App_Function_Libraries.Gradio_UI.Podcast_tab import create_podcast_tab
-from App_Function_Libraries.Gradio_UI.RAG_QA_Chat_tab import create_rag_qa_chat_tab
-from App_Function_Libraries.Gradio_UI.Re_summarize_tab import create_resummary_tab
-from App_Function_Libraries.Gradio_UI.Search_Tab import create_prompt_view_tab, create_prompt_search_tab, \
-    create_search_summaries_tab, create_viewing_tab,  create_search_tab
-from App_Function_Libraries.Gradio_UI.RAG_Chat_tab import create_rag_tab
-from App_Function_Libraries.Gradio_UI.Embeddings_tab import create_embeddings_tab, create_view_embeddings_tab, \
-    create_purge_embeddings_tab
-from App_Function_Libraries.Gradio_UI.Trash import create_view_trash_tab, create_empty_trash_tab, \
-    create_delete_trash_tab, create_search_and_mark_trash_tab
-from App_Function_Libraries.Gradio_UI.Utilities import create_utilities_yt_timestamp_tab, create_utilities_yt_audio_tab, \
-    create_utilities_yt_video_tab
-from App_Function_Libraries.Gradio_UI.Video_transcription_tab import create_video_transcription_tab
-from App_Function_Libraries.Gradio_UI.View_tab import create_manage_items_tab
-from App_Function_Libraries.Gradio_UI.Website_scraping_tab import create_website_scraping_tab
-from App_Function_Libraries.Gradio_UI.Chat_Workflows import chat_workflows_tab
-#
-# Gradio UI Imports
-from App_Function_Libraries.Gradio_UI.Evaluations_Benchmarks_tab import create_geval_tab, create_infinite_bench_tab
-
-#
-#######################################################################################################################
-# Function Definitions
-#
-
-
-# Disable Gradio Analytics
-os.environ['GRADIO_ANALYTICS_ENABLED'] = 'False'
-
-
-custom_prompt_input = None
-server_mode = False
-share_public = False
-custom_prompt_summarize_bulleted_notes = ("""
-                    <s>You are a bulleted notes specialist. [INST]```When creating comprehensive bulleted notes, you should follow these guidelines: Use multiple headings based on the referenced topics, not categories like quotes or terms. Headings should be surrounded by bold formatting and not be listed as bullet points themselves. Leave no space between headings and their corresponding list items underneath. Important terms within the content should be emphasized by setting them in bold font. Any text that ends with a colon should also be bolded. Before submitting your response, review the instructions, and make any corrections necessary to adhered to the specified format. Do not reference these instructions within the notes.``` \nBased on the content between backticks create comprehensive bulleted notes.[/INST]
-                        **Bulleted Note Creation Guidelines**
-
-                        **Headings**:
-                        - Based on referenced topics, not categories like quotes or terms
-                        - Surrounded by **bold** formatting 
-                        - Not listed as bullet points
-                        - No space between headings and list items underneath
-
-                        **Emphasis**:
-                        - **Important terms** set in bold font
-                        - **Text ending in a colon**: also bolded
-
-                        **Review**:
-                        - Ensure adherence to specified format
-                        - Do not reference these instructions in your response.</s>[INST] {{ .Prompt }} [/INST]
-                    """)
-#
-# End of globals
-#######################################################################################################################
-#
-# Start of Video/Audio Transcription and Summarization Functions
-#
-# Functions:
-# FIXME
-#
-#
-################################################################################################################
-# Functions for Re-Summarization
-#
-# Functions:
-# FIXME
-# End of Re-Summarization Functions
-#
-############################################################################################################################################################################################################################
-#
-# Explain/Summarize This Tab
-#
-# Functions:
-# FIXME
-#
-#
-############################################################################################################################################################################################################################
-#
-# Transcript Comparison Tab
-#
-# Functions:
-# FIXME
-#
-#
-###########################################################################################################################################################################################################################
-#
-# Search Tab
-#
-# Functions:
-# FIXME
-#
-# End of Search Tab Functions
-#
-##############################################################################################################################################################################################################################
-#
-# Llamafile Tab
-#
-# Functions:
-# FIXME
-#
-# End of Llamafile Tab Functions
-##############################################################################################################################################################################################################################
-#
-# Chat Interface Tab Functions
-#
-# Functions:
-# FIXME
-#
-#
-# End of Chat Interface Tab Functions
-################################################################################################################################################################################################################################
-#
-# Media Edit Tab Functions
-# Functions:
-# Fixme
-# create_media_edit_tab():
-##### Trash Tab
-# FIXME
-# Functions:
-#
-# End of Media Edit Tab Functions
-################################################################################################################
-#
-# Import Items Tab Functions
-#
-# Functions:
-#FIXME
-# End of Import Items Tab Functions
-################################################################################################################
-#
-# Export Items Tab Functions
-#
-# Functions:
-# FIXME
-#
-#
-# End of Export Items Tab Functions
-################################################################################################################
-#
-# Keyword Management Tab Functions
-#
-# Functions:
-#  create_view_keywords_tab():
-# FIXME
-#
-# End of Keyword Management Tab Functions
-################################################################################################################
-#
-# Document Editing Tab Functions
-#
-# Functions:
-#   #FIXME
-#
-#
-################################################################################################################
-#
-# Utilities Tab Functions
-# Functions:
-#   create_utilities_yt_video_tab():
-# #FIXME
-
-#
-# End of Utilities Tab Functions
-################################################################################################################
-
-# FIXME - Prompt sample box
-#
-# # Sample data
-# prompts_category_1 = [
-#     "What are the key points discussed in the video?",
-#     "Summarize the main arguments made by the speaker.",
-#     "Describe the conclusions of the study presented."
-# ]
-#
-# prompts_category_2 = [
-#     "How does the proposed solution address the problem?",
-#     "What are the implications of the findings?",
-#     "Can you explain the theory behind the observed phenomenon?"
-# ]
-#
-# all_prompts2 = prompts_category_1 + prompts_category_2
-
-
-def launch_ui(share_public=None, server_mode=False):
-    share=share_public
-    css = """
-    .result-box {
-        margin-bottom: 20px;
-        border: 1px solid #ddd;
-        padding: 10px;
-    }
-    .result-box.error {
-        border-color: #ff0000;
-        background-color: #ffeeee;
-    }
-    .transcription, .summary {
-        max-height: 300px;
-        overflow-y: auto;
-        border: 1px solid #eee;
-        padding: 10px;
-        margin-top: 10px;
-    }
-    """
-
-    with gr.Blocks(theme='bethecloud/storj_theme',css=css) as iface:
-        db_config = get_db_config()
-        db_type = db_config['type']
-        gr.Markdown(f"# tl/dw: Your LLM-powered Research Multi-tool")
-        gr.Markdown(f"(Using {db_type.capitalize()} Database)")
-        with gr.Tabs():
-            with gr.TabItem("Transcription / Summarization / Ingestion"):
-                with gr.Tabs():
-                    create_video_transcription_tab()
-                    create_audio_processing_tab()
-                    create_podcast_tab()
-                    create_import_book_tab()
-                    create_website_scraping_tab()
-                    create_pdf_ingestion_tab()
-                    create_pdf_ingestion_test_tab()
-                    create_resummary_tab()
-                    create_summarize_explain_tab()
-                    #create_live_recording_tab()
-
-            with gr.TabItem("Text Search "):
-                create_search_tab()
-                create_search_summaries_tab()
-
-            with gr.TabItem("RAG Search"):
-                create_rag_tab()
-                create_rag_qa_chat_tab()
-
-            with gr.TabItem("Chat with an LLM"):
-                create_chat_interface()
-                create_chat_interface_stacked()
-                create_chat_interface_multi_api()
-                create_chat_interface_four()
-                create_chat_with_llamafile_tab()
-                create_chat_management_tab()
-                chat_workflows_tab()
-                create_multiple_character_chat_tab()
-                create_narrator_controlled_conversation_tab()
-                create_character_card_interaction_tab()
-
-
-
-            with gr.TabItem("View DB Items"):
-                create_viewing_tab()
-                create_prompt_view_tab()
-
-            with gr.TabItem("Prompts"):
-                create_prompt_view_tab()
-                create_prompt_search_tab()
-                create_prompt_edit_tab()
-                create_prompt_clone_tab()
-
-
-            with gr.TabItem("Manage / Edit Existing Items"):
-                create_media_edit_tab()
-                create_manage_items_tab()
-                create_media_edit_and_clone_tab()
-                # FIXME
-                #create_compare_transcripts_tab()
-
-            with gr.TabItem("Embeddings Management"):
-                create_embeddings_tab()
-                create_view_embeddings_tab()
-                create_purge_embeddings_tab()
-
-            with gr.TabItem("Writing Tools"):
-                with gr.Tabs():
-                    from App_Function_Libraries.Gradio_UI.Writing_tab import create_document_feedback_tab
-                    create_document_feedback_tab()
-                    from App_Function_Libraries.Gradio_UI.Writing_tab import create_grammar_style_check_tab
-                    create_grammar_style_check_tab()
-                    from App_Function_Libraries.Gradio_UI.Writing_tab import create_tone_adjustment_tab
-                    create_tone_adjustment_tab()
-                    from App_Function_Libraries.Gradio_UI.Writing_tab import create_creative_writing_tab
-                    create_creative_writing_tab()
-                    from App_Function_Libraries.Gradio_UI.Writing_tab import create_mikupad_tab
-                    create_mikupad_tab()
-
-
-            with gr.TabItem("Keywords"):
-                create_view_keywords_tab()
-                create_add_keyword_tab()
-                create_delete_keyword_tab()
-                create_export_keywords_tab()
-
-            with gr.TabItem("Import"):
-                create_import_item_tab()
-                create_import_obsidian_vault_tab()
-                create_import_single_prompt_tab()
-                create_import_multiple_prompts_tab()
-                create_mediawiki_import_tab()
-                create_mediawiki_config_tab()
-
-            with gr.TabItem("Export"):
-                create_export_tab()
-
-            with gr.TabItem("Backup Management"):
-                create_backup_tab()
-                create_view_backups_tab()
-                create_restore_backup_tab()
-
-            with gr.TabItem("Utilities"):
-                create_utilities_yt_video_tab()
-                create_utilities_yt_audio_tab()
-                create_utilities_yt_timestamp_tab()
-
-            with gr.TabItem("Trashcan"):
-                create_search_and_mark_trash_tab()
-                create_view_trash_tab()
-                create_delete_trash_tab()
-                create_empty_trash_tab()
-
-            with gr.TabItem("Evaluations"):
-                create_geval_tab()
-                create_infinite_bench_tab()
-
-            with gr.TabItem("Introduction/Help"):
-                create_introduction_tab()
-
-            with gr.TabItem("Config Editor"):
-                create_config_editor_tab()
-
-    # Launch the interface
-    server_port_variable = 7860
-    os.environ['GRADIO_ANALYTICS_ENABLED'] = 'False'
-    if share==True:
-        iface.launch(share=True)
-    elif server_mode and not share_public:
-        iface.launch(share=False, server_name="0.0.0.0", server_port=server_port_variable, )
-    else:
-        try:
-            iface.launch(share=False, server_name="0.0.0.0", server_port=server_port_variable, )
-        except Exception as e:
-            logging.error(f"Error launching interface: {str(e)}")
+# Gradio_Related.py
+#########################################
+# Gradio UI Functions Library
+# I fucking hate Gradio.
+# Yea, fuck Gradio. https://github.com/gradio-app/gradio/pull/8263 & https://github.com/gradio-app/gradio/issues/7968
+#
+#########################################
+#
+# Built-In Imports
+import logging
+import os
+#
+# Import 3rd-Party Libraries
+import gradio as gr
+#
+# Local Imports
+from App_Function_Libraries.DB.DB_Manager import get_db_config
+from App_Function_Libraries.Gradio_UI.Audio_ingestion_tab import create_audio_processing_tab
+from App_Function_Libraries.Gradio_UI.Book_Ingestion_tab import create_import_book_tab
+from App_Function_Libraries.Gradio_UI.Character_Interaction_tab import create_character_card_interaction_tab, \
+    create_multiple_character_chat_tab, create_narrator_controlled_conversation_tab
+from App_Function_Libraries.Gradio_UI.Chat_ui import create_chat_management_tab, \
+    create_chat_interface_four, create_chat_interface_multi_api, create_chat_interface_stacked, create_chat_interface
+from App_Function_Libraries.Gradio_UI.Config_tab import create_config_editor_tab
+from App_Function_Libraries.Gradio_UI.Explain_summarize_tab import create_summarize_explain_tab
+from App_Function_Libraries.Gradio_UI.Export_Functionality import create_export_tab
+from App_Function_Libraries.Gradio_UI.Backup_Functionality import create_backup_tab, create_view_backups_tab, \
+    create_restore_backup_tab
+from App_Function_Libraries.Gradio_UI.Import_Functionality import create_import_single_prompt_tab, \
+    create_import_obsidian_vault_tab, create_import_item_tab, create_import_multiple_prompts_tab
+from App_Function_Libraries.Gradio_UI.Introduction_tab import create_introduction_tab
+from App_Function_Libraries.Gradio_UI.Keywords import create_view_keywords_tab, create_add_keyword_tab, \
+    create_delete_keyword_tab, create_export_keywords_tab
+from App_Function_Libraries.Gradio_UI.Live_Recording import create_live_recording_tab
+from App_Function_Libraries.Gradio_UI.Llamafile_tab import create_chat_with_llamafile_tab
+from App_Function_Libraries.Gradio_UI.MMLU_Pro_tab import create_mmlu_pro_tab
+from App_Function_Libraries.Gradio_UI.Media_edit import create_prompt_clone_tab, create_prompt_edit_tab, \
+    create_media_edit_and_clone_tab, create_media_edit_tab
+from App_Function_Libraries.Gradio_UI.Media_wiki_tab import create_mediawiki_import_tab, create_mediawiki_config_tab
+from App_Function_Libraries.Gradio_UI.PDF_ingestion_tab import create_pdf_ingestion_tab, create_pdf_ingestion_test_tab
+from App_Function_Libraries.Gradio_UI.Plaintext_tab_import import create_plain_text_import_tab
+from App_Function_Libraries.Gradio_UI.Podcast_tab import create_podcast_tab
+from App_Function_Libraries.Gradio_UI.RAG_QA_Chat_tab import create_rag_qa_chat_tab
+from App_Function_Libraries.Gradio_UI.Re_summarize_tab import create_resummary_tab
+from App_Function_Libraries.Gradio_UI.Search_Tab import create_prompt_search_tab, \
+    create_search_summaries_tab, create_search_tab
+from App_Function_Libraries.Gradio_UI.RAG_Chat_tab import create_rag_tab
+from App_Function_Libraries.Gradio_UI.Embeddings_tab import create_embeddings_tab, create_view_embeddings_tab, \
+    create_purge_embeddings_tab
+from App_Function_Libraries.Gradio_UI.Trash import create_view_trash_tab, create_empty_trash_tab, \
+    create_delete_trash_tab, create_search_and_mark_trash_tab
+from App_Function_Libraries.Gradio_UI.Utilities import create_utilities_yt_timestamp_tab, create_utilities_yt_audio_tab, \
+    create_utilities_yt_video_tab
+from App_Function_Libraries.Gradio_UI.Video_transcription_tab import create_video_transcription_tab
+from App_Function_Libraries.Gradio_UI.View_tab import create_manage_items_tab
+from App_Function_Libraries.Gradio_UI.Website_scraping_tab import create_website_scraping_tab
+from App_Function_Libraries.Gradio_UI.Chat_Workflows import chat_workflows_tab
+from App_Function_Libraries.Gradio_UI.View_DB_Items_tab import create_prompt_view_tab, create_viewing_tab, \
+    create_view_all_with_versions_tab, create_viewing_tab
+#
+# Gradio UI Imports
+from App_Function_Libraries.Gradio_UI.Evaluations_Benchmarks_tab import create_geval_tab, create_infinite_bench_tab
+
+#
+#######################################################################################################################
+# Function Definitions
+#
+
+
+# Disable Gradio Analytics
+os.environ['GRADIO_ANALYTICS_ENABLED'] = 'False'
+
+
+custom_prompt_input = None
+server_mode = False
+share_public = False
+custom_prompt_summarize_bulleted_notes = ("""
+                    <s>You are a bulleted notes specialist. [INST]```When creating comprehensive bulleted notes, you should follow these guidelines: Use multiple headings based on the referenced topics, not categories like quotes or terms. Headings should be surrounded by bold formatting and not be listed as bullet points themselves. Leave no space between headings and their corresponding list items underneath. Important terms within the content should be emphasized by setting them in bold font. Any text that ends with a colon should also be bolded. Before submitting your response, review the instructions, and make any corrections necessary to adhered to the specified format. Do not reference these instructions within the notes.``` \nBased on the content between backticks create comprehensive bulleted notes.[/INST]
+                        **Bulleted Note Creation Guidelines**
+
+                        **Headings**:
+                        - Based on referenced topics, not categories like quotes or terms
+                        - Surrounded by **bold** formatting 
+                        - Not listed as bullet points
+                        - No space between headings and list items underneath
+
+                        **Emphasis**:
+                        - **Important terms** set in bold font
+                        - **Text ending in a colon**: also bolded
+
+                        **Review**:
+                        - Ensure adherence to specified format
+                        - Do not reference these instructions in your response.</s>[INST] {{ .Prompt }} [/INST]
+                    """)
+#
+# End of globals
+#######################################################################################################################
+#
+# Start of Video/Audio Transcription and Summarization Functions
+#
+# Functions:
+# FIXME
+#
+#
+################################################################################################################
+# Functions for Re-Summarization
+#
+# Functions:
+# FIXME
+# End of Re-Summarization Functions
+#
+############################################################################################################################################################################################################################
+#
+# Explain/Summarize This Tab
+#
+# Functions:
+# FIXME
+#
+#
+############################################################################################################################################################################################################################
+#
+# Transcript Comparison Tab
+#
+# Functions:
+# FIXME
+#
+#
+###########################################################################################################################################################################################################################
+#
+# Search Tab
+#
+# Functions:
+# FIXME
+#
+# End of Search Tab Functions
+#
+##############################################################################################################################################################################################################################
+#
+# Llamafile Tab
+#
+# Functions:
+# FIXME
+#
+# End of Llamafile Tab Functions
+##############################################################################################################################################################################################################################
+#
+# Chat Interface Tab Functions
+#
+# Functions:
+# FIXME
+#
+#
+# End of Chat Interface Tab Functions
+################################################################################################################################################################################################################################
+#
+# Media Edit Tab Functions
+# Functions:
+# Fixme
+# create_media_edit_tab():
+##### Trash Tab
+# FIXME
+# Functions:
+#
+# End of Media Edit Tab Functions
+################################################################################################################
+#
+# Import Items Tab Functions
+#
+# Functions:
+#FIXME
+# End of Import Items Tab Functions
+################################################################################################################
+#
+# Export Items Tab Functions
+#
+# Functions:
+# FIXME
+#
+#
+# End of Export Items Tab Functions
+################################################################################################################
+#
+# Keyword Management Tab Functions
+#
+# Functions:
+#  create_view_keywords_tab():
+# FIXME
+#
+# End of Keyword Management Tab Functions
+################################################################################################################
+#
+# Document Editing Tab Functions
+#
+# Functions:
+#   #FIXME
+#
+#
+################################################################################################################
+#
+# Utilities Tab Functions
+# Functions:
+#   create_utilities_yt_video_tab():
+# #FIXME
+
+#
+# End of Utilities Tab Functions
+################################################################################################################
+
+# FIXME - Prompt sample box
+#
+# # Sample data
+# prompts_category_1 = [
+#     "What are the key points discussed in the video?",
+#     "Summarize the main arguments made by the speaker.",
+#     "Describe the conclusions of the study presented."
+# ]
+#
+# prompts_category_2 = [
+#     "How does the proposed solution address the problem?",
+#     "What are the implications of the findings?",
+#     "Can you explain the theory behind the observed phenomenon?"
+# ]
+#
+# all_prompts2 = prompts_category_1 + prompts_category_2
+
+
+def launch_ui(share_public=None, server_mode=False):
+    share=share_public
+    css = """
+    .result-box {
+        margin-bottom: 20px;
+        border: 1px solid #ddd;
+        padding: 10px;
+    }
+    .result-box.error {
+        border-color: #ff0000;
+        background-color: #ffeeee;
+    }
+    .transcription, .summary {
+        max-height: 300px;
+        overflow-y: auto;
+        border: 1px solid #eee;
+        padding: 10px;
+        margin-top: 10px;
+    }
+    """
+
+    with gr.Blocks(theme='bethecloud/storj_theme',css=css) as iface:
+        db_config = get_db_config()
+        db_type = db_config['type']
+        gr.Markdown(f"# tl/dw: Your LLM-powered Research Multi-tool")
+        gr.Markdown(f"(Using {db_type.capitalize()} Database)")
+        with gr.Tabs():
+            with gr.TabItem("Transcription / Summarization / Ingestion"):
+                with gr.Tabs():
+                    create_video_transcription_tab()
+                    create_audio_processing_tab()
+                    create_podcast_tab()
+                    create_import_book_tab()
+                    create_plain_text_import_tab()
+                    create_website_scraping_tab()
+                    create_pdf_ingestion_tab()
+                    create_pdf_ingestion_test_tab()
+                    create_resummary_tab()
+                    create_summarize_explain_tab()
+                    create_live_recording_tab()
+
+            with gr.TabItem("Text Search "):
+                create_search_tab()
+                create_search_summaries_tab()
+
+            with gr.TabItem("RAG Search"):
+                create_rag_tab()
+                create_rag_qa_chat_tab()
+
+            with gr.TabItem("Chat with an LLM"):
+                create_chat_interface()
+                create_chat_interface_stacked()
+                create_chat_interface_multi_api()
+                create_chat_interface_four()
+                create_chat_with_llamafile_tab()
+                create_chat_management_tab()
+                chat_workflows_tab()
+                create_multiple_character_chat_tab()
+                create_narrator_controlled_conversation_tab()
+                create_character_card_interaction_tab()
+
+            with gr.TabItem("View DB Items"):
+                # This one works
+                create_view_all_with_versions_tab()
+                # This one is WIP
+                create_viewing_tab()
+                create_prompt_view_tab()
+
+            with gr.TabItem("Prompts"):
+                create_prompt_view_tab()
+                create_prompt_search_tab()
+                create_prompt_edit_tab()
+                create_prompt_clone_tab()
+
+
+            with gr.TabItem("Manage / Edit Existing Items"):
+                create_media_edit_tab()
+                create_manage_items_tab()
+                create_media_edit_and_clone_tab()
+                # FIXME
+                #create_compare_transcripts_tab()
+
+            with gr.TabItem("Embeddings Management"):
+                create_embeddings_tab()
+                create_view_embeddings_tab()
+                create_purge_embeddings_tab()
+
+            with gr.TabItem("Writing Tools"):
+                with gr.Tabs():
+                    from App_Function_Libraries.Gradio_UI.Writing_tab import create_document_feedback_tab
+                    create_document_feedback_tab()
+                    from App_Function_Libraries.Gradio_UI.Writing_tab import create_grammar_style_check_tab
+                    create_grammar_style_check_tab()
+                    from App_Function_Libraries.Gradio_UI.Writing_tab import create_tone_adjustment_tab
+                    create_tone_adjustment_tab()
+                    from App_Function_Libraries.Gradio_UI.Writing_tab import create_creative_writing_tab
+                    create_creative_writing_tab()
+                    from App_Function_Libraries.Gradio_UI.Writing_tab import create_mikupad_tab
+                    create_mikupad_tab()
+
+
+            with gr.TabItem("Keywords"):
+                create_view_keywords_tab()
+                create_add_keyword_tab()
+                create_delete_keyword_tab()
+                create_export_keywords_tab()
+
+            with gr.TabItem("Import"):
+                create_import_item_tab()
+                create_import_obsidian_vault_tab()
+                create_import_single_prompt_tab()
+                create_import_multiple_prompts_tab()
+                create_mediawiki_import_tab()
+                create_mediawiki_config_tab()
+
+            with gr.TabItem("Export"):
+                create_export_tab()
+
+            with gr.TabItem("Backup Management"):
+                create_backup_tab()
+                create_view_backups_tab()
+                create_restore_backup_tab()
+
+            with gr.TabItem("Utilities"):
+                create_utilities_yt_video_tab()
+                create_utilities_yt_audio_tab()
+                create_utilities_yt_timestamp_tab()
+
+            with gr.TabItem("Trashcan"):
+                create_search_and_mark_trash_tab()
+                create_view_trash_tab()
+                create_delete_trash_tab()
+                create_empty_trash_tab()
+
+            with gr.TabItem("Evaluations"):
+                create_geval_tab()
+                create_infinite_bench_tab()
+                create_mmlu_pro_tab()
+
+            with gr.TabItem("Introduction/Help"):
+                create_introduction_tab()
+
+            with gr.TabItem("Config Editor"):
+                create_config_editor_tab()
+
+    # Launch the interface
+    server_port_variable = 7860
+    os.environ['GRADIO_ANALYTICS_ENABLED'] = 'False'
+    if share==True:
+        iface.launch(share=True)
+    elif server_mode and not share_public:
+        iface.launch(share=False, server_name="0.0.0.0", server_port=server_port_variable, )
+    else:
+        try:
+            iface.launch(share=False, server_name="0.0.0.0", server_port=server_port_variable, )
+        except Exception as e:
+            logging.error(f"Error launching interface: {str(e)}")

App_Function_Libraries/MediaWiki/Media_Wiki.py

@@ -23,7 +23,8 @@ from App_Function_Libraries.RAG.ChromaDB_Library import process_and_store_conten
 # Functions:
 # Load configuration
 def load_mediawiki_import_config():
-    with open(os.path.join('Config_Files', 'mediawiki_import_config.yaml'), 'r') as f:
+    config_path = os.path.join(os.path.dirname(__file__), '..', '..', 'Config_Files', 'mediawiki_import_config.yaml')
+    with open(config_path, 'r') as f:
         return yaml.safe_load(f)
 
 config = load_mediawiki_import_config()

App_Function_Libraries/Plaintext/Plaintext_Files.py

@@ -0,0 +1,18 @@
+# Plaintext_Files.py
+# Description: This file contains functions for reading and writing plaintext files.
+#
+# Import necessary libraries
+import os
+import re
+from datetime import datetime
+import logging
+import tempfile
+import zipfile
+#
+# Non-Local Imports
+#
+# Local Imports
+#
+#######################################################################################################################
+#
+# Function Definitions

App_Function_Libraries/RAG/CRAG_Pipeline.py

@@ -0,0 +1,125 @@
+# First gen
+
+# Install the necessary libraries
+# !pip install transformers
+# !pip install sentence-transformers
+# !pip install torch
+# !pip install requests
+# !pip install bs4
+
+import requests
+from bs4 import BeautifulSoup
+from transformers import pipeline, AutoTokenizer, AutoModelForSeq2SeqLM
+from sentence_transformers import SentenceTransformer, util
+import torch
+
+# Step 1: Load Models for Summarization and Similarity
+model_name = "facebook/bart-large-cnn"  # Summarization model
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+model = AutoModelForSeq2SeqLM.from_pretrained(model_name)
+
+# Summarization pipeline
+summarizer = pipeline("summarization", model=model, tokenizer=tokenizer)
+
+# Sentence similarity model
+similarity_model = SentenceTransformer('paraphrase-MiniLM-L6-v2')
+
+
+# Step 2: Define Retrieval Evaluator
+def evaluate_retrieval(query, retrieved_docs):
+    """
+    Evaluate the relevance of retrieved documents using cosine similarity
+    with sentence embeddings.
+    """
+    query_embedding = similarity_model.encode(query, convert_to_tensor=True)
+    doc_embeddings = similarity_model.encode(retrieved_docs, convert_to_tensor=True)
+
+    # Calculate cosine similarity between the query and each document
+    similarities = [util.pytorch_cos_sim(query_embedding, doc_embedding).item() for doc_embedding in doc_embeddings]
+
+    # Set a threshold for relevance (adjustable)
+    relevance_threshold = 0.5
+    relevance_scores = ['Correct' if sim > relevance_threshold else 'Incorrect' for sim in similarities]
+
+    return relevance_scores
+
+
+# Step 3: Knowledge Refinement (Decompose-then-Recompose)
+def decompose_then_recompose(retrieved_docs):
+    """
+    Refine the retrieved documents by summarizing their key information.
+    """
+    refined_knowledge = []
+    for doc in retrieved_docs:
+        summary = summarizer(doc, max_length=50, min_length=20, do_sample=False)[0]['summary_text']
+        refined_knowledge.append(summary)
+    return refined_knowledge
+
+
+# Step 4: Web Search for External Knowledge
+def web_search(query):
+    """
+    Perform a web search to retrieve additional external knowledge if the
+    retrieved documents are not relevant.
+    """
+    search_url = f"https://www.google.com/search?q={query.replace(' ', '+')}"
+    headers = {'User-Agent': 'Mozilla/5.0'}
+    response = requests.get(search_url, headers=headers)
+    soup = BeautifulSoup(response.text, 'html.parser')
+
+    # Extract URLs from search results (simplified)
+    links = []
+    for item in soup.find_all('a'):
+        link = item.get('href')
+        if link and "http" in link:
+            links.append(link)
+    return links[:5]  # Return the first 5 URLs
+
+
+# Step 5: Generate Final Output
+def generate_final_output(query, refined_knowledge):
+    """
+    Generate the final output summary using the refined knowledge.
+    """
+    combined_knowledge = " ".join(refined_knowledge)
+    final_summary = summarizer(combined_knowledge, max_length=100, min_length=50, do_sample=False)[0]['summary_text']
+    return final_summary
+
+
+# Step 6: CRAG Workflow Integration
+def crag_workflow(query, retrieved_docs):
+    """
+    Full CRAG workflow integrating evaluation, knowledge refinement,
+    and web search to generate a robust output summary.
+    """
+    # Step 1: Evaluate retrieval
+    relevance_scores = evaluate_retrieval(query, retrieved_docs)
+
+    if 'Correct' in relevance_scores:
+        # Step 2: Decompose-then-Recompose for correct documents
+        refined_knowledge = decompose_then_recompose(
+            [doc for doc, score in zip(retrieved_docs, relevance_scores) if score == 'Correct'])
+    else:
+        # Step 3: Web search if retrieval is incorrect
+        web_results = web_search(query)
+        refined_knowledge = decompose_then_recompose(web_results)
+
+    # Step 4: Generate final output
+    final_summary = generate_final_output(query, refined_knowledge)
+
+    return final_summary
+
+
+# Example Usage
+if __name__ == "__main__":
+    # Example query and retrieved documents
+    query = "What are the latest advancements in renewable energy?"
+    retrieved_docs = [
+        "Renewable energy is becoming increasingly important in today's world...",
+        "Solar energy has seen significant advancements in the past decade...",
+        "Wind energy technology is rapidly evolving, with new innovations expected soon..."
+    ]
+
+    # Perform the CRAG workflow
+    final_summary = crag_workflow(query, retrieved_docs)
+    print("Final Summary:", final_summary)

App_Function_Libraries/RAG/ChromaDB_Library.py

@@ -1,290 +1,244 @@
-import configparser
-import logging
-import sqlite3
-from typing import List, Dict, Any
-
-import chromadb
-import requests
-from chromadb import Settings
-
-from App_Function_Libraries.Chunk_Lib import improved_chunking_process
-from App_Function_Libraries.DB.DB_Manager import add_media_chunk, update_fts_for_media
-from App_Function_Libraries.LLM_API_Calls import get_openai_embeddings
-
-#######################################################################################################################
-#
-# Functions for ChromaDB
-
-# Get ChromaDB settings
-# Load configuration
-config = configparser.ConfigParser()
-config.read('config.txt')
-chroma_db_path = config.get('Database', 'chroma_db_path', fallback='chroma_db')
-chroma_client = chromadb.PersistentClient(path=chroma_db_path, settings=Settings(anonymized_telemetry=False))
-
-import os
-os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python"
-
-# Get embedding settings
-embedding_provider = config.get('Embeddings', 'provider', fallback='openai')
-embedding_model = config.get('Embeddings', 'model', fallback='text-embedding-3-small')
-embedding_api_key = config.get('Embeddings', 'api_key', fallback='')
-embedding_api_url = config.get('Embeddings', 'api_url', fallback='')
-
-# Get chunking options
-chunk_options = {
-    'method': config.get('Chunking', 'method', fallback='words'),
-    'max_size': config.getint('Chunking', 'max_size', fallback=400),
-    'overlap': config.getint('Chunking', 'overlap', fallback=200),
-    'adaptive': config.getboolean('Chunking', 'adaptive', fallback=False),
-    'multi_level': config.getboolean('Chunking', 'multi_level', fallback=False),
-    'language': config.get('Chunking', 'language', fallback='english')
-}
-
-
-def auto_update_chroma_embeddings(media_id: int, content: str):
-    """
-    Automatically update ChromaDB embeddings when a new item is ingested into the SQLite database.
-
-    :param media_id: The ID of the newly ingested media item
-    :param content: The content of the newly ingested media item
-    """
-    collection_name = f"media_{media_id}"
-
-    # Initialize or get the ChromaDB collection
-    collection = chroma_client.get_or_create_collection(name=collection_name)
-
-    # Check if embeddings already exist for this media_id
-    existing_embeddings = collection.get(ids=[f"{media_id}_chunk_{i}" for i in range(len(content))])
-
-    if existing_embeddings and len(existing_embeddings) > 0:
-        logging.info(f"Embeddings already exist for media ID {media_id}, skipping...")
-    else:
-        # Process and store content if embeddings do not already exist
-        process_and_store_content(content, collection_name, media_id)
-        logging.info(f"Updated ChromaDB embeddings for media ID: {media_id}")
-
-
-# Function to process content, create chunks, embeddings, and store in ChromaDB and SQLite
-def process_and_store_content(content: str, collection_name: str, media_id: int):
-    # Process the content into chunks
-    chunks = improved_chunking_process(content, chunk_options)
-    texts = [chunk['text'] for chunk in chunks]
-
-    # Generate embeddings for each chunk
-    embeddings = [create_embedding(text) for text in texts]
-
-    # Create unique IDs for each chunk using the media_id and chunk index
-    ids = [f"{media_id}_chunk_{i}" for i in range(len(texts))]
-
-    # Store the texts, embeddings, and IDs in ChromaDB
-    store_in_chroma(collection_name, texts, embeddings, ids)
-
-    # Store the chunk metadata in SQLite
-    for i, chunk in enumerate(chunks):
-        add_media_chunk(media_id, chunk['text'], chunk['start'], chunk['end'], ids[i])
-
-    # Update the FTS table
-    update_fts_for_media(media_id)
-
-# Function to store documents and their embeddings in ChromaDB
-def store_in_chroma(collection_name: str, texts: List[str], embeddings: List[List[float]], ids: List[str]):
-    collection = chroma_client.get_or_create_collection(name=collection_name)
-    collection.add(
-        documents=texts,
-        embeddings=embeddings,
-        ids=ids
-    )
-
-# Function to perform vector search using ChromaDB
-def vector_search(collection_name: str, query: str, k: int = 10) -> List[str]:
-    query_embedding = create_embedding(query)
-    collection = chroma_client.get_collection(name=collection_name)
-    results = collection.query(
-        query_embeddings=[query_embedding],
-        n_results=k
-    )
-    return results['documents'][0]
-
-
-def create_embedding(text: str) -> List[float]:
-    global embedding_provider, embedding_model, embedding_api_url, embedding_api_key
-
-    if embedding_provider == 'openai':
-        return get_openai_embeddings(text, embedding_model)
-    elif embedding_provider == 'local':
-        response = requests.post(
-            embedding_api_url,
-            json={"text": text, "model": embedding_model},
-            headers={"Authorization": f"Bearer {embedding_api_key}"}
-        )
-        return response.json()['embedding']
-    elif embedding_provider == 'huggingface':
-        from transformers import AutoTokenizer, AutoModel
-        import torch
-
-        tokenizer = AutoTokenizer.from_pretrained(embedding_model)
-        model = AutoModel.from_pretrained(embedding_model)
-
-        inputs = tokenizer(text, return_tensors="pt", padding=True, truncation=True, max_length=512)
-        with torch.no_grad():
-            outputs = model(**inputs)
-
-        # Use the mean of the last hidden state as the sentence embedding
-        embeddings = outputs.last_hidden_state.mean(dim=1)
-        return embeddings[0].tolist()  # Convert to list for consistency
-    else:
-        raise ValueError(f"Unsupported embedding provider: {embedding_provider}")
-
-
-def create_all_embeddings(api_choice: str, model_or_url: str) -> str:
-    try:
-        all_content = get_all_content_from_database()
-
-        if not all_content:
-            return "No content found in the database."
-
-        texts_to_embed = []
-        embeddings_to_store = []
-        ids_to_store = []
-        collection_name = "all_content_embeddings"
-
-        # Initialize or get the ChromaDB collection
-        collection = chroma_client.get_or_create_collection(name=collection_name)
-
-        for content_item in all_content:
-            media_id = content_item['id']
-            text = content_item['content']
-
-            # Check if the embedding already exists in ChromaDB
-            embedding_exists = collection.get(ids=[f"doc_{media_id}"])
-
-            if embedding_exists:
-                logging.info(f"Embedding already exists for media ID {media_id}, skipping...")
-                continue  # Skip if embedding already exists
-
-            # Create the embedding
-            if api_choice == "openai":
-                embedding = create_openai_embedding(text, model_or_url)
-            else:  # Llama.cpp
-                embedding = create_llamacpp_embedding(text, model_or_url)
-
-            # Collect the text, embedding, and ID for batch storage
-            texts_to_embed.append(text)
-            embeddings_to_store.append(embedding)
-            ids_to_store.append(f"doc_{media_id}")
-
-        # Store all new embeddings in ChromaDB
-        if texts_to_embed and embeddings_to_store:
-            store_in_chroma(collection_name, texts_to_embed, embeddings_to_store, ids_to_store)
-
-        return "Embeddings created and stored successfully for all new content."
-    except Exception as e:
-        logging.error(f"Error during embedding creation: {str(e)}")
-        return f"Error: {str(e)}"
-
-
-def create_openai_embedding(text: str, model: str) -> List[float]:
-    openai_api_key = config['API']['openai_api_key']
-    embedding = get_openai_embeddings(text, model)
-    return embedding
-
-
-def create_llamacpp_embedding(text: str, api_url: str) -> List[float]:
-    response = requests.post(
-        api_url,
-        json={"input": text}
-    )
-    if response.status_code == 200:
-        return response.json()['embedding']
-    else:
-        raise Exception(f"Error from Llama.cpp API: {response.text}")
-
-
-def get_all_content_from_database() -> List[Dict[str, Any]]:
-    """
-    Retrieve all media content from the database that requires embedding.
-
-    Returns:
-        List[Dict[str, Any]]: A list of dictionaries, each containing the media ID, content, title, and other relevant fields.
-    """
-    try:
-        from App_Function_Libraries.DB.DB_Manager import db
-        with db.get_connection() as conn:
-            cursor = conn.cursor()
-            cursor.execute("""
-                SELECT id, content, title, author, type
-                FROM Media
-                WHERE is_trash = 0  -- Exclude items marked as trash
-            """)
-            media_items = cursor.fetchall()
-
-            # Convert the results into a list of dictionaries
-            all_content = [
-                {
-                    'id': item[0],
-                    'content': item[1],
-                    'title': item[2],
-                    'author': item[3],
-                    'type': item[4]
-                }
-                for item in media_items
-            ]
-
-        return all_content
-
-    except sqlite3.Error as e:
-        logging.error(f"Error retrieving all content from database: {e}")
-        from App_Function_Libraries.DB.SQLite_DB import DatabaseError
-        raise DatabaseError(f"Error retrieving all content from database: {e}")
-
-
-def store_in_chroma_with_citation(collection_name: str, texts: List[str], embeddings: List[List[float]], ids: List[str], sources: List[str]):
-    collection = chroma_client.get_or_create_collection(name=collection_name)
-    collection.add(
-        documents=texts,
-        embeddings=embeddings,
-        ids=ids,
-        metadatas=[{'source': source} for source in sources]
-    )
-
-
-def check_embedding_status(selected_item):
-    if not selected_item:
-        return "Please select an item", ""
-    item_id = selected_item.split('(')[0].strip()
-    collection = chroma_client.get_or_create_collection(name="all_content_embeddings")
-    result = collection.get(ids=[f"doc_{item_id}"])
-    if result['ids']:
-        embedding = result['embeddings'][0]
-        embedding_preview = str(embedding[:50])  # Convert first 50 elements to string
-        return f"Embedding exists for item: {item_id}", f"Embedding preview: {embedding_preview}..."
-    else:
-        return f"No embedding found for item: {item_id}", ""
-
-
-def create_new_embedding(selected_item, api_choice, openai_model, llamacpp_url):
-    if not selected_item:
-        return "Please select an item"
-    item_id = selected_item.split('(')[0].strip()
-    items = get_all_content_from_database()
-    item = next((item for item in items if item['title'] == item_id), None)
-    if not item:
-        return f"Item not found: {item_id}"
-
-    try:
-        if api_choice == "OpenAI":
-            embedding = create_embedding(item['content'])
-        else:  # Llama.cpp
-            embedding = create_embedding(item['content'])
-
-        collection_name = "all_content_embeddings"
-        store_in_chroma(collection_name, [item['content']], [embedding], [f"doc_{item['id']}"])
-        return f"New embedding created and stored for item: {item_id}"
-    except Exception as e:
-        return f"Error creating embedding: {str(e)}"
-
-
-#
-# End of Functions for ChromaDB
+# ChromaDB_Library.py
+# Description: Functions for managing embeddings in ChromaDB
+#
+# Imports:
+import logging
+from typing import List, Dict, Any
+# 3rd-Party Imports:
+import chromadb
+from chromadb import Settings
+from itertools import islice
+#
+# Local Imports:
+from App_Function_Libraries.Chunk_Lib import chunk_for_embedding, chunk_options
+from App_Function_Libraries.DB.SQLite_DB import process_chunks
+from App_Function_Libraries.RAG.Embeddings_Create import create_embeddings_batch
+# FIXME - related to Chunking
+from App_Function_Libraries.RAG.Embeddings_Create import create_embedding
+from App_Function_Libraries.Summarization.Summarization_General_Lib import summarize
+from App_Function_Libraries.Utils.Utils import get_database_path, ensure_directory_exists, \
+    load_comprehensive_config
+#
+#######################################################################################################################
+#
+# Config Settings for ChromaDB Functions
+#
+# FIXME - Refactor so that all globals are set in summarize.py
+# Set up logging
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+logger = logging.getLogger(__name__)
+#
+# Load config
+config = load_comprehensive_config()
+#
+# ChromaDB settings
+chroma_db_path = config.get('Database', 'chroma_db_path', fallback=get_database_path('chroma_db'))
+ensure_directory_exists(chroma_db_path)
+chroma_client = chromadb.PersistentClient(path=chroma_db_path, settings=Settings(anonymized_telemetry=False))
+#
+# Embedding settings
+embedding_provider = config.get('Embeddings', 'embedding_provider', fallback='openai')
+embedding_model = config.get('Embeddings', 'embedding_model', fallback='text-embedding-3-small')
+embedding_api_key = config.get('Embeddings', 'api_key', fallback='')
+embedding_api_url = config.get('Embeddings', 'api_url', fallback='')
+#
+# End of Config Settings
+#######################################################################################################################
+#
+# Functions:
+
+def batched(iterable, n):
+    "Batch data into lists of length n. The last batch may be shorter."
+    it = iter(iterable)
+    while True:
+        batch = list(islice(it, n))
+        if not batch:
+            return
+        yield batch
+
+
+# FIXME - Fix summarization of entire document/storign in chunk issue
+# FIXME - update all uses to reflect 'api_name' parameter
+def process_and_store_content(database, content: str, collection_name: str, media_id: int, file_name: str,
+                              create_embeddings: bool = False, create_summary: bool = False, api_name: str = None,
+                              chunk_options: Dict = None, embedding_provider: str = None,
+                              embedding_model: str = None, embedding_api_url: str = None):
+    try:
+        logger.info(f"Processing content for media_id {media_id} in collection {collection_name}")
+
+        full_summary = None
+        if create_summary and api_name:
+            full_summary = summarize(content, None, api_name, None, None, None)
+
+        chunks = chunk_for_embedding(content, file_name, full_summary, chunk_options)
+
+        # Process chunks synchronously
+        process_chunks(database, chunks, media_id)
+
+        if create_embeddings:
+            texts = [chunk['text'] for chunk in chunks]
+            embeddings = create_embeddings_batch(texts, embedding_provider, embedding_model, embedding_api_url)
+            ids = [f"{media_id}_chunk_{i}" for i in range(1, len(chunks) + 1)]
+            metadatas = [{
+                "media_id": str(media_id),
+                "chunk_index": i,
+                "total_chunks": len(chunks),
+                "start_index": int(chunk['metadata']['start_index']),
+                "end_index": int(chunk['metadata']['end_index']),
+                "file_name": str(file_name),
+                "relative_position": float(chunk['metadata']['relative_position'])
+            } for i, chunk in enumerate(chunks, 1)]
+
+            store_in_chroma(collection_name, texts, embeddings, ids, metadatas)
+
+        # Update full-text search index
+        database.execute_query(
+            "INSERT OR REPLACE INTO media_fts (rowid, title, content) SELECT id, title, content FROM Media WHERE id = ?",
+            (media_id,)
+        )
+
+        logger.info(f"Finished processing and storing content for media_id {media_id}")
+
+    except Exception as e:
+        logger.error(f"Error in process_and_store_content for media_id {media_id}: {str(e)}")
+        raise
+
+# Usage example:
+# process_and_store_content(db, content, "my_collection", 1, "example.txt", create_embeddings=True, create_summary=True, api_name="gpt-3.5-turbo")
+
+
+def check_embedding_status(selected_item, item_mapping):
+    if not selected_item:
+        return "Please select an item", ""
+
+    try:
+        item_id = item_mapping.get(selected_item)
+        if item_id is None:
+            return f"Invalid item selected: {selected_item}", ""
+
+        item_title = selected_item.rsplit(' (', 1)[0]
+        collection = chroma_client.get_or_create_collection(name="all_content_embeddings")
+
+        result = collection.get(ids=[f"doc_{item_id}"], include=["embeddings", "metadatas"])
+        logging.info(f"ChromaDB result for item '{item_title}' (ID: {item_id}): {result}")
+
+        if not result['ids']:
+            return f"No embedding found for item '{item_title}' (ID: {item_id})", ""
+
+        if not result['embeddings'] or not result['embeddings'][0]:
+            return f"Embedding data missing for item '{item_title}' (ID: {item_id})", ""
+
+        embedding = result['embeddings'][0]
+        metadata = result['metadatas'][0] if result['metadatas'] else {}
+        embedding_preview = str(embedding[:50])
+        status = f"Embedding exists for item '{item_title}' (ID: {item_id})"
+        return status, f"First 50 elements of embedding:\n{embedding_preview}\n\nMetadata: {metadata}"
+
+    except Exception as e:
+        logging.error(f"Error in check_embedding_status: {str(e)}")
+        return f"Error processing item: {selected_item}. Details: {str(e)}", ""
+
+def reset_chroma_collection(collection_name: str):
+    try:
+        chroma_client.delete_collection(collection_name)
+        chroma_client.create_collection(collection_name)
+        logging.info(f"Reset ChromaDB collection: {collection_name}")
+    except Exception as e:
+        logging.error(f"Error resetting ChromaDB collection: {str(e)}")
+
+
+def store_in_chroma(collection_name: str, texts: List[str], embeddings: List[List[float]], ids: List[str], metadatas: List[Dict[str, Any]]):
+    try:
+        collection = chroma_client.get_or_create_collection(name=collection_name)
+
+        # Log the inputs for debugging
+        logging.debug(f"Storing in ChromaDB - Collection: {collection_name}")
+        logging.debug(f"Texts (first 100 chars): {texts[0][:100]}...")
+        logging.debug(f"Embeddings (first 5 values): {embeddings[0][:5]}")
+        logging.debug(f"IDs: {ids}")
+        logging.debug(f"Metadatas: {metadatas}")
+
+        # Use upsert instead of add/update
+        collection.upsert(
+            documents=texts,
+            embeddings=embeddings,
+            ids=ids,
+            metadatas=metadatas
+        )
+
+        # Verify storage
+        for doc_id in ids:
+            result = collection.get(ids=[doc_id], include=["embeddings"])
+            if not result['embeddings'] or result['embeddings'][0] is None:
+                logging.error(f"Failed to store embedding for {doc_id}")
+            else:
+                logging.info(f"Embedding stored successfully for {doc_id}")
+
+    except Exception as e:
+        logging.error(f"Error storing embeddings in ChromaDB: {str(e)}")
+        raise
+
+
+# Function to perform vector search using ChromaDB + Keywords from the media_db
+def vector_search(collection_name: str, query: str, k: int = 10) -> List[Dict[str, Any]]:
+    try:
+        query_embedding = create_embedding(query, embedding_provider, embedding_model, embedding_api_url)
+        collection = chroma_client.get_collection(name=collection_name)
+        results = collection.query(
+            query_embeddings=[query_embedding],
+            n_results=k,
+            include=["documents", "metadatas"]
+        )
+        return [{"content": doc, "metadata": meta} for doc, meta in zip(results['documents'][0], results['metadatas'][0])]
+    except Exception as e:
+        logging.error(f"Error in vector_search: {str(e)}")
+        raise
+
+def schedule_embedding(media_id: int, content: str, media_name: str, summary: str):
+    try:
+        chunks = chunk_for_embedding(content, media_name, summary, chunk_options)
+        texts = [chunk['text'] for chunk in chunks]
+        embeddings = create_embeddings_batch(texts, embedding_provider, embedding_model, embedding_api_url)
+        ids = [f"{media_id}_chunk_{i}" for i in range(len(chunks))]
+        metadatas = [{
+            "media_id": str(media_id),
+            "chunk_index": i,
+            "total_chunks": len(chunks),
+            "start_index": chunk['metadata']['start_index'],
+            "end_index": chunk['metadata']['end_index'],
+            "file_name": media_name,
+            "relative_position": chunk['metadata']['relative_position']
+        } for i, chunk in enumerate(chunks)]
+
+        store_in_chroma("all_content_embeddings", texts, embeddings, ids, metadatas)
+
+    except Exception as e:
+        logging.error(f"Error scheduling embedding for media_id {media_id}: {str(e)}")
+
+
+# Function to process content, create chunks, embeddings, and store in ChromaDB and SQLite
+# def process_and_store_content(content: str, collection_name: str, media_id: int):
+#     # Process the content into chunks
+#     chunks = improved_chunking_process(content, chunk_options)
+#     texts = [chunk['text'] for chunk in chunks]
+#
+#     # Generate embeddings for each chunk
+#     embeddings = [create_embedding(text) for text in texts]
+#
+#     # Create unique IDs for each chunk using the media_id and chunk index
+#     ids = [f"{media_id}_chunk_{i}" for i in range(len(texts))]
+#
+#     # Store the texts, embeddings, and IDs in ChromaDB
+#     store_in_chroma(collection_name, texts, embeddings, ids)
+#
+#     # Store the chunk metadata in SQLite
+#     for i, chunk in enumerate(chunks):
+#         add_media_chunk(media_id, chunk['text'], chunk['start'], chunk['end'], ids[i])
+#
+#     # Update the FTS table
+#     update_fts_for_media(media_id)
+
+
+#
+# End of Functions for ChromaDB
 #######################################################################################################################

App_Function_Libraries/RAG/Embeddings_Create.py

@@ -1,168 +1,224 @@
-# Embeddings_Create.py
-# Description: Functions for Creating and managing Embeddings in ChromaDB with LLama.cpp/OpenAI/Transformers
-#
-# Imports:
-import logging
-from typing import List, Dict, Any
-
-import numpy as np
-#
-# 3rd-Party Imports:
-import requests
-from transformers import AutoTokenizer, AutoModel
-import torch
-#
-# Local Imports:
-from App_Function_Libraries.LLM_API_Calls import get_openai_embeddings
-from App_Function_Libraries.Summarization_General_Lib import summarize
-from App_Function_Libraries.Utils.Utils import load_comprehensive_config
-from App_Function_Libraries.Chunk_Lib import chunk_options, improved_chunking_process#, determine_chunk_position
-#
-#
-#######################################################################################################################
-#
-# Functions:
-
-# FIXME - Add all globals to summarize.py
-loaded_config = load_comprehensive_config()
-embedding_provider = 'openai'
-
-embedding_model = 'text-embedding-3-large'
-embedding_api_url = 'https://127.0.0.1'
-embedding_api_key = 'busted_api_key_lol'
-
-# Embedding Chunking Settings
-chunk_size = '500'
-overlap = '200'
-
-
-# FIXME - Add logging
-
-# FIXME - refactor/setup to use config file & perform chunking
-def create_embedding(text: str, provider: str, model: str, api_url: str = None, api_key: str = None) -> List[float]:
-    try:
-        if provider == 'openai':
-            embedding = get_openai_embeddings(text, model)
-        elif provider == 'local':
-            embedding = create_local_embedding(text, model, api_url, api_key)
-        elif provider == 'huggingface':
-            embedding = create_huggingface_embedding(text, model)
-        elif provider == 'llamacpp':
-            embedding = create_llamacpp_embedding(text, api_url)
-        else:
-            raise ValueError(f"Unsupported embedding provider: {provider}")
-
-        if isinstance(embedding, np.ndarray):
-            embedding = embedding.tolist()
-        elif isinstance(embedding, torch.Tensor):
-            embedding = embedding.detach().cpu().numpy().tolist()
-
-        return embedding
-
-    except Exception as e:
-        logging.error(f"Error creating embedding: {str(e)}")
-        raise
-
-
-def create_huggingface_embedding(text: str, model: str) -> List[float]:
-    tokenizer = AutoTokenizer.from_pretrained(model)
-    model = AutoModel.from_pretrained(model)
-
-    inputs = tokenizer(text, return_tensors="pt", padding=True, truncation=True, max_length=512)
-    with torch.no_grad():
-        outputs = model(**inputs)
-
-    embeddings = outputs.last_hidden_state.mean(dim=1)
-    return embeddings[0].tolist()
-
-
-# FIXME
-def create_stella_embeddings(text: str) -> List[float]:
-    if embedding_provider == 'local':
-        # Load the model and tokenizer
-        tokenizer = AutoTokenizer.from_pretrained("dunzhang/stella_en_400M_v5")
-        model = AutoModel.from_pretrained("dunzhang/stella_en_400M_v5")
-
-        # Tokenize and encode the text
-        inputs = tokenizer(text, return_tensors="pt", padding=True, truncation=True, max_length=512)
-
-        # Generate embeddings
-        with torch.no_grad():
-            outputs = model(**inputs)
-
-        # Use the mean of the last hidden state as the sentence embedding
-        embeddings = outputs.last_hidden_state.mean(dim=1)
-
-        return embeddings[0].tolist()  # Convert to list for consistency
-    elif embedding_provider == 'openai':
-        return get_openai_embeddings(text, embedding_model)
-    else:
-        raise ValueError(f"Unsupported embedding provider: {embedding_provider}")
-
-
-def create_llamacpp_embedding(text: str, api_url: str) -> List[float]:
-    response = requests.post(
-        api_url,
-        json={"input": text}
-    )
-    response.raise_for_status()
-    return response.json()['embedding']
-
-
-def create_local_embedding(text: str, model: str, api_url: str, api_key: str) -> List[float]:
-    response = requests.post(
-        api_url,
-        json={"text": text, "model": model},
-        headers={"Authorization": f"Bearer {api_key}"}
-    )
-    response.raise_for_status()
-    return response.json().get('embedding', None)
-
-
-def chunk_for_embedding(text: str, file_name: str, api_name, custom_chunk_options: Dict[str, Any] = None) -> List[Dict[str, Any]]:
-    options = chunk_options.copy()
-    if custom_chunk_options:
-        options.update(custom_chunk_options)
-
-
-    # FIXME
-    if api_name is not None:
-        # Generate summary of the full document
-        full_summary = summarize(text, None, api_name, None, None, None)
-    else:
-        full_summary = "Full document summary not available."
-
-    chunks = improved_chunking_process(text, options)
-    total_chunks = len(chunks)
-
-    chunked_text_with_headers = []
-    for i, chunk in enumerate(chunks, 1):
-        chunk_text = chunk['text']
-        chunk_position = 1#DIRTY HACK #determine_chunk_position(chunk['metadata']['relative_position'])
-
-        chunk_header = f"""
-        Original Document: {file_name}
-        Full Document Summary: {full_summary}
-        Chunk: {i} of {total_chunks}
-        Position: {chunk_position}
-
-        --- Chunk Content ---
-        """
-
-        full_chunk_text = chunk_header + chunk_text
-        chunk['text'] = full_chunk_text
-        chunk['metadata']['file_name'] = file_name
-        chunked_text_with_headers.append(chunk)
-
-    return chunked_text_with_headers
-
-
-def create_openai_embedding(text: str, model: str) -> List[float]:
-    embedding = get_openai_embeddings(text, model)
-    return embedding
-
-
-
-#
-# End of File.
-#######################################################################################################################
+# Embeddings_Create.py
+# Description: Functions for Creating and managing Embeddings in ChromaDB with LLama.cpp/OpenAI/Transformers
+#
+# Imports:
+import logging
+import time
+from functools import wraps
+from threading import Lock, Timer
+from typing import List
+#
+# 3rd-Party Imports:
+import requests
+from transformers import AutoTokenizer, AutoModel
+import torch
+#
+# Local Imports:
+from App_Function_Libraries.LLM_API_Calls import get_openai_embeddings
+from App_Function_Libraries.Utils.Utils import load_comprehensive_config
+#
+#######################################################################################################################
+#
+# Functions:
+
+# FIXME - Add all globals to summarize.py
+loaded_config = load_comprehensive_config()
+embedding_provider = loaded_config['Embeddings']['embedding_provider']
+embedding_model = loaded_config['Embeddings']['embedding_model']
+embedding_api_url = loaded_config['Embeddings']['embedding_api_url']
+embedding_api_key = loaded_config['Embeddings']['embedding_api_key']
+
+# Embedding Chunking Settings
+chunk_size = loaded_config['Embeddings']['chunk_size']
+overlap = loaded_config['Embeddings']['overlap']
+
+
+# FIXME - Add logging
+
+
+class HuggingFaceEmbedder:
+    def __init__(self, model_name, timeout_seconds=120):  # Default timeout of 2 minutes
+        self.model_name = model_name
+        self.tokenizer = None
+        self.model = None
+        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        self.timeout_seconds = timeout_seconds
+        self.last_used_time = 0
+        self.unload_timer = None
+
+    def load_model(self):
+        if self.model is None:
+            self.tokenizer = AutoTokenizer.from_pretrained(self.model_name)
+            self.model = AutoModel.from_pretrained(self.model_name)
+            self.model.to(self.device)
+        self.last_used_time = time.time()
+        self.reset_timer()
+
+    def unload_model(self):
+        if self.model is not None:
+            del self.model
+            del self.tokenizer
+            if torch.cuda.is_available():
+                torch.cuda.empty_cache()
+            self.model = None
+            self.tokenizer = None
+        if self.unload_timer:
+            self.unload_timer.cancel()
+
+    def reset_timer(self):
+        if self.unload_timer:
+            self.unload_timer.cancel()
+        self.unload_timer = Timer(self.timeout_seconds, self.unload_model)
+        self.unload_timer.start()
+
+    def create_embeddings(self, texts):
+        self.load_model()
+        inputs = self.tokenizer(texts, return_tensors="pt", padding=True, truncation=True, max_length=512)
+        inputs = {k: v.to(self.device) for k, v in inputs.items()}
+        with torch.no_grad():
+            outputs = self.model(**inputs)
+        embeddings = outputs.last_hidden_state.mean(dim=1)
+        return embeddings.cpu().numpy()
+
+# Global variable to hold the embedder
+huggingface_embedder = None
+
+
+class RateLimiter:
+    def __init__(self, max_calls, period):
+        self.max_calls = max_calls
+        self.period = period
+        self.calls = []
+        self.lock = Lock()
+
+    def __call__(self, func):
+        def wrapper(*args, **kwargs):
+            with self.lock:
+                now = time.time()
+                self.calls = [call for call in self.calls if call > now - self.period]
+                if len(self.calls) >= self.max_calls:
+                    sleep_time = self.calls[0] - (now - self.period)
+                    time.sleep(sleep_time)
+                self.calls.append(time.time())
+            return func(*args, **kwargs)
+        return wrapper
+
+
+def exponential_backoff(max_retries=5, base_delay=1):
+    def decorator(func):
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            for attempt in range(max_retries):
+                try:
+                    return func(*args, **kwargs)
+                except Exception as e:
+                    if attempt == max_retries - 1:
+                        raise
+                    delay = base_delay * (2 ** attempt)
+                    logging.warning(f"Attempt {attempt + 1} failed. Retrying in {delay} seconds. Error: {str(e)}")
+                    time.sleep(delay)
+        return wrapper
+    return decorator
+
+
+# FIXME - refactor/setup to use config file & perform chunking
+@exponential_backoff()
+@RateLimiter(max_calls=50, period=60)  # Adjust these values based on API limits
+def create_embeddings_batch(texts: List[str], provider: str, model: str, api_url: str, timeout_seconds: int = 300) -> \
+List[List[float]]:
+    global huggingface_embedder
+
+    if provider.lower() == 'huggingface':
+        if huggingface_embedder is None or huggingface_embedder.model_name != model:
+            if huggingface_embedder is not None:
+                huggingface_embedder.unload_model()
+            huggingface_embedder = HuggingFaceEmbedder(model, timeout_seconds)
+
+        embeddings = huggingface_embedder.create_embeddings(texts).tolist()
+        return embeddings
+
+    elif provider.lower() == 'openai':
+        logging.debug(f"Creating embeddings for {len(texts)} texts using OpenAI API")
+        return [create_openai_embedding(text, model) for text in texts]
+
+    elif provider.lower() == 'local':
+        response = requests.post(
+            api_url,
+            json={"texts": texts, "model": model},
+            headers={"Authorization": f"Bearer {embedding_api_key}"}
+        )
+        if response.status_code == 200:
+            return response.json()['embeddings']
+        else:
+            raise Exception(f"Error from local API: {response.text}")
+    else:
+        raise ValueError(f"Unsupported embedding provider: {provider}")
+
+def create_embedding(text: str, provider: str, model: str, api_url: str) -> List[float]:
+    return create_embeddings_batch([text], provider, model, api_url)[0]
+
+# FIXME
+def create_stella_embeddings(text: str) -> List[float]:
+    if embedding_provider == 'local':
+        # Load the model and tokenizer
+        tokenizer = AutoTokenizer.from_pretrained("dunzhang/stella_en_400M_v5")
+        model = AutoModel.from_pretrained("dunzhang/stella_en_400M_v5")
+
+        # Tokenize and encode the text
+        inputs = tokenizer(text, return_tensors="pt", padding=True, truncation=True, max_length=512)
+
+        # Generate embeddings
+        with torch.no_grad():
+            outputs = model(**inputs)
+
+        # Use the mean of the last hidden state as the sentence embedding
+        embeddings = outputs.last_hidden_state.mean(dim=1)
+
+        return embeddings[0].tolist()  # Convert to list for consistency
+    elif embedding_provider == 'openai':
+        return get_openai_embeddings(text, embedding_model)
+    else:
+        raise ValueError(f"Unsupported embedding provider: {embedding_provider}")
+
+
+def create_openai_embedding(text: str, model: str) -> List[float]:
+    embedding = get_openai_embeddings(text, model)
+    return embedding
+
+
+
+
+#Dead
+# def create_local_embedding(text: str, model: str, api_url: str, api_key: str) -> List[float]:
+#     response = requests.post(
+#         api_url,
+#         json={"text": text, "model": model},
+#         headers={"Authorization": f"Bearer {api_key}"}
+#     )
+#     response.raise_for_status()
+#     return response.json().get('embedding', None)
+
+# Dead
+# def create_llamacpp_embedding(text: str, api_url: str) -> List[float]:
+#     response = requests.post(
+#         api_url,
+#         json={"input": text}
+#     )
+#     response.raise_for_status()
+#     return response.json()['embedding']
+
+# dead
+# def create_huggingface_embedding(text: str, model: str) -> List[float]:
+#     tokenizer = AutoTokenizer.from_pretrained(model)
+#     model = AutoModel.from_pretrained(model)
+#
+#     inputs = tokenizer(text, return_tensors="pt", padding=True, truncation=True, max_length=512)
+#     with torch.no_grad():
+#         outputs = model(**inputs)
+#
+#     embeddings = outputs.last_hidden_state.mean(dim=1)
+#     return embeddings[0].tolist()
+
+#
+# End of File.
+#######################################################################################################################

App_Function_Libraries/RAG/RAG_Libary_2.py

@@ -1,210 +1,340 @@
-# RAG_Library_2.py
-# Description: This script contains the main RAG pipeline function and related functions for the RAG pipeline.
-#
-# Import necessary modules and functions
-import configparser
-import logging
-import os
-from typing import Dict, Any, List, Optional
-# Local Imports
-#from App_Function_Libraries.RAG.ChromaDB_Library import process_and_store_content, vector_search, chroma_client
-from App_Function_Libraries.Article_Extractor_Lib import scrape_article
-from App_Function_Libraries.DB.DB_Manager import add_media_to_database, search_db, get_unprocessed_media, \
-    fetch_keywords_for_media
-from App_Function_Libraries.Utils.Utils import load_comprehensive_config
-#
-# 3rd-Party Imports
-import openai
-#
-########################################################################################################################
-#
-# Functions:
-
-# Initialize OpenAI client (adjust this based on your API key management)
-openai.api_key = "your-openai-api-key"
-
-# Get the directory of the current script
-current_dir = os.path.dirname(os.path.abspath(__file__))
-# Construct the path to the config file
-config_path = os.path.join(current_dir, 'Config_Files', 'config.txt')
-# Read the config file
-config = configparser.ConfigParser()
-# Read the configuration file
-config.read('config.txt')
-
-
-
-
-# RAG Search with keyword filtering
-def enhanced_rag_pipeline(query: str, api_choice: str, keywords: str = None) -> Dict[str, Any]:
-    try:
-        # Load embedding provider from config, or fallback to 'openai'
-        embedding_provider = config.get('Embeddings', 'provider', fallback='openai')
-
-        # Log the provider used
-        logging.debug(f"Using embedding provider: {embedding_provider}")
-
-        # Process keywords if provided
-        keyword_list = [k.strip().lower() for k in keywords.split(',')] if keywords else []
-        logging.debug(f"enhanced_rag_pipeline - Keywords: {keyword_list}")
-
-        # Fetch relevant media IDs based on keywords if keywords are provided
-        relevant_media_ids = fetch_relevant_media_ids(keyword_list) if keyword_list else None
-        logging.debug(f"enhanced_rag_pipeline - relevant media IDs: {relevant_media_ids}")
-
-        # Perform vector search
-        vector_results = perform_vector_search(query, relevant_media_ids)
-        logging.debug(f"enhanced_rag_pipeline - Vector search results: {vector_results}")
-
-        # Perform full-text search
-        fts_results = perform_full_text_search(query, relevant_media_ids)
-        logging.debug(f"enhanced_rag_pipeline - Full-text search results: {fts_results}")
-
-        # Combine results
-        all_results = vector_results + fts_results
-        # FIXME
-        if not all_results:
-            logging.info(f"No results found. Query: {query}, Keywords: {keywords}")
-            return {
-                "answer": "I couldn't find any relevant information based on your query and keywords.",
-                "context": ""
-            }
-
-        # FIXME - Apply Re-Ranking of results here
-        apply_re_ranking = False
-        if apply_re_ranking:
-            # Implement re-ranking logic here
-            pass
-        # Extract content from results
-        context = "\n".join([result['content'] for result in all_results[:10]])  # Limit to top 10 results
-        logging.debug(f"Context length: {len(context)}")
-        logging.debug(f"Context: {context[:200]}")
-        # Generate answer using the selected API
-        answer = generate_answer(api_choice, context, query)
-
-        return {
-            "answer": answer,
-            "context": context
-        }
-    except Exception as e:
-        logging.error(f"Error in enhanced_rag_pipeline: {str(e)}")
-        return {
-            "answer": "An error occurred while processing your request.",
-            "context": ""
-        }
-
-
-
-def generate_answer(api_choice: str, context: str, query: str) -> str:
-    logging.debug("Entering generate_answer function")
-    config = load_comprehensive_config()
-    logging.debug(f"Config sections: {config.sections()}")
-    prompt = f"Context: {context}\n\nQuestion: {query}"
-    if api_choice == "OpenAI":
-        from App_Function_Libraries.Summarization_General_Lib import summarize_with_openai
-        return summarize_with_openai(config['API']['openai_api_key'], prompt, "")
-    elif api_choice == "Anthropic":
-        from App_Function_Libraries.Summarization_General_Lib import summarize_with_anthropic
-        return summarize_with_anthropic(config['API']['anthropic_api_key'], prompt, "")
-    elif api_choice == "Cohere":
-        from App_Function_Libraries.Summarization_General_Lib import summarize_with_cohere
-        return summarize_with_cohere(config['API']['cohere_api_key'], prompt, "")
-    elif api_choice == "Groq":
-        from App_Function_Libraries.Summarization_General_Lib import summarize_with_groq
-        return summarize_with_groq(config['API']['groq_api_key'], prompt, "")
-    elif api_choice == "OpenRouter":
-        from App_Function_Libraries.Summarization_General_Lib import summarize_with_openrouter
-        return summarize_with_openrouter(config['API']['openrouter_api_key'], prompt, "")
-    elif api_choice == "HuggingFace":
-        from App_Function_Libraries.Summarization_General_Lib import summarize_with_huggingface
-        return summarize_with_huggingface(config['API']['huggingface_api_key'], prompt, "")
-    elif api_choice == "DeepSeek":
-        from App_Function_Libraries.Summarization_General_Lib import summarize_with_deepseek
-        return summarize_with_deepseek(config['API']['deepseek_api_key'], prompt, "")
-    elif api_choice == "Mistral":
-        from App_Function_Libraries.Summarization_General_Lib import summarize_with_mistral
-        return summarize_with_mistral(config['API']['mistral_api_key'], prompt, "")
-    elif api_choice == "Local-LLM":
-        from App_Function_Libraries.Local_Summarization_Lib import summarize_with_local_llm
-        return summarize_with_local_llm(config['API']['local_llm_path'], prompt, "")
-    elif api_choice == "Llama.cpp":
-        from App_Function_Libraries.Local_Summarization_Lib import summarize_with_llama
-        return summarize_with_llama(config['API']['llama_api_key'], prompt, "")
-    elif api_choice == "Kobold":
-        from App_Function_Libraries.Local_Summarization_Lib import summarize_with_kobold
-        return summarize_with_kobold(config['API']['kobold_api_key'], prompt, "")
-    elif api_choice == "Ooba":
-        from App_Function_Libraries.Local_Summarization_Lib import summarize_with_oobabooga
-        return summarize_with_oobabooga(config['API']['ooba_api_key'], prompt, "")
-    elif api_choice == "TabbyAPI":
-        from App_Function_Libraries.Local_Summarization_Lib import summarize_with_tabbyapi
-        return summarize_with_tabbyapi(config['API']['tabby_api_key'], prompt, "")
-    elif api_choice == "vLLM":
-        from App_Function_Libraries.Local_Summarization_Lib import summarize_with_vllm
-        return summarize_with_vllm(config['API']['vllm_api_key'], prompt, "")
-    elif api_choice == "ollama":
-        from App_Function_Libraries.Local_Summarization_Lib import summarize_with_ollama
-        return summarize_with_ollama(config['API']['ollama_api_key'], prompt, "")
-    else:
-        raise ValueError(f"Unsupported API choice: {api_choice}")
-
-
-def perform_full_text_search(query: str, relevant_media_ids: List[str] = None) -> List[Dict[str, Any]]:
-    fts_results = search_db(query, ["content"], "", page=1, results_per_page=5)
-    filtered_fts_results = [
-        {
-            "content": result['content'],
-            "metadata": {"media_id": result['id']}
-        }
-        for result in fts_results
-        if relevant_media_ids is None or result['id'] in relevant_media_ids
-    ]
-    return filtered_fts_results
-
-
-def fetch_relevant_media_ids(keywords: List[str]) -> List[int]:
-    relevant_ids = set()
-    try:
-        for keyword in keywords:
-            media_ids = fetch_keywords_for_media(keyword)
-            relevant_ids.update(media_ids)
-    except Exception as e:
-        logging.error(f"Error fetching relevant media IDs: {str(e)}")
-    return list(relevant_ids)
-
-
-
-
-# Example usage:
-# 1. Initialize the system:
-# create_tables(db)  # Ensure FTS tables are set up
-#
-# 2. Create ChromaDB
-# chroma_client = ChromaDBClient()
-#
-# 3. Create Embeddings
-# Store embeddings in ChromaDB
-# preprocess_all_content() or create_embeddings()
-#
-# 4. Perform RAG search across all content:
-# result = rag_search("What are the key points about climate change?")
-# print(result['answer'])
-#
-# (Extra)5. Perform RAG on a specific URL:
-# result = rag_pipeline("https://example.com/article", "What is the main topic of this article?")
-# print(result['answer'])
-#
-########################################################################################################################
-
-
-############################################################################################################
-#
-# ElasticSearch Retriever
-
-# https://github.com/langchain-ai/langchain/tree/44e3e2391c48bfd0a8e6a20adde0b6567f4f43c3/templates/rag-elasticsearch
-#
-# https://github.com/langchain-ai/langchain/tree/44e3e2391c48bfd0a8e6a20adde0b6567f4f43c3/templates/rag-self-query
-
-#
-# End of RAG_Library_2.py
-############################################################################################################
+# RAG_Library_2.py
+# Description: This script contains the main RAG pipeline function and related functions for the RAG pipeline.
+#
+# Import necessary modules and functions
+import configparser
+import logging
+import os
+from typing import Dict, Any, List, Optional
+# Local Imports
+from App_Function_Libraries.RAG.ChromaDB_Library import process_and_store_content, vector_search, chroma_client
+from App_Function_Libraries.Web_Scraping.Article_Extractor_Lib import scrape_article
+from App_Function_Libraries.DB.DB_Manager import add_media_to_database, search_db, get_unprocessed_media, \
+    fetch_keywords_for_media
+from App_Function_Libraries.Utils.Utils import load_comprehensive_config
+#
+# 3rd-Party Imports
+import openai
+#
+########################################################################################################################
+#
+# Functions:
+
+# Initialize OpenAI client (adjust this based on your API key management)
+openai.api_key = "your-openai-api-key"
+
+# Get the directory of the current script
+current_dir = os.path.dirname(os.path.abspath(__file__))
+# Construct the path to the config file
+config_path = os.path.join(current_dir, 'Config_Files', 'config.txt')
+# Read the config file
+config = configparser.ConfigParser()
+# Read the configuration file
+config.read('config.txt')
+
+# Main RAG pipeline function
+def rag_pipeline(url: str, query: str, api_choice=None) -> Dict[str, Any]:
+    try:
+        # Extract content
+        try:
+            article_data = scrape_article(url)
+            content = article_data['content']
+            title = article_data['title']
+        except Exception as e:
+            logging.error(f"Error scraping article: {str(e)}")
+            return {"error": "Failed to scrape article", "details": str(e)}
+
+        # Store the article in the database and get the media_id
+        try:
+            media_id = add_media_to_database(url, title, 'article', content)
+        except Exception as e:
+            logging.error(f"Error adding article to database: {str(e)}")
+            return {"error": "Failed to store article in database", "details": str(e)}
+
+        # Process and store content
+        collection_name = f"article_{media_id}"
+        try:
+            # FIXME
+            # def process_and_store_content(content: str, collection_name: str, media_id: int, file_name: str,
+            #                               create_embeddings: bool = False, create_summary: bool = False,
+            #                               api_name: str = None):
+            process_and_store_content(content, collection_name, media_id, title)
+        except Exception as e:
+            logging.error(f"Error processing and storing content: {str(e)}")
+            return {"error": "Failed to process and store content", "details": str(e)}
+
+        # Perform searches
+        try:
+            vector_results = vector_search(collection_name, query, k=5)
+            fts_results = search_db(query, ["content"], "", page=1, results_per_page=5)
+        except Exception as e:
+            logging.error(f"Error performing searches: {str(e)}")
+            return {"error": "Failed to perform searches", "details": str(e)}
+
+        # Combine results with error handling for missing 'content' key
+        all_results = []
+        for result in vector_results + fts_results:
+            if isinstance(result, dict) and 'content' in result:
+                all_results.append(result['content'])
+            else:
+                logging.warning(f"Unexpected result format: {result}")
+                all_results.append(str(result))
+
+        context = "\n".join(all_results)
+
+        # Generate answer using the selected API
+        try:
+            answer = generate_answer(api_choice, context, query)
+        except Exception as e:
+            logging.error(f"Error generating answer: {str(e)}")
+            return {"error": "Failed to generate answer", "details": str(e)}
+
+        return {
+            "answer": answer,
+            "context": context
+        }
+
+    except Exception as e:
+        logging.error(f"Unexpected error in rag_pipeline: {str(e)}")
+        return {"error": "An unexpected error occurred", "details": str(e)}
+
+
+
+# RAG Search with keyword filtering
+def enhanced_rag_pipeline(query: str, api_choice: str, keywords: str = None) -> Dict[str, Any]:
+    try:
+        # Load embedding provider from config, or fallback to 'openai'
+        embedding_provider = config.get('Embeddings', 'provider', fallback='openai')
+
+        # Log the provider used
+        logging.debug(f"Using embedding provider: {embedding_provider}")
+
+        # Process keywords if provided
+        keyword_list = [k.strip().lower() for k in keywords.split(',')] if keywords else []
+        logging.debug(f"enhanced_rag_pipeline - Keywords: {keyword_list}")
+
+        # Fetch relevant media IDs based on keywords if keywords are provided
+        relevant_media_ids = fetch_relevant_media_ids(keyword_list) if keyword_list else None
+        logging.debug(f"enhanced_rag_pipeline - relevant media IDs: {relevant_media_ids}")
+
+        # Perform vector search
+        vector_results = perform_vector_search(query, relevant_media_ids)
+        logging.debug(f"enhanced_rag_pipeline - Vector search results: {vector_results}")
+
+        # Perform full-text search
+        fts_results = perform_full_text_search(query, relevant_media_ids)
+        logging.debug(f"enhanced_rag_pipeline - Full-text search results: {fts_results}")
+
+        # Combine results
+        all_results = vector_results + fts_results
+        # FIXME
+        if not all_results:
+            logging.info(f"No results found. Query: {query}, Keywords: {keywords}")
+            return {
+                "answer": "I couldn't find any relevant information based on your query and keywords.",
+                "context": ""
+            }
+
+        # FIXME - Apply Re-Ranking of results here
+        apply_re_ranking = False
+        if apply_re_ranking:
+            # Implement re-ranking logic here
+            pass
+        # Extract content from results
+        context = "\n".join([result['content'] for result in all_results[:10]])  # Limit to top 10 results
+        logging.debug(f"Context length: {len(context)}")
+        logging.debug(f"Context: {context[:200]}")
+        # Generate answer using the selected API
+        answer = generate_answer(api_choice, context, query)
+
+        return {
+            "answer": answer,
+            "context": context
+        }
+    except Exception as e:
+        logging.error(f"Error in enhanced_rag_pipeline: {str(e)}")
+        return {
+            "answer": "An error occurred while processing your request.",
+            "context": ""
+        }
+
+
+def generate_answer(api_choice: str, context: str, query: str) -> str:
+    logging.debug("Entering generate_answer function")
+    config = load_comprehensive_config()
+    logging.debug(f"Config sections: {config.sections()}")
+    prompt = f"Context: {context}\n\nQuestion: {query}"
+    if api_choice == "OpenAI":
+        from App_Function_Libraries.Summarization.Summarization_General_Lib import summarize_with_openai
+        return summarize_with_openai(config['API']['openai_api_key'], prompt, "")
+    elif api_choice == "Anthropic":
+        from App_Function_Libraries.Summarization.Summarization_General_Lib import summarize_with_anthropic
+        return summarize_with_anthropic(config['API']['anthropic_api_key'], prompt, "")
+    elif api_choice == "Cohere":
+        from App_Function_Libraries.Summarization.Summarization_General_Lib import summarize_with_cohere
+        return summarize_with_cohere(config['API']['cohere_api_key'], prompt, "")
+    elif api_choice == "Groq":
+        from App_Function_Libraries.Summarization.Summarization_General_Lib import summarize_with_groq
+        return summarize_with_groq(config['API']['groq_api_key'], prompt, "")
+    elif api_choice == "OpenRouter":
+        from App_Function_Libraries.Summarization.Summarization_General_Lib import summarize_with_openrouter
+        return summarize_with_openrouter(config['API']['openrouter_api_key'], prompt, "")
+    elif api_choice == "HuggingFace":
+        from App_Function_Libraries.Summarization.Summarization_General_Lib import summarize_with_huggingface
+        return summarize_with_huggingface(config['API']['huggingface_api_key'], prompt, "")
+    elif api_choice == "DeepSeek":
+        from App_Function_Libraries.Summarization.Summarization_General_Lib import summarize_with_deepseek
+        return summarize_with_deepseek(config['API']['deepseek_api_key'], prompt, "")
+    elif api_choice == "Mistral":
+        from App_Function_Libraries.Summarization.Summarization_General_Lib import summarize_with_mistral
+        return summarize_with_mistral(config['API']['mistral_api_key'], prompt, "")
+    elif api_choice == "Local-LLM":
+        from App_Function_Libraries.Summarization.Local_Summarization_Lib import summarize_with_local_llm
+        return summarize_with_local_llm(config['API']['local_llm_path'], prompt, "")
+    elif api_choice == "Llama.cpp":
+        from App_Function_Libraries.Summarization.Local_Summarization_Lib import summarize_with_llama
+        return summarize_with_llama(config['API']['llama_api_key'], prompt, "")
+    elif api_choice == "Kobold":
+        from App_Function_Libraries.Summarization.Local_Summarization_Lib import summarize_with_kobold
+        return summarize_with_kobold(config['API']['kobold_api_key'], prompt, "")
+    elif api_choice == "Ooba":
+        from App_Function_Libraries.Summarization.Local_Summarization_Lib import summarize_with_oobabooga
+        return summarize_with_oobabooga(config['API']['ooba_api_key'], prompt, "")
+    elif api_choice == "TabbyAPI":
+        from App_Function_Libraries.Summarization.Local_Summarization_Lib import summarize_with_tabbyapi
+        return summarize_with_tabbyapi(config['API']['tabby_api_key'], prompt, "")
+    elif api_choice == "vLLM":
+        from App_Function_Libraries.Summarization.Local_Summarization_Lib import summarize_with_vllm
+        return summarize_with_vllm(config['API']['vllm_api_key'], prompt, "")
+    elif api_choice == "ollama":
+        from App_Function_Libraries.Summarization.Local_Summarization_Lib import summarize_with_ollama
+        return summarize_with_ollama(config['API']['ollama_api_key'], prompt, "")
+    else:
+        raise ValueError(f"Unsupported API choice: {api_choice}")
+
+# Function to preprocess and store all existing content in the database
+def preprocess_all_content():
+    unprocessed_media = get_unprocessed_media()
+    for row in unprocessed_media:
+        media_id = row[0]
+        content = row[1]
+        media_type = row[2]
+        collection_name = f"{media_type}_{media_id}"
+        # FIXME
+        # def process_and_store_content(content: str, collection_name: str, media_id: int, file_name: str,
+        #                               create_embeddings: bool = False, create_summary: bool = False,
+        #                               api_name: str = None):
+        process_and_store_content(content, collection_name, media_id, "")
+
+
+def perform_vector_search(query: str, relevant_media_ids: List[str] = None) -> List[Dict[str, Any]]:
+    all_collections = chroma_client.list_collections()
+    vector_results = []
+    for collection in all_collections:
+        collection_results = vector_search(collection.name, query, k=5)
+        filtered_results = [
+            result for result in collection_results
+            if relevant_media_ids is None or result['metadata'].get('media_id') in relevant_media_ids
+        ]
+        vector_results.extend(filtered_results)
+    return vector_results
+
+
+def perform_full_text_search(query: str, relevant_media_ids: List[str] = None) -> List[Dict[str, Any]]:
+    fts_results = search_db(query, ["content"], "", page=1, results_per_page=5)
+    filtered_fts_results = [
+        {
+            "content": result['content'],
+            "metadata": {"media_id": result['id']}
+        }
+        for result in fts_results
+        if relevant_media_ids is None or result['id'] in relevant_media_ids
+    ]
+    return filtered_fts_results
+
+
+def fetch_relevant_media_ids(keywords: List[str]) -> List[int]:
+    relevant_ids = set()
+    try:
+        for keyword in keywords:
+            media_ids = fetch_keywords_for_media(keyword)
+            relevant_ids.update(media_ids)
+    except Exception as e:
+        logging.error(f"Error fetching relevant media IDs: {str(e)}")
+    return list(relevant_ids)
+
+
+def filter_results_by_keywords(results: List[Dict[str, Any]], keywords: List[str]) -> List[Dict[str, Any]]:
+    if not keywords:
+        return results
+
+    filtered_results = []
+    for result in results:
+        try:
+            metadata = result.get('metadata', {})
+            if metadata is None:
+                logging.warning(f"No metadata found for result: {result}")
+                continue
+            if not isinstance(metadata, dict):
+                logging.warning(f"Unexpected metadata type: {type(metadata)}. Expected dict.")
+                continue
+
+            media_id = metadata.get('media_id')
+            if media_id is None:
+                logging.warning(f"No media_id found in metadata: {metadata}")
+                continue
+
+            media_keywords = fetch_keywords_for_media(media_id)
+            if any(keyword.lower() in [mk.lower() for mk in media_keywords] for keyword in keywords):
+                filtered_results.append(result)
+        except Exception as e:
+            logging.error(f"Error processing result: {result}. Error: {str(e)}")
+
+    return filtered_results
+
+# FIXME: to be implememted
+def extract_media_id_from_result(result: str) -> Optional[int]:
+    # Implement this function based on how you store the media_id in your results
+    # For example, if it's stored at the beginning of each result:
+    try:
+        return int(result.split('_')[0])
+    except (IndexError, ValueError):
+        logging.error(f"Failed to extract media_id from result: {result}")
+        return None
+
+
+
+
+# Example usage:
+# 1. Initialize the system:
+# create_tables(db)  # Ensure FTS tables are set up
+#
+# 2. Create ChromaDB
+# chroma_client = ChromaDBClient()
+#
+# 3. Create Embeddings
+# Store embeddings in ChromaDB
+# preprocess_all_content() or create_embeddings()
+#
+# 4. Perform RAG search across all content:
+# result = rag_search("What are the key points about climate change?")
+# print(result['answer'])
+#
+# (Extra)5. Perform RAG on a specific URL:
+# result = rag_pipeline("https://example.com/article", "What is the main topic of this article?")
+# print(result['answer'])
+#
+########################################################################################################################
+
+
+############################################################################################################
+#
+# ElasticSearch Retriever
+
+# https://github.com/langchain-ai/langchain/tree/44e3e2391c48bfd0a8e6a20adde0b6567f4f43c3/templates/rag-elasticsearch
+#
+# https://github.com/langchain-ai/langchain/tree/44e3e2391c48bfd0a8e6a20adde0b6567f4f43c3/templates/rag-self-query
+
+#
+# End of RAG_Library_2.py
+############################################################################################################

App_Function_Libraries/Summarization/Summarization_General_Lib.py

@@ -710,11 +710,9 @@ def summarize_with_openrouter(api_key, input_data, custom_prompt_arg, temp=None,
 
 def summarize_with_huggingface(api_key, input_data, custom_prompt_arg, temp=None):
     loaded_config_data = load_and_log_configs()
-    global huggingface_api_key
     logging.debug("HuggingFace: Summarization process starting...")
     try:
         logging.debug("HuggingFace: Loading and validating configurations")
-        loaded_config_data = load_and_log_configs()
         if loaded_config_data is None:
             logging.error("Failed to load configuration data")
             huggingface_api_key = None
@@ -726,6 +724,7 @@ def summarize_with_huggingface(api_key, input_data, custom_prompt_arg, temp=None
             else:
                 # If no parameter is provided, use the key from the config
                 huggingface_api_key = loaded_config_data['api_keys'].get('huggingface')
+                logging.debug(f"HuggingFace: API key from config: {huggingface_api_key[:5]}...{huggingface_api_key[-5:]}")
                 if huggingface_api_key:
                     logging.info("HuggingFace: Using API key from config file")
                 else:
@@ -738,7 +737,6 @@ def summarize_with_huggingface(api_key, input_data, custom_prompt_arg, temp=None
             # FIXME
             # For example: raise ValueError("No valid Anthropic API key available")
 
-
         logging.debug(f"HuggingFace: Using API Key: {huggingface_api_key[:5]}...{huggingface_api_key[-5:]}")
 
         if isinstance(input_data, str) and os.path.isfile(input_data):
@@ -775,21 +773,24 @@ def summarize_with_huggingface(api_key, input_data, custom_prompt_arg, temp=None
         if temp is None:
             temp = 0.1
         temp = float(temp)
-        huggingface_prompt = f"{text}\n\n\n\n{custom_prompt_arg}"
+        huggingface_prompt = f"{custom_prompt_arg}\n\n\n{text}"
         logging.debug("huggingface: Prompt being sent is {huggingface_prompt}")
         data = {
-            "inputs": text,
-            "parameters": {"max_length": 512, "min_length": 100}  # You can adjust max_length and min_length as needed
+            "inputs": huggingface_prompt,
+            "max_tokens": 4096,
+            "stream": False,
+            "temperature": temp
         }
 
         logging.debug("huggingface: Submitting request...")
         response = requests.post(API_URL, headers=headers, json=data)
 
         if response.status_code == 200:
-            summary = response.json()[0]['generated_text'].strip()
+            print(response.json())
+            chat_response = response.json()[0]['generated_text'].strip()
             logging.debug("huggingface: Summarization successful")
-            print("Summarization successful.")
-            return summary
+            print("Chat request successful.")
+            return chat_response
         else:
             logging.error(f"huggingface: Summarization failed with status code {response.status_code}: {response.text}")
             return f"Failed to process summary, status code {response.status_code}: {response.text}"

App_Function_Libraries/Utils/Utils.py

@@ -18,6 +18,7 @@
 #
 #
 ####################
+#
 # Import necessary libraries
 import configparser
 import hashlib
@@ -29,14 +30,15 @@ import time
 from datetime import timedelta
 from typing import Union, AnyStr
 from urllib.parse import urlparse, parse_qs, urlencode, urlunparse
-
+#
+# Non-Local Imports
 import requests
 import unicodedata
 from tqdm import tqdm
-
+#
 #######################################################################################################################
-# Function Definitions
 #
+# Function Definitions
 
 def extract_text_from_segments(segments):
     logging.debug(f"Segments received: {segments}")
@@ -63,10 +65,6 @@ def extract_text_from_segments(segments):
         logging.error(f"Unable to extract text from segments: {segments}")
         return "Error: Unable to extract transcription"
 
-def import_data(file):
-    # Implement this function to import data from a file
-    pass
-
 #
 #
 #######################
@@ -124,23 +122,35 @@ def load_comprehensive_config():
 
 
 def get_project_root():
-    """Get the project root directory."""
-    return os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+    # Get the directory of the current file (Utils.py)
+    current_dir = os.path.dirname(os.path.abspath(__file__))
+    # Go up two levels to reach the project root
+    # Assuming the structure is: project_root/App_Function_Libraries/Utils/Utils.py
+    project_root = os.path.dirname(os.path.dirname(current_dir))
+    return project_root
 
 def get_database_dir():
-    return get_project_relative_path('Databases')
+    """Get the database directory (/tldw/Databases/)."""
+    db_dir = os.path.join(get_project_root(), 'Databases')
+    logging.debug(f"Database directory: {db_dir}")
+    return db_dir
 
 def get_database_path(db_name: Union[str, os.PathLike[AnyStr]]) -> str:
     """Get the full path for a database file."""
-    return os.path.join(get_database_dir(), str(db_name))
+    path = os.path.join(get_database_dir(), str(db_name))
+    logging.debug(f"Database path for {db_name}: {path}")
+    return path
 
 def get_project_relative_path(relative_path: Union[str, os.PathLike[AnyStr]]) -> str:
     """Convert a relative path to a path relative to the project root."""
-    return os.path.join(get_project_root(), str(relative_path))
+    path = os.path.join(get_project_root(), str(relative_path))
+    logging.debug(f"Project relative path for {relative_path}: {path}")
+    return path
 
 def get_chromadb_path():
-    project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-    return os.path.join(project_root, 'Databases', 'chroma_db')
+    path = os.path.join(get_project_root(), 'Databases', 'chroma_db')
+    logging.debug(f"ChromaDB path: {path}")
+    return path
 
 def ensure_directory_exists(path):
     """Ensure that a directory exists, creating it if necessary."""
@@ -676,3 +686,18 @@ def get_db_config():
 #
 # End of DB Config Loading
 #######################################################################################################################
+
+def format_text_with_line_breaks(text):
+    # Split the text into sentences and add line breaks
+    sentences = text.replace('. ', '.<br>').replace('? ', '?<br>').replace('! ', '!<br>')
+    return sentences
+
+#######################################################################################################################
+#
+# File Handling Functions
+
+
+
+#
+# End of File Handling Functions
+#######################################################################################################################

App_Function_Libraries/Utils/__init__.py

@@ -1,5 +0,0 @@
-from .Utils import is_valid_url, load_and_log_configs, extract_text_from_segments, load_comprehensive_config, format_metadata_as_text, convert_to_seconds, save_to_file,\
-    save_segments_to_json, download_file, create_download_directory, safe_read_file, generate_unique_filename, generate_unique_identifier, is_valid_url, verify_checksum,\
-    normalize_title, clean_youtube_url, sanitize_filename, format_transcription, format_file_path, get_db_config
-
-downloaded_files = []