import os import re from langchain.agents import initialize_agent, Tool from langchain_community.vectorstores import FAISS from langchain_openai import OpenAIEmbeddings from langchain_openai import ChatOpenAI from langchain_core.prompts import PromptTemplate from langchain_core.output_parsers import StrOutputParser import pandas as pd from pandasai.llm.openai import OpenAI from pandasai import SmartDataframe # Initialize a blank DataFrame as a global variable global_df = pd.DataFrame() class ChatHandler: def __init__(self, vector_db_path, open_api_key, grok_api_key,db_final): self.vector_db_path = vector_db_path self.openai_embeddings = OpenAIEmbeddings(api_key=open_api_key) self.llm_openai = ChatOpenAI(model_name="gpt-4o-mini", api_key=open_api_key, max_tokens=500, temperature=0.2) self.grok_api_key = grok_api_key self.openai_api_key = open_api_key self.sql_db = db_final def _load_documents_from_vector_db(self, query): """Fetch relevant documents from the vector database.""" results = [] # Debug: Print the query being processed print(f"Processing query: {query}") for root, dirs, files in os.walk(self.vector_db_path): print(f"Searching in directory: {root}") # Debug: Current directory being processed for dir in dirs: index_path = os.path.join(root, dir, "index.faiss") # Debug: Check if FAISS index exists if os.path.exists(index_path): print(f"Found FAISS index at: {index_path}") # Load the FAISS vector store try: vector_store = FAISS.load_local( os.path.join(root, dir), self.openai_embeddings, allow_dangerous_deserialization=True ) print(f"Loaded FAISS vector store from: {os.path.join(root, dir)}") except Exception as e: print(f"Error loading FAISS store: {e}") continue # Perform similarity search try: response_with_scores = vector_store.similarity_search_with_relevance_scores(query, k=100) #print(response_with_scores) print(f"Similarity search returned {len(response_with_scores)} results.") filtered_results = [ (doc, score) for doc, score in response_with_scores if score is not None and score > 0.7 #and material_name.lower() in doc.page_content.lower() # Check material name in document ] print(f"Filtered results: {filtered_results}") response_with_scores = filtered_results # Debug: Print each document and score for doc, score in response_with_scores: print(f"Document: {doc.page_content[:100]}... Score: {score}") results.extend([(doc.page_content, score) for doc, score in response_with_scores]) except Exception as e: print(f"Error during similarity search: {e}") # Sort and return results sorted_results = [doc for doc, score in sorted(results, key=lambda x: -x[1])] print(f"Total results after sorting: {len(sorted_results)}") return sorted_results def _load_schema_from_database(self, query): """ Fetch database schema, generate a SQL query from the user's question, and execute it. """ try: # Fetch the schema schema = self.sql_db.get_table_info() # Define the prompt template template_query_generation = """ Based on the table schema below, write a mySQL query with correct syntax that would answer the user's question. Only write the SQL query without explanations. Schema: {schema} Question: {question} SQL Query: """ prompt = PromptTemplate( input_variables=["schema", "question"], template=template_query_generation ) # Initialize the language modelgpt-4o-mini llm = ChatOpenAI(model_name="gpt-4o-mini", api_key=self.openai_api_key, max_tokens=500, temperature=0.2) # Create the runnable sequence chain = prompt | llm | StrOutputParser() # Generate the SQL query sql_query = chain.invoke({"schema": schema, "question": query}).strip() if not sql_query: return "Could not generate an SQL query for your question." # Execute the SQL query try: result = self.sql_db.run(sql_query) print(f"SQL query executed successfully. Result: {result}") except Exception as e: print(f"Error executing SQL query: {str(e)}") return f"As you know I am still learning at this moment I am not able to respond to your question.\nThank you for your patience!" # If no result, return an appropriate message if not result: return "Query executed, but no results were returned." # Return the result return result except Exception as e: print( f"Error fetching schema details or processing query: {str(e)}") return f"As you know I am still learning at this moment I am not able to respond to your question.\nThank you for your patience!" def answer_question(self, query, visual_query): global global_df """Determine whether to use vector database or SQL database for the query.""" tools = [ # { # "name": "Document Vector Store", # "function": lambda q: "\n".join(self._load_documents_from_vector_db(q)), # "description": """Search within the uploaded documents stored in the vector database. # Display the response as a combination of response summary and the response data in the form of table. # If the user requested comparison between two or more years, data should be shown for all the years. (For example, if the user requested from 2020 to 2024, then display the output table with the columns [Month, Material value in 2020, Material value in 2021, Material value in 2022, Material value in 2023, Material value in 2024]) so that the records will be displayed for all the months from Jaunary to December across the years. # display the material quantity in blue colour if it the 'Type' column value is 'actual'. # display the Material Quanity in red colour if its value is 'predicted'. # include the table data in the Final answer of agent executor invoke.""", # }, { "name": "Database Schema", "function": lambda q: self._load_schema_from_database(q), "description": """Search within the mysql database schema and generate SQL-based responses. write mySQL query with correct syntax. The database has single table 'tp_material_forecast' which contains the columns 'date', 'material_name', 'material_quantity', and 'type'. Frame the query only with these four columns. If the material name is given, frame the query in such a way that the material_name is not case-sensitive. If the material name is not present in the table, return the proper message as "This material name is not in the database". Do not give any false values if the material name is not available in database. display the response as a combination of response summary and the response data in the form of table. If the response has month column, display the month as name For example, January instead of displaying as 1. If the user requested comparison between two or more years or the user asks for the data for all years, data should be shown for all the years with month as first column and the years like 2020, 2021 etc as the adjacent columns. Do not show everything in the same column. (For example, if the user requested from 2020 to 2024, then display the output table with the columns [Month, Material value in 2020, Material value in 2020, Material value in 2021, Material value in 2022, Material value in 2023, Material value in 2024]) so that the records will be displayed for all the months from Jaunary to December across the years. Always display the table data at the end in the Final answer even if it has a single value. If there is any error while executing the user question, kindly display the error message as 'As you know I am still learning at this moment I am not able to respond to your question.\nThank you for your patience!'""", }, ] agent_prompt = PromptTemplate( input_variables=["input", "agent_scratchpad"], template=""" You are a highly skilled AI assistant specializing in mysql database. I have a mysql database for material demand forecasts with columns as 'date', 'material_name', 'material_quantity', and 'type'. The data includes historical demand information for various items. 1. The uploaded document includes: - **Date:** The date of demand entry. - **Material Name:** The name of the material or equipment. - **Material Quantity:** The number of units actual or predicted. - **Type:** Type contains actual or forecasted, actual represents the actual material utilized and forecasted represents the prediction by ai model. 2. I may ask questions such as: - Forecasting future demand for specific items. - Analyzing trends or patterns for materials over time. - Summarizing the highest or lowest demands within a specific date range. - Comparing demand values between two or more items. Your task: - If the query relates to forecasting, extract the necessary information from it and provide precise, professional, and data-driven responses. Make sure your answers are aligned with the uploaded document, depending on the context of the query. display the response as mentioned in the tool description. include the table data in the Final answer even if there is a single value. Do not display the first line and the last line of the table as ''' Tools available to you: {tools} Input Question: {input} {agent_scratchpad} """, ) # Initialize the agent agent = initialize_agent( tools=[Tool(name=t["name"], func=t["function"], description=t["description"]) for t in tools], llm=self.llm_openai, agent="zero-shot-react-description", verbose=True, prompt=agent_prompt ) try: response = agent.invoke(query, handle_parsing_errors=True) print(f"response:{response}") if isinstance(response, dict) and "output" in response: response = response["output"] # Extract and return only the output field else: response = response # Fallback if output field is not present if visual_query is not None: # Check if the response contains table-like formatting if "|" in response and "---" in response: print("Table data is present in the response.") #convert table data into dataframe # Extract table rows table_pattern = r"\|.*\|" import re table_data = re.findall(table_pattern, response) # Remove separator lines (like |---|---|) filtered_data = [row for row in table_data if not re.match(r"\|\-+\|", row)] # Split rows into columns split_data = [row.strip('|').split('|') for row in filtered_data] # Create DataFrame columns = [col.strip() for col in split_data[0]] # First row is the header data = [list(map(str.strip, row)) for row in split_data[1:]] # Remaining rows are data global_df = pd.DataFrame(data, columns=columns) # Function to convert datatypes global_df = convert_column_types(global_df) print(f"Dataframe created from response:\n{global_df}") visual_response = create_visualization_csv(visual_query) else: print("No table data found in the response.") global_df = unstructured_text_to_df(response) print(global_df) visual_response = create_visualization_csv(visual_query) print(visual_response) else: visual_response = None return response, visual_response except Exception as e: print(f"Error while processing your query: {str(e)}") return f"As you know I am still learning at this moment I am not able to respond to your question.\nThank you for your patience!" , None def create_visualization_csv(visual_query): global_df #import matplotlib #matplotlib.use('TkAgg') # Replace with 'QtAgg' or 'MacOSX' if on macOS visual_query = visual_query + """ create chart with suitable x and y axis as user requested. use proper axis scale and mention axis values properly. Do not miss any values. Mention only month name in date axis and not the numbers or the date. Do not place legend in the middle of the chart. Place the legend in such a way that the plotted chart is not hidden. Return the image path only after plotting all the values.""" llm_chart = OpenAI() #from pandasai import PandasAI #pandas_ai = PandasAI(llm_chart, show_plots=False) #pandas_ai = PandasAI(show_plots=False) # Avoids attempting to show plots sdf = SmartDataframe(global_df, config={"llm": llm_chart}) llm_response = sdf.chat(visual_query) if "no result" in llm_response: return f"As you know I am still learning at this moment I am not able to respond to your question.\nThank you for your patience!" return llm_response def convert_column_types(df): for col in df.columns: # Try to convert to integer if all(df[col].str.isdigit()): df[col] = df[col].astype(int) # Try to convert to datetime else: try: df[col] = pd.to_datetime(df[col], format='%Y-%m-%d', errors='raise') except ValueError: # Leave as string if neither integer nor date pass return df def unstructured_text_to_df(text): import openai import pandas as pd import os import json # Your OpenAI API key openai.api_key = os.getenv("OPENAI_API_KEY", "") # Unstructured text text = """The top 3 predicted materials in 2025 are: 1. 9762_NUT_CHNNL with a total quantity of 753 2. 8268_KIT_TOOL with a total quantity of 738 3. 5960_CABLE with a total quantity of 729""" # OpenAI prompt to structure the data prompt = f""" Extract the materials and their quantities from the following text and format them as a structured JSON: {text} """ # Call OpenAI API response = openai.chat.completions.create( model="gpt-4o-mini", messages=[ {"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": prompt} ], temperature=0 ) print(f"response: {response}") # Extract the response content response_content = response.choices[0].message.content.strip() # Debugging: Print raw response to check its format print("Raw Response:", response_content) # Step 1: Extract the JSON part from the markdown # Split the response content to isolate the JSON part json_part = response_content.split("```json\n")[1].split("\n```")[0] # Step 2: Parse the JSON content try: structured_data = json.loads(json_part) # Parse the JSON content print("Parsed JSON:", structured_data) except json.JSONDecodeError: print("Error: Response content is not valid JSON.") # Convert the structured data into a DataFrame df = pd.DataFrame(structured_data["materials"]) # Rename columns to desired format df.columns = ["material_name", "material_quantity"] # Print the DataFrame print(df) return df