utils.py

from langchain_openai import OpenAI
from pypdf import PdfReader
import pandas as pd
import re
import replicate
from langchain.prompts import PromptTemplate


def get_pdf_text(pdf_doc):
    text = ''
    pdf_reader = PdfReader(pdf_doc)
    for page in pdf_reader.pages:
        text += page.extract_text()
    return text


def extracted_data(pages_data):
    template = """Extract all the following values : invoice no. Description, Quantinty, data, Unit price, Amount, Total
    email, phone number and address from this data: {pages}
    
    Expected output: remove any dollar symbols {{'Invoice no.': '1001329', 'Description': 'Office Chair', 'Quantity': '2', 'Date': '5/4/2018', 'Unit price': '100', 'Amount': '20', 'Total': '12020' and so on}}"""

    prompt_template = PromptTemplate(input_variables=["pages"], template=template)

    llm = OpenAI(temperature=.7)
    full_response = llm(prompt_template.format(pages=pages_data))

    # input_prompt = {
    #     "top_p": .9,
    #     "prompt": prompt_template.format(pages=pages_data),
    #     "temperature": .1,
    #     "max_new_tokens": 500
    # }
    # full_response = ''
    # for event in replicate.stream("meta/llama-2-70b-chat", input=input_prompt):
    #     full_response += event

    return full_response


def create_docs(user_pdf_list):
    """Iterate over files in that user uploaded PDF files, one by one"""
    print("Started creating docs")
    df = pd.DataFrame({"Invoice no.": pd.Series(dtype='str'),
                       "Description": pd.Series(dtype='str'),
                       "Quantinty": pd.Series(dtype='str'),
                       "Date": pd.Series(dtype='str'),
                       "Unit price": pd.Series(dtype='str'),
                       "Amount": pd.Series(dtype='int'),
                       "Total": pd.Series(dtype='str'),
                       "Email": pd.Series(dtype='str'),
                       "Phone number": pd.Series(dtype='str'),
                       "Address": pd.Series(dtype='str')})

    for filename in user_pdf_list:
        print('filename is {}'.format(filename))
        raw_data = get_pdf_text(filename)
        print('raw_data is {}'.format(raw_data))

        llm_extracted_data = extracted_data(raw_data)
        print('llm_extracted_data is {}'.format(llm_extracted_data))

        # Capturing just the required data and excluding andy unwanted test from the LLM response
        pattern = r'{(.+)}'
        match = re.search(pattern, llm_extracted_data, re.DOTALL)

        if match:
            extracted_text = match.group(1)
            # Converting the extracted text to a dictionary
            data_dict = eval('{' + extracted_text + '}')
            print('data_dict is {}'.format(data_dict))
        else:
            print("No match found.")
        new_row = pd.DataFrame([data_dict])  # Creating a DataFrame from the dictionary
        df = pd.concat([df, new_row], ignore_index=True)  # Concatenating the new row
        print('*' * 10 + 'DONE' + '*' * 10)

    df.head()
    return df