app.py

import gradio as gr
import psutil
import subprocess
import time

#placeholders for api use
def generate_response_by_api(user_message):
    FinalOutput = ""
    #return FinalOutput
    pass

def custom_generate_response_by_api(cust_user_message, prompt_index, prompts_list):
    prompt, ending = prompts_list[prompt_index]  # Unpack the prompt and its ending from the provided list
    cust_user_message = f"{prompt}\n\n{cust_user_message}\n\n{ending}"
    #return generate_response(cust_user_message)
    pass

#-----------------------------------------------------------------------------------------------------------------------

#Local gguf model using llama.cpp
def generate_response(user_message): #generate_response_token_by_token
    cmd = [
        "/app/llama.cpp/main",  # Path to the executable
        "-m", "/app/llama.cpp/models/stablelm-2-zephyr-1_6b-Q4_0.gguf",
        "-p", user_message,
        "-n", "400",
        "-e"
    ]

    process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, bufsize=1)
    process_monitor = psutil.Process(process.pid)


    start_time = time.time()
    monitor_start_time = time.time()
    alltokens = ""
    token_buffer = ''
    tokencount = 0
    try:
        while True:
            # Read one character at a time
            char = process.stdout.read(1)
            if char == '' and process.poll() is not None:
                break
            if char != '':
                token_buffer += char
                if char == ' ' or char == '\n':  # Token delimiters
                    elapsed_time = time.time() - start_time  # Calculate elapsed time
                    alltokens += token_buffer
                    tokencount += 1
                    yield f"{alltokens} \n\n [Inference time: {elapsed_time:.2f} seconds | Tokens: { tokencount }]"
                    token_buffer = ''  # Reset token buffer
            # Log resource usage every minute
            if time.time() - monitor_start_time > 60:
                cpu_usage = process_monitor.cpu_percent()
                memory_usage = process_monitor.memory_info().rss  # in bytes 
                print(f"Subprocess CPU Usage: {cpu_usage}%, Memory Usage: {memory_usage / 1024 ** 2} MB")
                monitor_start_time = time.time()  # Reset the timer
    
        # Yield the last token if there is any
        if token_buffer:
            elapsed_time = time.time() - start_time  # Calculate elapsed time
            alltokens += token_buffer
            yield f"{alltokens} \n\n [Inference time: {elapsed_time:.2f} seconds | Average Tokens per second: { round(tokencount / elapsed_time, 2) }]"

    finally:
        try:
            # Wait for the process to complete, with a timeout
            process.wait(timeout=60)  # Timeout in seconds
        except subprocess.TimeoutExpired:
            print("Process didn't complete within the timeout. Killing it.")
            process.kill()
            process.wait()  # Ensure proper cleanup
        # Wait for the subprocess to finish if it hasn't already
        process.stdout.close()
        process.stderr.close()

        # Check for any errors
        if process.returncode != 0:
            error_message = process.stderr.read()
            print(f"Error: {error_message}")
            
def custom_generate_response(cust_user_message, prompt_index, prompts_list):
    """
    Generates a custom response based on the user message, the selected prompt,
    and the provided list of prompts, including a custom ending specific to the prompt.

    Parameters:
    - cust_user_message: The message input from the user.
    - prompt_index: The index of the custom prompt to use.
    - prompts_list: The list of prompts to use for generating the response.
    """
    prompt, ending = prompts_list[prompt_index]  # Unpack the prompt and its ending from the provided list
    cust_user_message = f"{prompt}\n\n{cust_user_message}\n\n{ending}"
    yield from generate_response(cust_user_message)

Allprompts = {
    "Custom Prompts" : [
        ("Write a Class Diagram based on the following text:", "Class Diagram:"),
        ("Write a Pydot code based on the following text:", "Pydot Code:"),
        ("Describe what a standard happy scene in any movie would be planned in great detail, based on the following text:", "Scene Details"),
        ("Explain a teardown of the product mentioned in the following text:", "Teardown Details:"),
        ("Explain the manufacturing of the product mentioned in the following text:", "Manufacturing Details:"),
        ("Explain the marketing considerations of the product mentioned in the following text:", "Considerations:"),
        ("Explain the target users considerations of the product mentioned in the following text:", "Target Users Considerations:"),
        ("My problem to solve is", "- please make 10 sub problems have to solve from this:"),
    ],
    "Business Prompts" : [
        ("Suggest Product ideas just based off the following text:", "Products:"),
        ("Write an outline for a business plan for: " , ""),
        ("Write an example of a detailed report for a Executive Summary for " , "Executive Summary:"),
        ("Write an example of a detailed report for a Company Description for " , "Company Description:"),
        ("Write an example of a detailed report for a Market Analysis for " , "Market Analysis:"),
        ("Write an example of a detailed report for a Marketing and Sales Strategy for " , "Marketing and Sales Strategy:"),
        ("Write an example of a detailed report for a Product Development for " , "Product Development:"),
        ("Write an example of a detailed report for a Operations and Management for " , "Operations and Management:"),
        ("Write an example of a detailed report for a Financial Projections for " , "Financial Projections:"),
        ("Explain how this to make this product unique from competitors:", "Considerations:"),
    ],
    "Programming Pattern Prompts" : [
        ("Demonstrate a builder pattern in go:", ""),
        ("Demonstrate a zero cost abstractions in go:", ""),
        ("Demonstrate a builder pattern in rust:", ""),
        ("Demonstrate a Polymorphism in rust:", ""),
        ("Explain how RAII pattern affects rust:", ""),
        ("Demonstrate a builder pattern in c++:", ""),
        ("Explain when to consider using a builder pattern in go:", ""),
    ],
    "Creativity Prompts" : [
        ("Make the following text more vague:", "Vague version:"),
        ("Turn the following text into a bunch of rules:", "Rules:"),
        ("What Syllogisms can be made from this text:", "Syllogisms:"),
        ("Reimagine the following text:", ""),
    ],
    "Game Based" : [
        ("What obstacles to growth exist in the following text:", "Obstacles:"),
        ("Write a story for the basis of a random game", ""),
        ("What are common themes in games?", ""),
        ("Write Three factions and why they are at conflict based on the following text:", "Faction 1:"),
    ]
}

with gr.Blocks() as iface:
    with gr.Tab("Single prompt"):
        gr.HTML("<a href='https://huggingface.co/spaces/stabilityai/stablelm-2-1_6b-zephyr'> -- Original StabilityAI demo -- </a> | ")
        gr.Interface(
            fn=generate_response,
            inputs=gr.Textbox(lines=2, placeholder="Type your message here..."),
            outputs="text",
            title="Stable LM 2 Zephyr (1.6b) LLama.cpp Interface Test (Inconsistent Performance - 100 tokens in 50 secs (when this HF space is updated) or 800+ secs(HF space open for long))",
            description="No Prompt template used yet (Essentially autocomplete). No Message History for now - Enter your message and get a response.",
            flagging_dir="/usr/src/app/flagged",
        )

        gr.HTML("Any standard way of thinking / Repetitive idea / rule of thumb / advice can be turned into a button (In a timeline?)")
        gr.HTML("LLM powered Buttons as the new notetaking? (Youtube Video to prompt pipeline?)<br><br>List to buttons (Instead of buttons tabs and dropdowns maybe?)")

        MainOutput = gr.TextArea(placeholder='Output will show here')
        CustomButtonInput = gr.TextArea(lines=1, placeholder='Prompt goes here')

        # with gr.Accordion("Random Ideas"):
        #     with gr.Group():        
        #         for index, (prompt, _) in enumerate(CustomPrompts):
        #             button = gr.Button(prompt)
        #             # Pass CustomPrompts list as an argument
        #             button.click(custom_generate_response, inputs=[CustomButtonInput, gr.State(index), gr.State(CustomPrompts)], outputs=MainOutput)

        # with gr.Accordion("General Product and Business based", open=False):
        #     with gr.Group():
        #         for index, (prompt, _) in enumerate(BusinessPrompts):
        #             button = gr.Button(prompt)
        #             # Pass BusinessPrompts list as an argument
        #             button.click(custom_generate_response, inputs=[CustomButtonInput, gr.State(index), gr.State(BusinessPrompts)], outputs=MainOutput)

        # with gr.Accordion("General Programming Pattern based", open=False):
        #     with gr.Group():
        #         for index, (prompt, _) in enumerate(ProgrammingPatternPrompts):
        #             button = gr.Button(prompt)
        #             # Pass BusinessPrompts list as an argument
        #             button.click(custom_generate_response, inputs=[CustomButtonInput, gr.State(index), gr.State(ProgrammingPatternPrompts)], outputs=MainOutput)

        # with gr.Accordion("General Creativity Pattern based", open=False):
        #     with gr.Group():
        #         for index, (prompt, _) in enumerate(CreativityPrompts):
        #             button = gr.Button(prompt)
        #             # Pass BusinessPrompts list as an argument
        #             button.click(custom_generate_response, inputs=[CustomButtonInput, gr.State(index), gr.State(CreativityPrompts)], outputs=MainOutput)

        for category_name, category_prompts in Allprompts.items():
            with gr.Accordion(f"General {category_name} Pattern based", open=False):
                with gr.Group():
                    for index, (prompt, _) in enumerate(category_prompts):
                        button = gr.Button(prompt)
                        button.click(custom_generate_response, inputs=[CustomButtonInput, gr.State(index), gr.State(category_name)], outputs=MainOutput)


    with gr.Tab("Workflow Brainstom"):
        gr.HTML("Workflow = premeditated events --- need a timeline before prompts")

iface.queue().launch(server_name="0.0.0.0", share=True)