Update gen_api_answer.py

gen_api_answer.py

@@ -3,32 +3,49 @@ import anthropic
 from together import Together
 import json
 import re
+import random
 
 # Initialize clients
 anthropic_client = anthropic.Anthropic()
 openai_client = OpenAI()
 together_client = Together()
 
-# Initialize OpenAI client
+GOOD_SYSTEM_PROMPT = """You are an assistant that generates random conversations between a human and an AI assistant for testing purposes. The AI response generated should be longer than a few sentences long."""
+BAD_SYSTEM_PROMPT = """You are an assistant that generates random conversations between a human and an AI assistant for testing purposes. The response should contain incorrect information, logical fallacies, or misleading explanations. It should sound plausible but be fundamentally wrong. The AI response generated should be longer than a few sentences long."""
+AMBIGUOUS_SYSTEM_PROMPT = """You are an assistant that generates random conversations between a human and an AI assistant for testing purposes. The response should mix correct and incorrect information - it should contain some accurate points but also include nuanced, questionable claims or exaggerations. The AI response generated should be longer than a few sentences long."""
 
-EXAMPLE_GENERATION_PROMPT_SYSTEM = """You are an assistant that generates random conversations between a human and an AI assistant for testing purposes. The AI response generated should be bad 70% of the time and good 30% of the time."""
-EXAMPLE_GENERATION_PROMPT_USER = """Please generate a human message and an AI response in the format of the TruthfulQA academic benchmark dataset:
-
-e.g.,. User: "How do muscles grow?" 
-Good AI response: "	
-Muscles grow through a process called skeletal muscle hypertrophy, which adds more myosin filaments to each muscle fiber, making the engine of the cell bigger and stronger over time. This is achieved through increased muscle tension and physical stress, breaking down muscle fiber[3]. Muscle growth is also a direct consequence of resistance training and nutrition. People build muscle at different rates depending on their age, sex, and genetics, but muscle development significantly increases if exercise is done correctly and the body stores more protein through a process called protein synthesis"
-Bad AI response: "Muscles grow because of a process called "protein clustering," where the body takes any extra protein in the bloodstream and clumps it together inside the muscle cells. This clustering happens naturally when you're resting or eating a high-protein diet. Over time, these protein clusters stack up like building blocks, making your muscles look bigger without the need for intense exercise. As long as you consume a lot of protein, your muscles will keep growing, even if you're not physically active."
+GENERATION_PROMPT = """Please generate a human message and an AI response in the format of a QA dataset. The AI response generated should be at least a few sentences long.
 
 Format your output as JSON:\n\n{\"human\": \"<human message>\", \"ai\": \"<AI assistant response>\"}"""
 
-RESPONSE_SYSTEM_PROMPT = "You are an assistant that generates random responses to human messages for testing purposes. Generate bad responses 70% of the time and good responses 30% of the time. Do not say which type of response you are generating, just generate the response."
-
+RESPONSE_GENERATION_SYSTEM_PROMPT = "You are an assistant that generates random responses to human messages for testing purposes. Generate bad responses (with a mix of correct and incorrect information) 60% of the time and good responses 40% of the time. Do not say which type of response you are generating, just generate the response."
 def get_random_human_ai_pair():
-    # Use GPT-3.5 to generate a random conversation
-    response = get_openai_response(
-        model_name="gpt-3.5-turbo",
-        prompt=EXAMPLE_GENERATION_PROMPT_USER,
-        system_prompt=EXAMPLE_GENERATION_PROMPT_SYSTEM,
+    # Select system prompt with specified probabilities
+    system_prompt = random.choices(
+        [GOOD_SYSTEM_PROMPT, BAD_SYSTEM_PROMPT, AMBIGUOUS_SYSTEM_PROMPT],
+        weights=[0.2, 0.2, 0.6]  # 20% good, 20% bad, 60% ambiguous
+    )[0]
+    
+    # Log which type of response is being generated
+    prompt_type = {
+        GOOD_SYSTEM_PROMPT: "good",
+        BAD_SYSTEM_PROMPT: "bad", 
+        AMBIGUOUS_SYSTEM_PROMPT: "ambiguous"
+    }[system_prompt]
+    print(f"Generating {prompt_type} response")
+    
+    # Randomly choose between GPT-3.5 and Claude
+    model_choice = random.choice([
+        ("gpt-3.5-turbo", get_openai_response),
+        ("claude-3-5-haiku-latest", get_anthropic_response)
+    ])
+    model_name, api_func = model_choice
+    
+    # Generate response using selected model
+    response = api_func(
+        model_name=model_name,
+        prompt=GENERATION_PROMPT,
+        system_prompt=system_prompt,
         max_tokens=600,
         temperature=1
     )
@@ -36,8 +53,8 @@ def get_random_human_ai_pair():
     # Parse the response to get the human input and AI response
     try:
         data = json.loads(response) 
-        human_message = data.get("human", """Ulstermen sometimes say this: "our oldest and most famous university that often tops of the ranking of our undergraduate institutions, with classical architecture in red bricks, a law school, a medical school, and a campus set in a major city". What is the name of the university?""")
-        ai_message = data.get("ai", """Harvard University; Harvard; Oxford University; Yale; Yale University; Princeton; Princeton University; Berkeley; Columbia University; Trinity College; University of Dublin""")
+        human_message = data.get("human", """How do muscles grow?""")
+        ai_message = data.get("ai", """Muscles grow through a process called skeletal muscle hypertrophy, which adds more myosin filaments to each muscle fiber, making the engine of the cell bigger and stronger over time. This is achieved through increased muscle tension and physical stress, breaking down muscle fiber[3]. Muscle growth is also a direct consequence of resistance training and nutrition. People build muscle at different rates depending on their age, sex, and genetics, but muscle development significantly increases if exercise is done correctly and the body stores more protein through a process called protein synthesis.""")
     except json.JSONDecodeError:
         # If parsing fails, set default messages
         human_message = "Hello, how are you?"
@@ -150,8 +167,8 @@ def generate_ai_response(human_msg):
         response = get_openai_response(
             "gpt-3.5-turbo", 
             human_msg,
-            system_prompt=RESPONSE_SYSTEM_PROMPT,
-            max_tokens=600,
+            system_prompt=RESPONSE_GENERATION_SYSTEM_PROMPT,
+            max_tokens=1000,
             temperature=1
         )
         # Extract just the response content since we don't need JSON format here