init

api_keys.py

@@ -0,0 +1,2 @@
+meshy_api_key = ""
+gpt_api_key = "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"

app.py

@@ -0,0 +1,72 @@
+import time 
+import gradio as gr
+import openai
+from prompt import setup_prompt
+from api_keys import gpt_api_key
+import utils
+
+openai.api_key = gpt_api_key 
+messages = []
+
+# system_msg = input("What type of chatbot would you like to create?\n")
+messages.append({"role": "system", "content": setup_prompt})
+messages.append({"role": "user", "content": "hi, I want to build a 3d model"})
+
+response = openai.ChatCompletion.create(
+        model="gpt-4",
+        messages=messages, temperature = 1.2)
+reply = response["choices"][0]["message"]["content"]
+messages.append({"role": "assistant", "content": reply})
+print("\n" + reply + "\n")
+
+final_prompt = ""
+
+model_path =  "../house_light/model.glb"
+
+cnt = 0
+
+
+def solve():
+  global model_path
+  time.sleep(3)
+  return model_path
+
+def slow_echo(message, history):
+  global cnt
+  global messages
+  global final_prompt
+  global model_path
+  messages.append({"role": "user", "content": message})
+  response = openai.ChatCompletion.create(
+      model="gpt-4",
+      messages=messages, temperature = 1.2)
+  reply = response["choices"][0]["message"]["content"]
+  messages.append({"role": "assistant", "content": reply})
+  print("\n" + reply + "\n")
+  if reply[len(reply)-1] == '1' or reply[len(reply)-2] == '1':
+      print("start generating")
+      final_prompt = reply
+      reply = "Generating..."
+      cnt += 1
+      if cnt == 1 :
+        model_path = "../house_light/model.glb"
+      elif cnt == 2 :
+        model_path = "../house_dark/model.glb"
+
+  for i in range(len(reply)):
+    time.sleep(0.02)
+    yield reply[:i+1]
+
+with gr.Blocks() as demo:
+  with gr.Row():
+    with gr.Column():
+      chatbot = gr.ChatInterface(fn = slow_echo, title="SpacialSynergy", examples=["hi, I want to build a 3d model"]).queue()
+    with gr.Column():
+      interface = gr.Interface(
+        fn=solve, 
+        title = "3D Model",
+        inputs=None, 
+        outputs = ["model3d"],
+      )
+
+demo.launch(share = False)

chatbot.py

@@ -0,0 +1,45 @@
+import openai
+from prompt import setup_prompt
+from api_keys import gpt_api_key
+import utils
+
+openai.api_key = gpt_api_key 
+messages = []
+
+# system_msg = input("What type of chatbot would you like to create?\n")
+messages.append({"role": "system", "content": setup_prompt})
+messages.append({"role": "user", "content": "hi, I want to build a 3d model"})
+
+response = openai.ChatCompletion.create(
+        model="gpt-4",
+        messages=messages, temperature = 1.2)
+reply = response["choices"][0]["message"]["content"]
+messages.append({"role": "assistant", "content": reply})
+print("\n" + reply + "\n")
+
+final_prompt = ""
+
+# print("Your 3d builder  assistant is ready!")
+while input != "quit()":
+    message = input()
+    messages.append({"role": "user", "content": message})
+    response = openai.ChatCompletion.create(
+        model="gpt-4",
+        messages=messages, temperature = 1.2)
+    reply = response["choices"][0]["message"]["content"]
+
+    messages.append({"role": "assistant", "content": reply})
+    print("\n" + reply + "\n")
+    if reply[len(reply)-1] == '1' or reply[len(reply)-2] == '1':
+        print("start generating")
+        final_prompt = reply
+        break
+
+
+# extracted_text = utils.extract_text_surrounded_by_backticks(final_prompt)
+# begin = final_prompt.find("{")
+# end = final_prompt.find("}")
+# print(final_prompt[begin:end+1])
+# payload = utils.text_to_3d_gen(final_prompt[begin:end+1])
+# taskid = utils.create_meshy_object(payload)
+# utils.download_model(taskid)

prompt.py

@@ -0,0 +1,57 @@
+setup_prompt = """
+Suppose you are a 3d model design assistant, your job is to help user generate detailed prompt which is used to feed another 3d generative AI.
+**Job Description**
+- You should inspire users and making them more creative
+- You can only response to 3d-model building related problems, if users ask you non-related problems, you should not answer those questions
+- Users' responses should not have anything relates to pornography, racism and any form of discrimination
+
+First, here is the general prompt guideline for 3d model prompting.
+**Basic Prompts**
+    In Meshy, a basic prompt is to describe an object you want to generate or retexture, e.g. a sword, a helmet, a house, a treasure chest, etc.
+**Advanced Prompts**
+    If you want to add more details to the model, you'll need to provide the AI with more information through prompts. It is recommended that your prompts be specific and descriptive. Try describing the shape, color, size, style, and other attributes of the object you want to generate. Longer prompts don't necessarily equate to better results, focus on the key concepts!
+    Here we have some useful terms that may help improve the result you're gonna get:
+    - Related to detail:
+        highly detailed, high resolution, highest quality, best quality, 4K, 8K, HDR, studio quality
+    - Related to style:
+        beautiful, elegant, realistic, ultra realistic, trending on artstation, masterpiece, cinema 4d, unreal engine, octane render
+    - Related to lighting:
+        ambient lighting, soft lighting, sunlight, moonlight, fluorescent, glowing
+**Negative Prompts**:
+    A negative prompt is what you don't want to see in the generated result. If you're using the web app, you can simply type a negative prompt in the negative prompt box. For Discord users, you can use the --no parameter in your prompt.
+    Here we have some commonly used negative prompts for you:
+    bad anatomy/proportions, deformed, dull, duplicate, extra arms/fingers/legs, low quality, missing arms/fingers/legs, obscure, poor lighting, ugly, unnatural colors, worst quality
+
+Second, you should guide users step by step in the following procedure to help users generate good prompt according to the guideline above 
+1 - Object: Ask user what 3d object they wany to create
+2 - Style: Ask user things relate to detail, style, lighting according to the prompt guidline mentioned before.
+3 - Negative Prompt: Ask user what they do not want to see in 3d generation
+4 - Art Style: Ask user to choose an art style from the following options: {Realistic, Voxel, 2.5D Cartoon, Japanese Anime, Cartoon Line Art, Realistic Hand-drawn, 2.5D Hand-drawn, Oriental Comic Ink}
+5 - Texture Resolution: Ask user to choose texture resolution from the following options: {1K, 2K, 4K}
+6 - Confirmation: Show user the prompt you generated 
+- Style: <style> \n - Negative Prompt: <negative prompt> \n - Ary Style: <art style> \n - Texture Resolution: <resolution> \n <0>}. Note: everything in <> should be keywords, not a complete sentence or verbs. After showing, ask user if he/she wants to add more things
+7 - Output: Show the final prompt in JSON with the following keys:
+object_prompt, style_prompt, negative_prompt, art_style. Then, you should add number ```1``` to the end of your response, indicating the prompt generation is done.
+    - ```art_style``` key words to code conversion:
+        - Realistic style -> realistic
+        - 2.5D Cartoon style -> fake-3d-cartoon
+        - Japanese Anime style -> japanese-anime
+        - Cartoon Line Art style -> cartoon-line-art
+        - Realistic Hand-drawn style -> realistic-hand-drawn
+        - 2.5D Hand-drawn style -> fake-3d-hand-drawn
+        - Oriental Comic Ink style -> oriental-comic-ink
+    - ```negative_prompt``` key: ers do not say anything specific about their negative preferences, the value for the negative_prompt key should be an empty string ```""```
+    
+
+
+Finally, there are some good prompts that you can learn from:
+- Object: Wine barrel; Style: ancient, 4K, HDR, highest quality
+- Object: A treasure chest; Style: realistic, wooden, carved, highest quality
+- Object: Potion; Style: green glowing magical potion, highest quality
+- Object: Pistol; Style: golden pistol, unreal engine, game asset, highest quality
+- Object: Altar of Storms; Style: Metal, Viking pattern, black, old, scratches, iron, 8k, items, concept art trending artstation, high-res, realistic, Photographs, aaa game scene
+- Object: A monster mask; Style: Red fangs, Samurai outfit that fused with japanese batik style
+- Object: Medieval Small House; Style: ancient, best quality, 4k, trending on artstation
+- Object: A motorcycle from the era of steam engines in the 20th century; Style: steampunk, 4k, hdr
+- Object: Deity earring; Style: fancy, substantial, 4k, HDR, highest quality
+"""

utils.py

@@ -0,0 +1,122 @@
+import time
+import re
+import os
+import requests
+import json
+from api_keys import meshy_api_key
+from api_keys import gpt_api_key
+import openai
+
+def shouldStop(input_str, model="gpt-3.5-turbo"):
+    openai.api_key = gpt_api_key 
+    prompt = f"determine whether the text delimited by triple backticks contains number 1 at the end. {input_str} \n Format: 1/0"
+    messages = [{"role": "user", "content": prompt}]
+    response = openai.ChatCompletion.create(
+        model=model,
+        messages=messages,
+        temperature=0, # this is the degree of randomness of the model's output
+    )
+    return response.choices[0].message["content"]
+
+def extract_text_surrounded_by_backticks(input_string):
+    # Define a regular expression pattern to match text within triple backticks
+    pattern = r'```(.*?)```'
+    
+    # Use re.DOTALL to match across multiple lines, including newline characters
+    extracted_text = re.findall(pattern, input_string, re.DOTALL)
+    
+    # If extracted_text is not empty, return the first match; otherwise, return None
+    return extracted_text[0] if extracted_text else None
+
+def text_to_3d_gen(extracted_str):
+    payload = {"object_prompt":"", "style_prompt":"", "enable_pbr": True, "art_style": "", "negative_prompt":"low quality, low resolution, blurry, ugly, "}
+    response_dict = json.loads(extracted_str)
+    for i in payload.keys():
+        if i == "enable_pbr":
+            continue
+        if i != "negative_prompt":
+            payload[i] = response_dict[i]
+        else:
+            payload[i] += response_dict[i]
+    
+    return payload
+
+
+def create_meshy_object(payload, target="3d"):
+    headers = {
+        "Authorization": f"Bearer {meshy_api_key}"
+    }
+
+    response = requests.post(
+        f"https://api.meshy.ai/v1/text-to-{target}",
+        headers=headers,
+        json=payload,
+    )
+    response.raise_for_status()
+
+    meshy_response = response.json()
+    text_file = open("id.txt", "w")
+    text_file.write(meshy_response["result"])
+    text_file.close()
+    return meshy_response["result"]
+
+
+
+def download_model(task_id, target="3d"):
+    headers = {
+        "Authorization": f"Bearer {meshy_api_key}"
+    }
+
+        
+
+
+    # print(retrieve_response["progress"])
+    print_once = True;
+    # Extract the "model_url"
+    while True:
+        response = requests.get(
+        f"https://api.meshy.ai/v1/text-to-{target}/{task_id}",
+        headers=headers,
+        )
+        response.raise_for_status()
+
+        # Convert the JSON response to a Python dictionary
+        retrieve_response = json.loads(response.text)
+
+        if retrieve_response["progress"] == 100:
+            model_url = retrieve_response["model_url"]
+
+            # Get the directory of the script
+            script_dir = os.path.dirname(os.path.abspath(__file__))
+
+            # Define the file name
+            file_name = f"{task_id}.glb"
+
+            # Define the complete file path
+            file_path = os.path.join(script_dir, file_name)
+
+            # Check if the file already exists
+            if not os.path.isfile(file_path):
+                # Download the file
+                response = requests.get(model_url)
+
+                if response.status_code == 200:
+                    with open(file_path, "wb") as file:
+                        file.write(response.content)
+                    print(f"File downloaded successfully to {file_path}.")
+                else:
+                    print(f"File download failed with status code {response.status_code}.")
+                    if "error" in response.text:
+                        error_message = json.loads(response.text)["error"]
+                        print(f"Error message: {error_message}")
+            else:
+                print(f"File '{file_path}' already exists. Skipping download.")
+            break
+
+        else:
+            if print_once:
+                print("the model is still in progress...")
+                print_once = False
+            time.sleep(10)
+
+        #