diff --git a/.gitignore b/.gitignore index 33270b3720a72b388eef1d5a3e8789f060c86036..afeece048b94c708b85834e432a16b4a32ae2377 100644 --- a/.gitignore +++ b/.gitignore @@ -175,4 +175,6 @@ detected_objects/ # [Gradio] demo_solver_cache/ +solver_cache/ backups/ +data/ diff --git a/app.py b/app.py index 6933e25afc1d9e0992fb90a9fbbb22f94b11d183..a04044de4e7161c689493101efe076142e3518b0 100644 --- a/app.py +++ b/app.py @@ -22,41 +22,125 @@ from octotools.models.memory import Memory from octotools.models.executor import Executor from octotools.models.utils import make_json_serializable -from utils import save_feedback - -########### Test Huggingface Dataset ########### from pathlib import Path from huggingface_hub import CommitScheduler -# Add these near the top of the file with other constants -DATASET_DIR = Path("feedback_dataset") -DATASET_DIR.mkdir(parents=True, exist_ok=True) -DATASET_PATH = DATASET_DIR / f"feedback-{time.strftime('%Y%m%d_%H%M%S')}.json" - # Get Huggingface token from environment variable HF_TOKEN = os.getenv("HUGGINGFACE_TOKEN") +########### Test Huggingface Dataset ########### +# Update the HuggingFace dataset constants +DATASET_DIR = Path("solver_cache") # the directory to save the dataset +DATASET_DIR.mkdir(parents=True, exist_ok=True) + +global QUERY_ID +QUERY_ID = None + scheduler = CommitScheduler( repo_id="lupantech/OctoTools-Gradio-Demo-User-Data", repo_type="dataset", folder_path=DATASET_DIR, - path_in_repo="data", + path_in_repo="solver_cache", # Update path in repo token=HF_TOKEN ) -def save_feedback(root_cache_dir: str, feedback_type: str, comment: str = None) -> None: - """Save user feedback to Huggingface dataset""" - with scheduler.lock: - with DATASET_PATH.open("a") as f: - feedback_data = { - "query_id": os.path.basename(root_cache_dir), - "feedback_type": feedback_type, - "comment": comment, - "datetime": time.strftime("%Y%m%d_%H%M%S") - } - json.dump(feedback_data, f) - f.write("\n") + +def save_query_data(query_id: str, query: str, image_path: str) -> None: + """Save query data to Huggingface dataset""" + # Save query metadata + query_cache_dir = DATASET_DIR / query_id + query_cache_dir.mkdir(parents=True, exist_ok=True) + query_file = query_cache_dir / "query_metadata.json" + + query_metadata = { + "query_id": query_id, + "query_text": query, + "datetime": time.strftime("%Y%m%d_%H%M%S"), + "image_path": image_path if image_path else None + } + + print(f"Saving query metadata to {query_file}") + with query_file.open("w") as f: + json.dump(query_metadata, f, indent=4) + + # # NOTE: As we are using the same name for the query cache directory as the dataset directory, + # # NOTE: we don't need to copy the content from the query cache directory to the query directory. + # # Copy all content from root_cache_dir to query_dir + # import shutil + # shutil.copytree(args.root_cache_dir, query_data_dir, dirs_exist_ok=True) + + +def save_feedback(query_id: str, feedback_type: str, feedback_text: str = None) -> None: + """ + Save user feedback to the query directory. + + Args: + query_id: Unique identifier for the query + feedback_type: Type of feedback ('upvote', 'downvote', or 'comment') + feedback_text: Optional text feedback from user + """ + + feedback_data_dir = DATASET_DIR / query_id + feedback_data_dir.mkdir(parents=True, exist_ok=True) + + feedback_data = { + "query_id": query_id, + "feedback_type": feedback_type, + "feedback_text": feedback_text, + "datetime": time.strftime("%Y%m%d_%H%M%S") + } + + # Save feedback in the query directory + feedback_file = feedback_data_dir / "feedback.json" + print(f"Saving feedback to {feedback_file}") + + # If feedback file exists, update it + if feedback_file.exists(): + with feedback_file.open("r") as f: + existing_feedback = json.load(f) + # Convert to list if it's a single feedback entry + if not isinstance(existing_feedback, list): + existing_feedback = [existing_feedback] + existing_feedback.append(feedback_data) + feedback_data = existing_feedback + + # Write feedback data + with feedback_file.open("w") as f: + json.dump(feedback_data, f, indent=4) + + +def save_steps_data(query_id: str, memory: Memory) -> None: + """Save steps data to Huggingface dataset""" + steps_file = DATASET_DIR / query_id / "all_steps.json" + + memory_actions = memory.get_actions() + memory_actions = make_json_serializable(memory_actions) # NOTE: make the memory actions serializable + print("Memory actions: ", memory_actions) + + with steps_file.open("w") as f: + json.dump(memory_actions, f, indent=4) + + +def save_module_data(query_id: str, key: str, value: Any) -> None: + """Save module data to Huggingface dataset""" + try: + key = key.replace(" ", "_").lower() + module_file = DATASET_DIR / query_id / f"{key}.json" + value = make_json_serializable(value) # NOTE: make the value serializable + with module_file.open("a") as f: + json.dump(value, f, indent=4) + except Exception as e: + print(f"Warning: Failed to save as JSON: {e}") + # Fallback to saving as text file + text_file = DATASET_DIR / query_id / f"{key}.txt" + try: + with text_file.open("a") as f: + f.write(str(value) + "\n") + print(f"Successfully saved as text file: {text_file}") + except Exception as e: + print(f"Error: Failed to save as text file: {e}") + ########### End of Test Huggingface Dataset ########### class Solver: @@ -72,7 +156,7 @@ class Solver: verbose: bool = True, max_steps: int = 10, max_time: int = 60, - root_cache_dir: str = "cache" + query_cache_dir: str = "solver_cache" ): self.planner = planner self.memory = memory @@ -83,7 +167,7 @@ class Solver: self.verbose = verbose self.max_steps = max_steps self.max_time = max_time - self.root_cache_dir = root_cache_dir + self.query_cache_dir = query_cache_dir self.output_types = output_types.lower().split(',') assert all(output_type in ["base", "final", "direct"] for output_type in self.output_types), "Invalid output type. Supported types are 'base', 'final', 'direct'." @@ -109,14 +193,14 @@ class Solver: # os.makedirs(os.path.join(self.root_cache_dir, 'images'), exist_ok=True) # img_path = os.path.join(self.root_cache_dir, 'images', str(uuid.uuid4()) + '.jpg') - img_path = os.path.join(self.root_cache_dir, 'query_image.jpg') + img_path = os.path.join(self.query_cache_dir, 'query_image.jpg') user_image.save(img_path) else: img_path = None # Set tool cache directory - _cache_dir = os.path.join(self.root_cache_dir, "tool_cache") # NOTE: This is the directory for tool cache - self.executor.set_query_cache_dir(_cache_dir) + _tool_cache_dir = os.path.join(self.query_cache_dir, "tool_cache") # NOTE: This is the directory for tool cache + self.executor.set_query_cache_dir(_tool_cache_dir) # NOTE: set query cache directory # Step 1: Display the received inputs if user_image: @@ -145,6 +229,13 @@ class Solver: metadata={"title": "🔍 Query Analysis"})) yield messages + # Save the query analysis data + query_analysis_data = { + "query_analysis": query_analysis, + "time": round(time.time() - start_time, 5) + } + save_module_data(QUERY_ID, "step_0_query_analysis", query_analysis_data) + # Step 5: Execution loop (similar to your step-by-step solver) while step_count < self.max_steps and (time.time() - start_time) < self.max_time: step_count += 1 @@ -158,6 +249,14 @@ class Solver: user_query, img_path, query_analysis, self.memory, step_count, self.max_steps ) context, sub_goal, tool_name = self.planner.extract_context_subgoal_and_tool(next_step) + step_data = { + "step_count": step_count, + "context": context, + "sub_goal": sub_goal, + "tool_name": tool_name, + "time": round(time.time() - start_time, 5) + } + save_module_data(QUERY_ID, f"step_{step_count}_action_prediction", step_data) # Display the step information messages.append(ChatMessage( @@ -183,6 +282,21 @@ class Solver: result = self.executor.execute_tool_command(tool_name, command) result = make_json_serializable(result) + # Save the command generation data + command_generation_data = { + "explanation": explanation, + "command": command, + "time": round(time.time() - start_time, 5) + } + save_module_data(QUERY_ID, f"step_{step_count}_command_generation", command_generation_data) + + # Save the command execution data + command_execution_data = { + "result": result, + "time": round(time.time() - start_time, 5) + } + save_module_data(QUERY_ID, f"step_{step_count}_command_execution", command_execution_data) + messages.append(ChatMessage( role="assistant", content=f"{json.dumps(result, indent=4)}", @@ -194,6 +308,14 @@ class Solver: stop_verification = self.planner.verificate_memory(user_query, img_path, query_analysis, self.memory) conclusion = self.planner.extract_conclusion(stop_verification) + # Save the context verification data + context_verification_data = { + "stop_verification": stop_verification, + "conclusion": conclusion, + "time": round(time.time() - start_time, 5) + } + save_module_data(QUERY_ID, f"step_{step_count}_context_verification", context_verification_data) + messages.append(ChatMessage( role="assistant", content=f"🛑 Step {step_count} Conclusion: {conclusion}")) @@ -208,15 +330,29 @@ class Solver: messages.append(ChatMessage(role="assistant", content=f"🎯 Final Output:\n{final_output}")) yield messages + # Save the final output data + final_output_data = { + "final_output": final_output, + "time": round(time.time() - start_time, 5) + } + save_module_data(QUERY_ID, "final_output", final_output_data) + if 'direct' in self.output_types: direct_output = self.planner.generate_direct_output(user_query, img_path, self.memory) messages.append(ChatMessage(role="assistant", content=f"🔹 Direct Output:\n{direct_output}")) yield messages + # Save the direct output data + direct_output_data = { + "direct_output": direct_output, + "time": round(time.time() - start_time, 5) + } + save_module_data(QUERY_ID, "direct_output", direct_output_data) + # Step 8: Completion Message messages.append(ChatMessage(role="assistant", content="✅ Problem-solving process completed.")) yield messages - + def parse_arguments(): parser = argparse.ArgumentParser(description="Run the OctoTools demo with specified parameters.") @@ -230,7 +366,8 @@ def parse_arguments(): help="Comma-separated list of required outputs (base,final,direct)" ) parser.add_argument("--enabled_tools", default="Generalist_Solution_Generator_Tool", help="List of enabled tools.") - parser.add_argument("--root_cache_dir", default="demo_solver_cache", help="Path to solver cache directory.") + parser.add_argument("--root_cache_dir", default="solver_cache", help="Path to solver cache directory.") + parser.add_argument("--query_id", default=None, help="Query ID.") parser.add_argument("--verbose", type=bool, default=True, help="Enable verbose output.") # NOTE: Add new arguments @@ -245,18 +382,28 @@ def solve_problem_gradio(user_query, user_image, max_steps=10, max_time=60, api_ Streams responses from `solver.stream_solve_user_problem` for real-time UI updates. """ - # Generate shorter ID (Date and first 8 characters of UUID) + # Generate Unique Query ID (Date and first 8 characters of UUID) query_id = time.strftime("%Y%m%d_%H%M%S") + "_" + str(uuid.uuid4())[:8] # e.g, 20250217_062225_612f2474 print(f"Query ID: {query_id}") + # NOTE: update the global variable to save the query ID + global QUERY_ID + QUERY_ID = query_id + # Create a directory for the query ID - query_dir = os.path.join(args.root_cache_dir, query_id) - os.makedirs(query_dir, exist_ok=True) - args.root_cache_dir = query_dir + query_cache_dir = os.path.join(DATASET_DIR.name, query_id) # NOTE + os.makedirs(query_cache_dir, exist_ok=True) if api_key is None: return [["assistant", "⚠️ Error: OpenAI API Key is required."]] + # Save the query data + save_query_data( + query_id=query_id, + query=user_query, + image_path=os.path.join(query_cache_dir, 'query_image.jpg') if user_image else None + ) + # # Initialize Tools # enabled_tools = args.enabled_tools.split(",") if args.enabled_tools else [] @@ -284,7 +431,7 @@ def solve_problem_gradio(user_query, user_image, max_steps=10, max_time=60, api_ # Instantiate Executor executor = Executor( llm_engine_name=llm_model_engine, - root_cache_dir=args.root_cache_dir, # NOTE + query_cache_dir=query_cache_dir, # NOTE enable_signal=False, api_key=api_key ) @@ -300,16 +447,23 @@ def solve_problem_gradio(user_query, user_image, max_steps=10, max_time=60, api_ verbose=args.verbose, max_steps=max_steps, max_time=max_time, - root_cache_dir=args.root_cache_dir # NOTE + query_cache_dir=query_cache_dir # NOTE ) if solver is None: return [["assistant", "⚠️ Error: Solver is not initialized. Please restart the application."]] + messages = [] # Initialize message list for message_batch in solver.stream_solve_user_problem(user_query, user_image, api_key, messages): yield [msg for msg in message_batch] # Ensure correct format for Gradio Chatbot + # Save steps + save_steps_data( + query_id=query_id, + memory=memory + ) + def main(args): #################### Gradio Interface #################### @@ -325,8 +479,8 @@ def main(args): [Website](https://octotools.github.io/) | [Github](https://github.com/octotools/octotools) | - [arXiv](https://arxiv.org/abs/2502.xxxxx) | - [Paper](https://arxiv.org/pdf/2502.xxxxx) | + [arXiv](https://arxiv.org/abs/2502.11271) | + [Paper](https://arxiv.org/pdf/2502.11271) | [Tool Cards](https://octotools.github.io/#tool-cards) | [Example Visualizations](https://octotools.github.io/#visualization) | [Discord](https://discord.gg/NMJx66DC) @@ -424,20 +578,20 @@ def main(args): # Update the button click handlers upvote_btn.click( - fn=lambda: save_feedback(args.root_cache_dir, "upvote"), + fn=lambda: save_feedback(QUERY_ID, "upvote"), inputs=[], outputs=[] ) downvote_btn.click( - fn=lambda: save_feedback(args.root_cache_dir, "downvote"), + fn=lambda: save_feedback(QUERY_ID, "downvote"), inputs=[], outputs=[] ) # Add handler for comment submission comment_textbox.submit( - fn=lambda comment: save_feedback(args.root_cache_dir, comment), + fn=lambda comment: save_feedback(QUERY_ID, "comment", comment), inputs=[comment_textbox], outputs=[] ) @@ -481,9 +635,6 @@ def main(args): if __name__ == "__main__": args = parse_arguments() - # Manually set enabled tools - # args.enabled_tools = "Generalist_Solution_Generator_Tool" - # All tools all_tools = [ "Generalist_Solution_Generator_Tool", @@ -504,5 +655,7 @@ if __name__ == "__main__": ] args.enabled_tools = ",".join(all_tools) + # NOTE: Use the same name for the query cache directory as the dataset directory + args.root_cache_dir = DATASET_DIR.name main(args) diff --git a/demo_solver_cache/20250217_062225_8ce3e482/query_image.jpg b/demo_solver_cache/20250217_062225_8ce3e482/query_image.jpg deleted file mode 100644 index 72fee9e5fa9564a483931225200846890da594c7..0000000000000000000000000000000000000000 Binary files a/demo_solver_cache/20250217_062225_8ce3e482/query_image.jpg and /dev/null differ diff --git a/demo_solver_cache/20250217_062225_8ce3e482/query_image_baseball_1.png b/demo_solver_cache/20250217_062225_8ce3e482/query_image_baseball_1.png deleted file mode 100644 index 7389d135a0881650d6052df9eaeae276a8a903b6..0000000000000000000000000000000000000000 Binary files a/demo_solver_cache/20250217_062225_8ce3e482/query_image_baseball_1.png and /dev/null differ diff --git a/demo_solver_cache/20250217_062225_8ce3e482/query_image_baseball_10.png b/demo_solver_cache/20250217_062225_8ce3e482/query_image_baseball_10.png deleted file mode 100644 index 2967b1a04e55d12786ac69a8ac3db9b6e5b75c11..0000000000000000000000000000000000000000 Binary files a/demo_solver_cache/20250217_062225_8ce3e482/query_image_baseball_10.png and /dev/null differ diff --git a/demo_solver_cache/20250217_062225_8ce3e482/query_image_baseball_11.png b/demo_solver_cache/20250217_062225_8ce3e482/query_image_baseball_11.png deleted file mode 100644 index a08cb6f76900da0e4c44b5aa3266e312fb1eaff7..0000000000000000000000000000000000000000 Binary files a/demo_solver_cache/20250217_062225_8ce3e482/query_image_baseball_11.png and /dev/null differ diff --git a/demo_solver_cache/20250217_062225_8ce3e482/query_image_baseball_12.png b/demo_solver_cache/20250217_062225_8ce3e482/query_image_baseball_12.png deleted file mode 100644 index 043d653898928793a7edb0c0def7a822660852b8..0000000000000000000000000000000000000000 Binary files a/demo_solver_cache/20250217_062225_8ce3e482/query_image_baseball_12.png and /dev/null differ diff --git a/demo_solver_cache/20250217_062225_8ce3e482/query_image_baseball_13.png b/demo_solver_cache/20250217_062225_8ce3e482/query_image_baseball_13.png deleted file mode 100644 index 2967b1a04e55d12786ac69a8ac3db9b6e5b75c11..0000000000000000000000000000000000000000 Binary files a/demo_solver_cache/20250217_062225_8ce3e482/query_image_baseball_13.png and /dev/null differ diff --git a/demo_solver_cache/20250217_062225_8ce3e482/query_image_baseball_14.png b/demo_solver_cache/20250217_062225_8ce3e482/query_image_baseball_14.png deleted file mode 100644 index 89e3c9c9859c656a14169cb774eb5d19f84cd84f..0000000000000000000000000000000000000000 Binary files a/demo_solver_cache/20250217_062225_8ce3e482/query_image_baseball_14.png and /dev/null differ diff --git a/demo_solver_cache/20250217_062225_8ce3e482/query_image_baseball_15.png b/demo_solver_cache/20250217_062225_8ce3e482/query_image_baseball_15.png deleted file mode 100644 index d89fc08775183a623530a0c4e490c0c52784bbf6..0000000000000000000000000000000000000000 Binary files a/demo_solver_cache/20250217_062225_8ce3e482/query_image_baseball_15.png and /dev/null differ diff --git a/demo_solver_cache/20250217_062225_8ce3e482/query_image_baseball_16.png b/demo_solver_cache/20250217_062225_8ce3e482/query_image_baseball_16.png deleted file mode 100644 index ec267c5e2f7a328b7c6800af35889149e31cfcbe..0000000000000000000000000000000000000000 Binary files a/demo_solver_cache/20250217_062225_8ce3e482/query_image_baseball_16.png and /dev/null differ diff --git a/demo_solver_cache/20250217_062225_8ce3e482/query_image_baseball_17.png b/demo_solver_cache/20250217_062225_8ce3e482/query_image_baseball_17.png deleted file mode 100644 index 1841dffdf9cb345f1ef636c230d0b486b2413b95..0000000000000000000000000000000000000000 Binary files a/demo_solver_cache/20250217_062225_8ce3e482/query_image_baseball_17.png and /dev/null differ diff --git a/demo_solver_cache/20250217_062225_8ce3e482/query_image_baseball_18.png b/demo_solver_cache/20250217_062225_8ce3e482/query_image_baseball_18.png deleted file mode 100644 index 437caaab2446ba5c5e57d5e1aee6217e1e989949..0000000000000000000000000000000000000000 Binary files a/demo_solver_cache/20250217_062225_8ce3e482/query_image_baseball_18.png and /dev/null differ diff --git a/demo_solver_cache/20250217_062225_8ce3e482/query_image_baseball_19.png b/demo_solver_cache/20250217_062225_8ce3e482/query_image_baseball_19.png deleted file mode 100644 index 8aebdc92a38294a8a24c8c8468bef977858b77d7..0000000000000000000000000000000000000000 Binary files a/demo_solver_cache/20250217_062225_8ce3e482/query_image_baseball_19.png and /dev/null differ diff --git a/demo_solver_cache/20250217_062225_8ce3e482/query_image_baseball_2.png b/demo_solver_cache/20250217_062225_8ce3e482/query_image_baseball_2.png deleted file mode 100644 index 9e8090ab7c3975e38774deaba1afc9bf43e1e625..0000000000000000000000000000000000000000 Binary files a/demo_solver_cache/20250217_062225_8ce3e482/query_image_baseball_2.png and /dev/null differ diff --git a/demo_solver_cache/20250217_062225_8ce3e482/query_image_baseball_20.png b/demo_solver_cache/20250217_062225_8ce3e482/query_image_baseball_20.png deleted file mode 100644 index 4d633f1a46a542849ce0d5b98286b9e0c741b74c..0000000000000000000000000000000000000000 Binary files a/demo_solver_cache/20250217_062225_8ce3e482/query_image_baseball_20.png and /dev/null differ diff --git a/demo_solver_cache/20250217_062225_8ce3e482/query_image_baseball_3.png b/demo_solver_cache/20250217_062225_8ce3e482/query_image_baseball_3.png deleted file mode 100644 index a61771375cf03591229969d0943f8a82071ff9c6..0000000000000000000000000000000000000000 Binary files a/demo_solver_cache/20250217_062225_8ce3e482/query_image_baseball_3.png and /dev/null differ diff --git a/demo_solver_cache/20250217_062225_8ce3e482/query_image_baseball_4.png b/demo_solver_cache/20250217_062225_8ce3e482/query_image_baseball_4.png deleted file mode 100644 index 043d653898928793a7edb0c0def7a822660852b8..0000000000000000000000000000000000000000 Binary files a/demo_solver_cache/20250217_062225_8ce3e482/query_image_baseball_4.png and /dev/null differ diff --git a/demo_solver_cache/20250217_062225_8ce3e482/query_image_baseball_5.png b/demo_solver_cache/20250217_062225_8ce3e482/query_image_baseball_5.png deleted file mode 100644 index 26161ccaaafccf973d606e3c7ad609aac40af4e9..0000000000000000000000000000000000000000 Binary files a/demo_solver_cache/20250217_062225_8ce3e482/query_image_baseball_5.png and /dev/null differ diff --git a/demo_solver_cache/20250217_062225_8ce3e482/query_image_baseball_6.png b/demo_solver_cache/20250217_062225_8ce3e482/query_image_baseball_6.png deleted file mode 100644 index 0bdefc08c022b33e7696429e232ec56e4fe77b85..0000000000000000000000000000000000000000 Binary files a/demo_solver_cache/20250217_062225_8ce3e482/query_image_baseball_6.png and /dev/null differ diff --git a/demo_solver_cache/20250217_062225_8ce3e482/query_image_baseball_7.png b/demo_solver_cache/20250217_062225_8ce3e482/query_image_baseball_7.png deleted file mode 100644 index 298446500299898bd1861d0b3336b9220c6e8b15..0000000000000000000000000000000000000000 Binary files a/demo_solver_cache/20250217_062225_8ce3e482/query_image_baseball_7.png and /dev/null differ diff --git a/demo_solver_cache/20250217_062225_8ce3e482/query_image_baseball_8.png b/demo_solver_cache/20250217_062225_8ce3e482/query_image_baseball_8.png deleted file mode 100644 index 978016084f9a4af665128e1ce7efd233718362ea..0000000000000000000000000000000000000000 Binary files a/demo_solver_cache/20250217_062225_8ce3e482/query_image_baseball_8.png and /dev/null differ diff --git a/demo_solver_cache/20250217_062225_8ce3e482/query_image_baseball_9.png b/demo_solver_cache/20250217_062225_8ce3e482/query_image_baseball_9.png deleted file mode 100644 index 97071b8e7101a5706a8f7d5c0a92e94c1bbaaea6..0000000000000000000000000000000000000000 Binary files a/demo_solver_cache/20250217_062225_8ce3e482/query_image_baseball_9.png and /dev/null differ diff --git a/demo_solver_cache/20250217_062225_8ce3e482/user_feedback.json b/demo_solver_cache/20250217_062225_8ce3e482/user_feedback.json deleted file mode 100644 index e971fb3e00b02186452595d7066eb19ed2292271..0000000000000000000000000000000000000000 --- a/demo_solver_cache/20250217_062225_8ce3e482/user_feedback.json +++ /dev/null @@ -1,22 +0,0 @@ -[ - { - "timestamp": "20250217_062307", - "feedback_type": "upvote", - "comment": null - }, - { - "timestamp": "20250217_062315", - "feedback_type": "downvote", - "comment": null - }, - { - "timestamp": "20250217_062322", - "feedback_type": "upvote", - "comment": null - }, - { - "timestamp": "20250217_062333", - "feedback_type": "It is helpful!", - "comment": null - } -] \ No newline at end of file diff --git a/demo_solver_cache/20250217_063316_09285db1/query_image.jpg b/demo_solver_cache/20250217_063316_09285db1/query_image.jpg deleted file mode 100644 index 72fee9e5fa9564a483931225200846890da594c7..0000000000000000000000000000000000000000 Binary files a/demo_solver_cache/20250217_063316_09285db1/query_image.jpg and /dev/null differ diff --git a/demo_solver_cache/20250217_063316_09285db1/tools/query_image_baseball_1.png b/demo_solver_cache/20250217_063316_09285db1/tools/query_image_baseball_1.png deleted file mode 100644 index 7389d135a0881650d6052df9eaeae276a8a903b6..0000000000000000000000000000000000000000 Binary files a/demo_solver_cache/20250217_063316_09285db1/tools/query_image_baseball_1.png and /dev/null differ diff --git a/demo_solver_cache/20250217_063316_09285db1/tools/query_image_baseball_10.png b/demo_solver_cache/20250217_063316_09285db1/tools/query_image_baseball_10.png deleted file mode 100644 index 2967b1a04e55d12786ac69a8ac3db9b6e5b75c11..0000000000000000000000000000000000000000 Binary files a/demo_solver_cache/20250217_063316_09285db1/tools/query_image_baseball_10.png and /dev/null differ diff --git a/demo_solver_cache/20250217_063316_09285db1/tools/query_image_baseball_11.png b/demo_solver_cache/20250217_063316_09285db1/tools/query_image_baseball_11.png deleted file mode 100644 index a08cb6f76900da0e4c44b5aa3266e312fb1eaff7..0000000000000000000000000000000000000000 Binary files a/demo_solver_cache/20250217_063316_09285db1/tools/query_image_baseball_11.png and /dev/null differ diff --git a/demo_solver_cache/20250217_063316_09285db1/tools/query_image_baseball_12.png b/demo_solver_cache/20250217_063316_09285db1/tools/query_image_baseball_12.png deleted file mode 100644 index 043d653898928793a7edb0c0def7a822660852b8..0000000000000000000000000000000000000000 Binary files a/demo_solver_cache/20250217_063316_09285db1/tools/query_image_baseball_12.png and /dev/null differ diff --git a/demo_solver_cache/20250217_063316_09285db1/tools/query_image_baseball_13.png b/demo_solver_cache/20250217_063316_09285db1/tools/query_image_baseball_13.png deleted file mode 100644 index 2967b1a04e55d12786ac69a8ac3db9b6e5b75c11..0000000000000000000000000000000000000000 Binary files a/demo_solver_cache/20250217_063316_09285db1/tools/query_image_baseball_13.png and /dev/null differ diff --git a/demo_solver_cache/20250217_063316_09285db1/tools/query_image_baseball_14.png b/demo_solver_cache/20250217_063316_09285db1/tools/query_image_baseball_14.png deleted file mode 100644 index 89e3c9c9859c656a14169cb774eb5d19f84cd84f..0000000000000000000000000000000000000000 Binary files a/demo_solver_cache/20250217_063316_09285db1/tools/query_image_baseball_14.png and /dev/null differ diff --git a/demo_solver_cache/20250217_063316_09285db1/tools/query_image_baseball_15.png b/demo_solver_cache/20250217_063316_09285db1/tools/query_image_baseball_15.png deleted file mode 100644 index d89fc08775183a623530a0c4e490c0c52784bbf6..0000000000000000000000000000000000000000 Binary files a/demo_solver_cache/20250217_063316_09285db1/tools/query_image_baseball_15.png and /dev/null differ diff --git a/demo_solver_cache/20250217_063316_09285db1/tools/query_image_baseball_16.png b/demo_solver_cache/20250217_063316_09285db1/tools/query_image_baseball_16.png deleted file mode 100644 index ec267c5e2f7a328b7c6800af35889149e31cfcbe..0000000000000000000000000000000000000000 Binary files a/demo_solver_cache/20250217_063316_09285db1/tools/query_image_baseball_16.png and /dev/null differ diff --git a/demo_solver_cache/20250217_063316_09285db1/tools/query_image_baseball_17.png b/demo_solver_cache/20250217_063316_09285db1/tools/query_image_baseball_17.png deleted file mode 100644 index 1841dffdf9cb345f1ef636c230d0b486b2413b95..0000000000000000000000000000000000000000 Binary files a/demo_solver_cache/20250217_063316_09285db1/tools/query_image_baseball_17.png and /dev/null differ diff --git a/demo_solver_cache/20250217_063316_09285db1/tools/query_image_baseball_18.png b/demo_solver_cache/20250217_063316_09285db1/tools/query_image_baseball_18.png deleted file mode 100644 index 437caaab2446ba5c5e57d5e1aee6217e1e989949..0000000000000000000000000000000000000000 Binary files a/demo_solver_cache/20250217_063316_09285db1/tools/query_image_baseball_18.png and /dev/null differ diff --git a/demo_solver_cache/20250217_063316_09285db1/tools/query_image_baseball_19.png b/demo_solver_cache/20250217_063316_09285db1/tools/query_image_baseball_19.png deleted file mode 100644 index 8aebdc92a38294a8a24c8c8468bef977858b77d7..0000000000000000000000000000000000000000 Binary files a/demo_solver_cache/20250217_063316_09285db1/tools/query_image_baseball_19.png and /dev/null differ diff --git a/demo_solver_cache/20250217_063316_09285db1/tools/query_image_baseball_2.png b/demo_solver_cache/20250217_063316_09285db1/tools/query_image_baseball_2.png deleted file mode 100644 index 9e8090ab7c3975e38774deaba1afc9bf43e1e625..0000000000000000000000000000000000000000 Binary files a/demo_solver_cache/20250217_063316_09285db1/tools/query_image_baseball_2.png and /dev/null differ diff --git a/demo_solver_cache/20250217_063316_09285db1/tools/query_image_baseball_20.png b/demo_solver_cache/20250217_063316_09285db1/tools/query_image_baseball_20.png deleted file mode 100644 index 4d633f1a46a542849ce0d5b98286b9e0c741b74c..0000000000000000000000000000000000000000 Binary files a/demo_solver_cache/20250217_063316_09285db1/tools/query_image_baseball_20.png and /dev/null differ diff --git a/demo_solver_cache/20250217_063316_09285db1/tools/query_image_baseball_3.png b/demo_solver_cache/20250217_063316_09285db1/tools/query_image_baseball_3.png deleted file mode 100644 index a61771375cf03591229969d0943f8a82071ff9c6..0000000000000000000000000000000000000000 Binary files a/demo_solver_cache/20250217_063316_09285db1/tools/query_image_baseball_3.png and /dev/null differ diff --git a/demo_solver_cache/20250217_063316_09285db1/tools/query_image_baseball_4.png b/demo_solver_cache/20250217_063316_09285db1/tools/query_image_baseball_4.png deleted file mode 100644 index 043d653898928793a7edb0c0def7a822660852b8..0000000000000000000000000000000000000000 Binary files a/demo_solver_cache/20250217_063316_09285db1/tools/query_image_baseball_4.png and /dev/null differ diff --git a/demo_solver_cache/20250217_063316_09285db1/tools/query_image_baseball_5.png b/demo_solver_cache/20250217_063316_09285db1/tools/query_image_baseball_5.png deleted file mode 100644 index 26161ccaaafccf973d606e3c7ad609aac40af4e9..0000000000000000000000000000000000000000 Binary files a/demo_solver_cache/20250217_063316_09285db1/tools/query_image_baseball_5.png and /dev/null differ diff --git a/demo_solver_cache/20250217_063316_09285db1/tools/query_image_baseball_6.png b/demo_solver_cache/20250217_063316_09285db1/tools/query_image_baseball_6.png deleted file mode 100644 index 0bdefc08c022b33e7696429e232ec56e4fe77b85..0000000000000000000000000000000000000000 Binary files a/demo_solver_cache/20250217_063316_09285db1/tools/query_image_baseball_6.png and /dev/null differ diff --git a/demo_solver_cache/20250217_063316_09285db1/tools/query_image_baseball_7.png b/demo_solver_cache/20250217_063316_09285db1/tools/query_image_baseball_7.png deleted file mode 100644 index 298446500299898bd1861d0b3336b9220c6e8b15..0000000000000000000000000000000000000000 Binary files a/demo_solver_cache/20250217_063316_09285db1/tools/query_image_baseball_7.png and /dev/null differ diff --git a/demo_solver_cache/20250217_063316_09285db1/tools/query_image_baseball_8.png b/demo_solver_cache/20250217_063316_09285db1/tools/query_image_baseball_8.png deleted file mode 100644 index 978016084f9a4af665128e1ce7efd233718362ea..0000000000000000000000000000000000000000 Binary files a/demo_solver_cache/20250217_063316_09285db1/tools/query_image_baseball_8.png and /dev/null differ diff --git a/demo_solver_cache/20250217_063316_09285db1/tools/query_image_baseball_9.png b/demo_solver_cache/20250217_063316_09285db1/tools/query_image_baseball_9.png deleted file mode 100644 index 97071b8e7101a5706a8f7d5c0a92e94c1bbaaea6..0000000000000000000000000000000000000000 Binary files a/demo_solver_cache/20250217_063316_09285db1/tools/query_image_baseball_9.png and /dev/null differ diff --git a/demo_solver_cache/20250217_063316_09285db1/user_feedback.json b/demo_solver_cache/20250217_063316_09285db1/user_feedback.json deleted file mode 100644 index cac9ba755ce003d6e7239af80bdc6d3ed8437a19..0000000000000000000000000000000000000000 --- a/demo_solver_cache/20250217_063316_09285db1/user_feedback.json +++ /dev/null @@ -1,12 +0,0 @@ -[ - { - "timestamp": "20250217_063350", - "feedback_type": "upvote", - "comment": null - }, - { - "timestamp": "20250217_063359", - "feedback_type": "Thanks! It is interesting!", - "comment": null - } -] \ No newline at end of file diff --git a/demo_solver_cache/20250217_183323_b0e58b32/query_image.jpg b/demo_solver_cache/20250217_183323_b0e58b32/query_image.jpg deleted file mode 100644 index 72fee9e5fa9564a483931225200846890da594c7..0000000000000000000000000000000000000000 Binary files a/demo_solver_cache/20250217_183323_b0e58b32/query_image.jpg and /dev/null differ diff --git a/demo_solver_cache/20250217_183323_b0e58b32/user_feedback.json b/demo_solver_cache/20250217_183323_b0e58b32/user_feedback.json deleted file mode 100644 index b130cb134e980a559e94208be5e0fd8633775f9f..0000000000000000000000000000000000000000 --- a/demo_solver_cache/20250217_183323_b0e58b32/user_feedback.json +++ /dev/null @@ -1,32 +0,0 @@ -[ - { - "timestamp": "20250217_190313", - "feedback_type": "upvote", - "comment": null - }, - { - "timestamp": "20250217_190319", - "feedback_type": "downvote", - "comment": null - }, - { - "timestamp": "20250217_190321", - "feedback_type": "upvote", - "comment": null - }, - { - "timestamp": "20250217_190322", - "feedback_type": "downvote", - "comment": null - }, - { - "timestamp": "20250217_190338", - "feedback_type": "Thanks! It is interesting!", - "comment": null - }, - { - "timestamp": "20250217_190341", - "feedback_type": "Thanks! It is interesting!", - "comment": null - } -] \ No newline at end of file diff --git a/feedback_dataset/feedback-20250217_212246.json b/feedback_dataset/feedback-20250217_212246.json deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/feedback_dataset/feedback-20250217_212401.json b/feedback_dataset/feedback-20250217_212401.json deleted file mode 100644 index f5ac10d96ce8295ebff388d5c1b4ada24304c80b..0000000000000000000000000000000000000000 --- a/feedback_dataset/feedback-20250217_212401.json +++ /dev/null @@ -1,10 +0,0 @@ -{"query_id": "20250217_212439_f48ed6ff", "feedback_type": "upvote", "comment": null, "datetime": "20250217_212450"} -{"query_id": "20250217_212439_f48ed6ff", "feedback_type": "upvote", "comment": null, "datetime": "20250217_212452"} -{"query_id": "20250217_212439_f48ed6ff", "feedback_type": "It is good!", "comment": null, "datetime": "20250217_212459"} -{"query_id": "20250217_212439_f48ed6ff", "feedback_type": "upvote", "comment": null, "datetime": "20250217_212523"} -{"query_id": "20250217_212439_f48ed6ff", "feedback_type": "upvote", "comment": null, "datetime": "20250217_212524"} -{"query_id": "20250217_212439_f48ed6ff", "feedback_type": "upvote", "comment": null, "datetime": "20250217_212524"} -{"query_id": "20250217_212439_f48ed6ff", "feedback_type": "downvote", "comment": null, "datetime": "20250217_212524"} -{"query_id": "20250217_212439_f48ed6ff", "feedback_type": "It is good!", "comment": null, "datetime": "20250217_212526"} -{"query_id": "20250217_212439_f48ed6ff", "feedback_type": "upvote", "comment": null, "datetime": "20250217_212619"} -{"query_id": "20250217_212439_f48ed6ff", "feedback_type": "It is good!", "comment": null, "datetime": "20250217_212650"} diff --git a/octotools/models/executor.py b/octotools/models/executor.py index af2d3d1609df54355fc87de4a5ff271a166a9678..c7af9851611910512f63689636e6a27db9e48a30 100644 --- a/octotools/models/executor.py +++ b/octotools/models/executor.py @@ -18,9 +18,9 @@ def timeout_handler(signum, frame): raise TimeoutError("Function execution timed out") class Executor: - def __init__(self, llm_engine_name: str, root_cache_dir: str = "solver_cache", num_threads: int = 1, max_time: int = 120, max_output_length: int = 100000, enable_signal: bool = True, api_key: str = None): + def __init__(self, llm_engine_name: str, query_cache_dir: str = "solver_cache", num_threads: int = 1, max_time: int = 120, max_output_length: int = 100000, enable_signal: bool = True, api_key: str = None): self.llm_engine_name = llm_engine_name - self.root_cache_dir = root_cache_dir + self.query_cache_dir = query_cache_dir self.num_threads = num_threads self.max_time = max_time self.max_output_length = max_output_length @@ -32,7 +32,7 @@ class Executor: self.query_cache_dir = query_cache_dir else: timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") - self.query_cache_dir = os.path.join(self.root_cache_dir, timestamp) + self.query_cache_dir = os.path.join(self.query_cache_dir, timestamp) os.makedirs(self.query_cache_dir, exist_ok=True) def generate_tool_command(self, question: str, image: str, context: str, sub_goal: str, tool_name: str, tool_metadata: Dict[str, Any], bytes_mode:bool = False) -> ToolCommand: diff --git a/utils.py b/utils.py deleted file mode 100644 index e49a641f2b5ca672aa5532c7c3ed2c2b3782f3c0..0000000000000000000000000000000000000000 --- a/utils.py +++ /dev/null @@ -1,26 +0,0 @@ -import os -import json -import time - - -def save_feedback(cache_dir: str, feedback_type: str, comment: str = None): - print(f"==> Saving feedback to {cache_dir}") - print(f"==> Feedback type: {feedback_type}") - print(f"==> Comment: {comment}") - - feedback_file = os.path.join(cache_dir, "user_feedback.json") - if os.path.exists(feedback_file): - with open(feedback_file, 'r') as f: - feedback_data = json.load(f) - else: - feedback_data = [] - - feedback_data.append({ - "timestamp": time.strftime("%Y%m%d_%H%M%S"), - "feedback_type": feedback_type, - "comment": comment - }) - - # Save feedback - with open(feedback_file, 'w') as f: - json.dump(feedback_data, f, indent=4)