Spaces:

mano-wii
/

tools

Running

App Files Files Community

Germano Cavalcante commited on Jan 31

Commit

35b7e57

1 Parent(s): 8c64fd6

Tools: find_related, wiki_search, gpu_checker, bpy_doc

Browse files

Files changed (22) hide show

.gitignore +3 -0
Dockerfile +20 -0
README.md +3 -4
__init__.py +0 -0
config.py +14 -0
main.py +58 -0
requirements-fastapi.txt +7 -0
routers/__init__.py +1 -0
routers/rag/__init__.py +226 -0
routers/rag/embeddings_dev_docs.pkl +3 -0
routers/rag/embeddings_issues.pkl +3 -0
routers/rag/embeddings_manual.pkl +3 -0
routers/tool_bpy_doc.py +65 -0
routers/tool_bpy_doc_v4_1.pkl +3 -0
routers/tool_calls.py +116 -0
routers/tool_find_related.py +296 -0
routers/tool_gpu_checker.py +213 -0
routers/tool_wiki_search.py +201 -0
routers/utils_gitea.py +126 -0
static/favicon.ico +0 -0
static/privace.txt +35 -0
utils/generate_bpy_doc.py +224 -0

.gitignore ADDED Viewed

	@@ -0,0 +1,3 @@

+.vs
+__pycache__/
+routers/cache

Dockerfile ADDED Viewed

	@@ -0,0 +1,20 @@

+FROM python:3.11-slim
+WORKDIR /code
+COPY requirements-fastapi.txt ./
+RUN pip install --no-cache-dir --upgrade -r /code/requirements-fastapi.txt
+RUN useradd -m -u 1000 user
+USER user
+ENV HOME=/home/user \
+	PATH=/home/user/.local/bin:$PATH
+WORKDIR $HOME/app
+COPY --chown=user . $HOME/app/
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "7860"]

README.md CHANGED Viewed

@@ -1,12 +1,11 @@
 ---
 title: Tools
-emoji: 🏆
-colorFrom: pink
-colorTo: pink
 sdk: docker
 pinned: false
 license: apache-2.0
-short_description: Tools
 ---
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

 ---
 title: Tools
+emoji: 🌍
+colorFrom: blue
+colorTo: purple
 sdk: docker
 pinned: false
 license: apache-2.0
 ---
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

__init__.py ADDED Viewed

File without changes

config.py ADDED Viewed

	@@ -0,0 +1,14 @@

+from pydantic_settings import BaseSettings
+import os
+class Settings(BaseSettings):
+    huggingface_key: str = os.environ.get("huggingface_key")
+    OPENAI_API_KEY: str = os.environ.get("OPENAI_API_KEY")
+    embedding_api: str = "sbert"
+    embedding_model: str = "mano-wii/BAAI_bge-base-en-v1.5-tunned-for-blender-issues"
+    # embedding_api: str = "openai"
+    # embedding_model: str = "text-embedding-ada-002"
+settings = Settings()

main.py ADDED Viewed

	@@ -0,0 +1,58 @@

+# main.py
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import HTMLResponse
+from fastapi.staticfiles import StaticFiles
+from huggingface_hub import login
+from config import settings
+from routers import tool_bpy_doc, tool_gpu_checker, tool_find_related, tool_wiki_search, tool_calls
+login(settings.huggingface_key)
+app = FastAPI(openapi_url="/api/v1/openapi.json",
+              docs_url="/api/v1/docs")
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["https://projects.blender.org"],
+    allow_methods=["GET", "POST"],
+    allow_headers=["Authorization", "Content-Type"],
+    allow_credentials=True,
+)
+app.include_router(
+    tool_bpy_doc.router, prefix="/api/v1", tags=["Tools"])
+app.include_router(
+    tool_gpu_checker.router, prefix="/api/v1", tags=["Tools"])
+app.include_router(
+    tool_find_related.router, prefix="/api/v1", tags=["Tools"])
+app.include_router(
+    tool_wiki_search.router, prefix="/api/v1", tags=["Tools"])
+app.include_router(
+    tool_calls.router, prefix="/api/v1", tags=["Function Calls"])
+@app.get("/", response_class=HTMLResponse)
+async def root():
+    return """
+<!DOCTYPE html>
+<html>
+<head>
+    <title>My Endpoints</title>
+</head>
+<body>
+    <h1>Welcome to @mano-wii API</h1>
+    <p>Click the button below to access the documentation:</p>
+    <a href="/api/v1/docs" style="text-decoration: none;">
+        <button style="padding: 10px 20px; background-color: #007BFF; color: #fff; border: none; cursor: pointer;">Go to Documentation</button>
+    </a>
+</body>
+</html>
+"""
+app.mount("/api/v1/static", StaticFiles(directory="static"), name="static")

requirements-fastapi.txt ADDED Viewed

	@@ -0,0 +1,7 @@

+fastapi
+uvicorn[standard]
+python-multipart
+pydantic-settings
+huggingface_hub
+sentence_transformers
+openai

routers/__init__.py ADDED Viewed

	@@ -0,0 +1 @@


1	+ # __init__.py

routers/rag/__init__.py ADDED Viewed

	@@ -0,0 +1,226 @@

+# routers/embedding/__init__.py
+import os
+import re
+import sys
+import threading
+import torch
+from sentence_transformers import SentenceTransformer, util
+from typing import Dict, List, Tuple, Set, LiteralString
+class EmbeddingContext:
+    # These don't change
+    TOKEN_LEN_MAX_FOR_EMBEDDING = 512
+    # Set when creating the object
+    lock = None
+    model = None
+    openai_client = None
+    model_name = ''
+    config_type = ''
+    embedding_shape = None
+    embedding_dtype = None
+    embedding_device = None
+    # Updates constantly
+    data = {}
+    def __init__(self):
+        try:
+            from config import settings
+        except:
+            sys.path.append(os.path.abspath(
+                os.path.join(os.path.dirname(__file__), '../..')))
+            from config import settings
+        self.lock = threading.Lock()
+        config_type = settings.embedding_api
+        model_name = settings.embedding_model
+        if config_type == 'sbert':
+            self.model = SentenceTransformer(model_name, use_auth_token=False)
+            self.model.max_seq_length = self.TOKEN_LEN_MAX_FOR_EMBEDDING
+            print("Max Sequence Length:", self.model.max_seq_length)
+            self.encode = self.encode_sbert
+            if torch.cuda.is_available():
+                self.model = self.model.to('cuda')
+        elif config_type == 'openai':
+            from openai import OpenAI
+            self.openai_client = OpenAI(
+                # base_url = settings.openai_api_base
+                api_key=settings.OPENAI_API_KEY,
+            )
+            self.encode = self.encode_openai
+        self.model_name = model_name
+        self.config_type = config_type
+        tmp = self.encode(['tmp'])
+        self.embedding_shape = tmp.shape[1:]
+        self.embedding_dtype = tmp.dtype
+        self.embedding_device = tmp.device
+    def encode(self, texts_to_embed):
+        pass
+    def encode_sbert(self, texts_to_embed):
+        return self.model.encode(texts_to_embed, show_progress_bar=True, convert_to_tensor=True, normalize_embeddings=True)
+    def encode_openai(self, texts_to_embed):
+        import math
+        import time
+        tokens_count = 0
+        for text in texts_to_embed:
+            tokens_count += len(self.get_tokens(text))
+            chunks_num = math.ceil(tokens_count / 500000)
+            chunk_size = math.ceil(len(texts_to_embed) / chunks_num)
+            embeddings = []
+            for i in range(chunks_num):
+                start = i * chunk_size
+                end = start + chunk_size
+                chunk = texts_to_embed[start:end]
+                embeddings_tmp = self.openai_client.embeddings.create(
+                    model=self.model_name,
+                    input=chunk,
+                ).data
+                if embeddings_tmp is None:
+                    break
+                embeddings.extend(embeddings_tmp)
+                if i < chunks_num - 1:
+                    time.sleep(60)  # Wait 1 minute before the next call
+            return torch.stack([torch.tensor(embedding.embedding, dtype=torch.float32) for embedding in embeddings])
+    def get_tokens(self, text):
+        if self.model:
+            return self.model.tokenizer.tokenize(text)
+        tokens = []
+        for token in re.split(r'(\W|\b)', text):
+            if token.strip():
+                tokens.append(token)
+        return tokens
+class SplitDocs:
+    def split_in_topics(self,
+                        filedir: LiteralString = None,
+                        *,
+                        pattern_filename=r'(?<!navigation)\.(md|rst)',
+                        pattern_content_sub=r'---\nhide:[\s\S]+?---\s*',
+                        patterns_titles=(
+                            r'^# (.+)', r'^## (.+)', r'^### (.+)'),
+                        ) -> List[Tuple[str, str]]:
+        def matches_pattern(filename):
+            return re.search(pattern_filename, filename) is not None
+        def split_patterns_recursive(patterns, text, index=-1):
+            sections = re.split(patterns[0], text, flags=re.MULTILINE)
+            for i, section in enumerate(sections):
+                if not section.strip():
+                    continue
+                is_match = bool(i & 1)
+                if is_match:
+                    yield (index, section)
+                elif len(patterns) > 1:
+                    for j, section_j in split_patterns_recursive(patterns[1:], section, index + 1):
+                        yield (j, section_j)
+                else:
+                    yield (-1, section)
+        for root, _, files in os.walk(filedir):
+            for name in files:
+                if not matches_pattern(name):
+                    continue
+                full_path = os.path.join(root, name)
+                with open(full_path, 'r', encoding='utf-8') as file:
+                    content = file.read()
+                if pattern_content_sub:
+                    content = re.sub(pattern_content_sub, '', content)
+                rel_path = full_path.replace(filedir, '').replace('\\', '/')
+                # Protect code parts
+                patterns = (r'(```[\s\S]+?```)', *patterns_titles)
+                last_titles = []
+                last_titles_index = []
+                content_accum = ''
+                for i, section in split_patterns_recursive(patterns, content):
+                    if i < 0:
+                        content_accum += section
+                        continue
+                    if content_accum:
+                        yield rel_path, last_titles, content_accum
+                        content_accum = ''
+                    if not last_titles_index or i > last_titles_index[-1]:
+                        last_titles_index.append(i)
+                        last_titles.append(section)
+                        continue
+                    while len(last_titles_index) > 1 and i < last_titles_index[-1]:
+                        last_titles_index.pop()
+                        last_titles.pop()
+                    # Replace
+                    last_titles_index[-1] = i
+                    last_titles[-1] = section
+                if content_accum or i != -1:
+                    yield rel_path, last_titles, content_accum
+    def reduce_text(_self, text):
+        text = re.sub(r'^\n+', '', text)  # Strip
+        text = re.sub(r'<.*?>', '', text)  # Remove HTML tags
+        text = re.sub(r':\S*: ', '', text)  # Remove [:...:] patterns
+        text = re.sub(r'\s*\n+', '\n', text)
+        return text
+    def embedding_header(_self, rel_path, titles):
+        return f"{rel_path}\n# {' | '.join(titles)}\n\n"
+    def split_for_embedding(self,
+                            filedir: LiteralString = None,
+                            *,
+                            pattern_filename=r'(?<!navigation)\.(md|rst)',
+                            pattern_content_sub=r'---\nhide:[\s\S]+?---\s*',
+                            patterns_titles=(
+                                r'^# (.+)', r'^## (.+)', r'^### (.+)'),
+                            ):
+        tokenizer = EMBEDDING_CTX.model.tokenizer
+        max_tokens = EMBEDDING_CTX.model.max_seq_length
+        texts = []
+        for rel_path, titles, content in self.split_in_topics(
+                filedir, pattern_filename=pattern_filename, pattern_content_sub=pattern_content_sub, patterns_titles=patterns_titles):
+            header = self.embedding_header(rel_path, titles)
+            tokens_pre_len = len(tokenizer.tokenize(header))
+            tokens_so_far = tokens_pre_len
+            text_so_far = header
+            for part in self.reduce_text(content).splitlines():
+                part += '\n'
+                part_tokens_len = len(tokenizer.tokenize(part))
+                if tokens_so_far + part_tokens_len > max_tokens:
+                    texts.append(text_so_far)
+                    text_so_far = header
+                    tokens_so_far = tokens_pre_len
+                text_so_far += part
+                tokens_so_far += part_tokens_len
+            if tokens_so_far != tokens_pre_len:
+                texts.append(text_so_far)
+        return texts
+EMBEDDING_CTX = EmbeddingContext()

routers/rag/embeddings_dev_docs.pkl ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:efb0a98ccf275d0a1b3b24e9ec29c741bacbb452eb825ee2ec7f1697c258b0c8
+size 17001110

routers/rag/embeddings_issues.pkl ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:fccb39b103030b69f46d734c9d29e8c4c6ca66d8cc6b7ffb995df8429d45a6c5
+size 748953800

routers/rag/embeddings_manual.pkl ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:3d8129bc22629ff95b72518f11c724043611e81ed78ad6c7f2e090973f79f71a
+size 23067144

routers/tool_bpy_doc.py ADDED Viewed

	@@ -0,0 +1,65 @@

+# bpydoc.py
+import pickle
+from fastapi import APIRouter
+from fastapi.responses import PlainTextResponse
+router = APIRouter()
+with open("routers/tool_bpy_doc_v4_1.pkl", 'rb') as file:
+    bpy_doc_map = pickle.load(file)
+    bpy_doc_map['__info'] = {'bases': None}
+def bpy_doc_get_documentation(api):
+    parts = api.split('.')
+    api = ""
+    data = bpy_doc_map
+    ctx = []
+    for part in parts:
+        try:
+            data = data[part]
+            api += part
+            ctx.append((api, data["__info"]['descr']))
+            api += '.'
+        except Exception as ex:
+            descr = f"{type(ex).__name__}: {ex}. Perhaps this object was implemented in a later version."
+            ctx.append((api, descr))
+            break
+    documentation = ""
+    for obj, descr in ctx:
+        documentation += f"{obj}:\n"
+        documentation += f"{descr}\n\n"
+    if len(data) > 1:
+        documentation += f"Members of {api}:\n"
+        info = data["__info"]
+        if info['bases']:
+            documentation += f"Inherits from {info['bases']}\n"
+        for key, val in data.items():
+            if key != "__info":
+                descr = ""
+                if isinstance(val, list):
+                    val = val[0]
+                    descr = "`bpy_prop_collection` of {}".format(
+                        val['__info']["descr"].replace('\n', ' '))
+                else:
+                    descr = val['__info']["descr"].replace('\n', ' ')
+                documentation += f"- {key}: {descr}\n"
+    return documentation
+@router.get("/bpy_doc", response_class=PlainTextResponse)
+def bpy_doc(api: str = ""):
+    message = bpy_doc_get_documentation(api)
+    return message
+if __name__ == "__main__":
+    test = bpy_doc("bpy.context")
+    print(test)

routers/tool_bpy_doc_v4_1.pkl ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:fd35f7e45cbd214cf92f14f53dccfe209deebc4d1f444061f8089dc4440d483c
+size 1876094

routers/tool_calls.py ADDED Viewed

	@@ -0,0 +1,116 @@

+import json
+import logging
+from fastapi import APIRouter, Body
+from typing import List, Dict
+from pydantic import BaseModel
+try:
+    from tool_gpu_checker import gpu_checker_get_message
+    from tool_bpy_doc import bpy_doc_get_documentation
+    from tool_find_related import find_related
+    from tool_wiki_search import wiki_search
+except:
+    from routers.tool_gpu_checker import gpu_checker_get_message
+    from routers.tool_bpy_doc import bpy_doc_get_documentation
+    from routers.tool_find_related import find_related
+    from routers.tool_wiki_search import wiki_search
+class ToolCallFunction(BaseModel):
+    name: str
+    arguments: str
+class ToolCallInput(BaseModel):
+    id: str
+    type: str
+    function: ToolCallFunction
+router = APIRouter()
+def process_tool_call(tool_call: ToolCallInput) -> Dict:
+    output = {"tool_call_id": tool_call.id, "output": ""}
+    function_name = tool_call.function.name
+    try:
+        function_args = json.loads(tool_call.function.arguments)
+        if function_name == "get_bpy_api_info":
+            output["output"] = bpy_doc_get_documentation(
+                function_args.get("api", ""))
+        elif function_name == "check_gpu":
+            output["output"] = gpu_checker_get_message(
+                function_args.get("gpu", ""))
+        elif function_name == "find_related":
+            output["output"] = find_related(
+                function_args["repo"], function_args["number"])
+        elif function_name == "wiki_search":
+            output["output"] = wiki_search(**function_args)
+    except json.JSONDecodeError as e:
+        error_message = f"Malformed JSON encountered at position {e.pos}: {e.msg}\n {tool_call.function.arguments}"
+        output["output"] = error_message
+        # Logging the error for further investigation
+        logging.error(f"JSONDecodeError in process_tool_call: {error_message}")
+    return output
+@router.post("/function_call", response_model=List[Dict])
+def function_call(tool_calls: List[ToolCallInput] = Body(..., description="List of tool calls in the request body")):
+    """
+    Endpoint to process tool calls.
+    Args:
+        tool_calls (List[ToolCallInput]): List of tool calls.
+    Returns:
+        List[Dict]: List of tool outputs with tool_call_id and output.
+    """
+    tool_outputs = [process_tool_call(tool_input) for tool_input in tool_calls]
+    return tool_outputs
+if __name__ == "__main__":
+    tool_calls_data = [
+        {
+            "id": "call_abc123",
+            "type": "function",
+            "function": {
+                "name": "get_bpy_api_info",
+                "arguments": "{\"api\":\"bpy.context.scene.world\"}"
+            }
+        },
+        {
+            "id": "call_abc456",
+            "type": "function",
+            "function": {
+                "name": "check_gpu",
+                "arguments": "{\"gpu\":\"Mesa Intel(R) Iris(R) Plus Graphics 640 (Kaby Lake GT3e) (KBL GT3) Intel 4.6 (Core Profile) Mesa 22.2.5\"}"
+            }
+        },
+        {
+            "id": "call_abc789",
+            "type": "function",
+            "function": {
+                "name": "find_related",
+                "arguments": "{\"repo\":\"blender\",\"number\":111434}"
+            }
+        },
+        {
+            "id": "call_abc101112",
+            "type": "function",
+            "function": {
+                "name": "wiki_search",
+                "arguments": "{\"query\":\"Set Snap Base\",\"groups\":[\"manual\"]}"
+            }
+        }
+    ]
+    tool_calls = [
+        ToolCallInput(id=tc['id'], type=tc['type'],
+                      function=ToolCallFunction(**tc['function']))
+        for tc in tool_calls_data
+    ]
+    test = function_call(tool_calls)
+    print(test)

routers/tool_find_related.py ADDED Viewed

	@@ -0,0 +1,296 @@

+# routers/find_related.py
+import os
+import pickle
+import torch
+import re
+from typing import List
+from datetime import datetime, timedelta
+from enum import Enum
+from sentence_transformers import util
+from fastapi import APIRouter
+from fastapi.responses import PlainTextResponse
+try:
+    from .rag import EMBEDDING_CTX
+    from .utils_gitea import gitea_fetch_issues, gitea_json_issue_get, gitea_issues_body_updated_at_get
+except:
+    from rag import EMBEDDING_CTX
+    from utils_gitea import gitea_fetch_issues, gitea_json_issue_get, gitea_issues_body_updated_at_get
+router = APIRouter()
+issue_attr_filter = {'number', 'title', 'body',
+                     'state', 'updated_at', 'created_at'}
+class State(str, Enum):
+    opened = "opened"
+    closed = "closed"
+    all = "all"
+class _Data(dict):
+    cache_path = "routers/rag/embeddings_issues.pkl"
+    @staticmethod
+    def _create_issue_string(title, body):
+        cleaned_body = body.replace('\r', '')
+        cleaned_body = cleaned_body.replace('**System Information**\n', '')
+        cleaned_body = cleaned_body.replace('**Blender Version**\n', '')
+        cleaned_body = cleaned_body.replace(
+            'Worked: (newest version of Blender that worked as expected)\n', '')
+        cleaned_body = cleaned_body.replace(
+            '**Short description of error**\n', '')
+        cleaned_body = cleaned_body.replace('**Addon Information**\n', '')
+        cleaned_body = cleaned_body.replace(
+            '**Exact steps for others to reproduce the error**\n', '')
+        cleaned_body = cleaned_body.replace(
+            '[Please describe the exact steps needed to reproduce the issue]\n', '')
+        cleaned_body = cleaned_body.replace(
+            '[Please fill out a short description of the error here]\n', '')
+        cleaned_body = cleaned_body.replace(
+            '[Based on the default startup or an attached .blend file (as simple as possible)]\n', '')
+        cleaned_body = re.sub(
+            r', branch: .+?, commit date: \d{4}-\d{2}-\d{2} \d{2}:\d{2}, hash: `.+?`', '', cleaned_body)
+        cleaned_body = re.sub(
+            r'\/?attachments\/[a-zA-Z0-9\-]+', 'attachment', cleaned_body)
+        cleaned_body = re.sub(
+            r'https?:\/\/[^\s/]+(?:\/[^\s/]+)*\/([^\s/]+)', lambda match: match.group(1), cleaned_body)
+        return title + '\n' + cleaned_body
+    @staticmethod
+    def _find_latest_date(issues, default_str=None):
+        # Handle the case where 'issues' is empty
+        if not issues:
+            return default_str
+        return max((issue['updated_at'] for issue in issues), default=default_str)
+    @classmethod
+    def _create_strings_to_embbed(cls, issues):
+        texts_to_embed = [cls._create_issue_string(
+            issue['title'], issue['body']) for issue in issues]
+        return texts_to_embed
+    def _data_ensure_size(self, repo, size_new):
+        ARRAY_CHUNK_SIZE = 4096
+        updated_at_old = None
+        arrays_size_old = 0
+        titles_old = []
+        try:
+            arrays_size_old = self[repo]['arrays_size']
+            if size_new <= arrays_size_old:
+                return
+            updated_at_old = self[repo]['updated_at']
+            titles_old = self[repo]['titles']
+        except:
+            pass
+        arrays_size_new = ARRAY_CHUNK_SIZE * \
+            (int(size_new / ARRAY_CHUNK_SIZE) + 1)
+        data_new = {
+            'updated_at': updated_at_old,
+            'arrays_size': arrays_size_new,
+            'titles': titles_old + [None] * (arrays_size_new - arrays_size_old),
+            'embeddings': torch.empty((arrays_size_new, *EMBEDDING_CTX.embedding_shape),
+                                      dtype=EMBEDDING_CTX.embedding_dtype,
+                                      device=EMBEDDING_CTX.embedding_device),
+            'opened':  torch.zeros(arrays_size_new, dtype=torch.bool),
+            'closed':  torch.zeros(arrays_size_new, dtype=torch.bool),
+        }
+        try:
+            data_new['embeddings'][:arrays_size_old] = self[repo]['embeddings']
+            data_new['opened'][:arrays_size_old] = self[repo]['opened']
+            data_new['closed'][:arrays_size_old] = self[repo]['closed']
+        except:
+            pass
+        self[repo] = data_new
+    def _embeddings_generate(self, repo):
+        if os.path.exists(self.cache_path):
+            with open(self.cache_path, 'rb') as file:
+                data = pickle.load(file)
+                self.update(data)
+                if repo in self:
+                    return
+        issues = gitea_fetch_issues('blender', repo, state='all', since=None,
+                                    issue_attr_filter=issue_attr_filter)
+        # issues = sorted(issues, key=lambda issue: int(issue['number']))
+        print("Embedding Issues...")
+        texts_to_embed = self._create_strings_to_embbed(issues)
+        embeddings = EMBEDDING_CTX.encode(texts_to_embed)
+        self._data_ensure_size(repo, int(issues[0]['number']))
+        self[repo]['updated_at'] = self._find_latest_date(issues)
+        titles = self[repo]['titles']
+        embeddings_new = self[repo]['embeddings']
+        opened = self[repo]['opened']
+        closed = self[repo]['closed']
+        for i, issue in enumerate(issues):
+            number = int(issue['number'])
+            titles[number] = issue['title']
+            embeddings_new[number] = embeddings[i]
+            if issue['state'] == 'open':
+                opened[number] = True
+            if issue['state'] == 'closed':
+                closed[number] = True
+    def _embeddings_updated_get(self, repo):
+        with EMBEDDING_CTX.lock:
+            if not repo in self:
+                self._embeddings_generate(repo)
+            date_old = self[repo]['updated_at']
+            issues = gitea_fetch_issues(
+                'blender', repo, since=date_old, issue_attr_filter=issue_attr_filter)
+            # Get the most recent date
+            date_new = self._find_latest_date(issues, date_old)
+            if date_new == date_old:
+                # Nothing changed
+                return self[repo]
+            self[repo]['updated_at'] = date_new
+    # autopep8: off
+            # Consider that if the time hasn't changed, it's the same issue.
+            issues = [issue for issue in issues if issue['updated_at'] != date_old]
+            self._data_ensure_size(repo, int(issues[0]['number']))
+            updated_at = gitea_issues_body_updated_at_get(issues)
+            issues_to_embed = []
+            for i, issue in enumerate(issues):
+                number = int(issue['number'])
+                self[repo]['opened'][number] = issue['state'] == 'open'
+                self[repo]['closed'][number] = issue['state'] == 'closed'
+                title_old = self[repo]['titles'][number]
+                if title_old != issue['title']:
+                    self[repo]['titles'][number] = issue['title']
+                    issues_to_embed.append(issue)
+                elif not updated_at or updated_at[i] >= date_old:
+                    issues_to_embed.append(issue)
+            if issues_to_embed:
+                print(f"Embedding {len(issues_to_embed)} issue{'s' if len(issues_to_embed) > 1 else ''}")
+                texts_to_embed = self._create_strings_to_embbed(issues_to_embed)
+                embeddings = EMBEDDING_CTX.encode(texts_to_embed)
+                for i, issue in enumerate(issues_to_embed):
+                    number = int(issue['number'])
+                    self[repo]['embeddings'][number] = embeddings[i]
+    # autopep8: on
+        return self[repo]
+    def _sort_similarity(self,
+                         repo: str,
+                         query_emb: List[torch.Tensor],
+                         limit: int,
+                         state: State = State.opened) -> list:
+        duplicates = []
+        data = self[repo]
+        embeddings = data['embeddings']
+        mask_opened = data["opened"]
+        if state == State.all:
+            mask = mask_opened | data["closed"]
+        else:
+            mask = data[state.value]
+        embeddings = embeddings[mask]
+        true_indices = mask.nonzero(as_tuple=True)[0]
+        ret = util.semantic_search(
+            query_emb, embeddings, top_k=limit, score_function=util.dot_score)
+        for score in ret[0]:
+            corpus_id = score['corpus_id']
+            number = true_indices[corpus_id].item()
+            closed_char = "" if mask_opened[number] else "~~"
+            text = f"{closed_char}#{number}{closed_char}: {data['titles'][number]}"
+            duplicates.append(text)
+        return duplicates
+    def find_relatedness(self, repo: str, number: int, limit: int = 20, state: State = State.opened):
+        data = self._embeddings_updated_get(repo)
+        # Check if the embedding already exists.
+        if data['titles'][number] is not None:
+            new_embedding = data['embeddings'][number]
+        else:
+            gitea_issue = gitea_json_issue_get('blender', repo, number)
+            text_to_embed = self._create_issue_string(
+                gitea_issue['title'], gitea_issue['body'])
+            new_embedding = EMBEDDING_CTX.encode([text_to_embed])
+        duplicates = self._sort_similarity(
+            repo, new_embedding, limit=limit, state=state)
+        if not duplicates:
+            return ''
+        if match := re.search(r'(~~)?#(\d+)(~~)?:', duplicates[0]):
+            number_cached = int(match.group(2))
+            if number_cached == number:
+                return '\n'.join(duplicates[1:])
+        return '\n'.join(duplicates)
+G_data = _Data()
+@router.get("/find_related/{repo}/{number}", response_class=PlainTextResponse)
+def find_related(repo: str = 'blender', number: int = 104399, limit: int = 15, state: State = State.opened) -> str:
+    related = G_data.find_relatedness(repo, number, limit=limit, state=state)
+    return related
+if __name__ == "__main__":
+    update_cache = True
+    if update_cache:
+        G_data._embeddings_updated_get('blender')
+        G_data._embeddings_updated_get('blender-addons')
+        with open(G_data.cache_path, "wb") as file:
+            # Converting the embeddings to be CPU compatible, as the virtual machine in use currently only supports the CPU.
+            for val in G_data.values():
+                val['embeddings'] = val['embeddings'].to(torch.device('cpu'))
+            pickle.dump(dict(G_data), file, protocol=pickle.HIGHEST_PROTOCOL)
+    # Converting the embeddings to be GPU.
+    for val in G_data.values():
+        val['embeddings'] = val['embeddings'].to(torch.device('cuda'))
+    # 'blender/blender/111434' must print #96153, #83604 and #79762
+    related1 = G_data.find_relatedness(
+        'blender', 111434, limit=20, state=State.all)
+    related2 = G_data.find_relatedness('blender-addons', 104399, limit=20)
+    print("These are the 20 most related issues:")
+    print(related1)
+    print()
+    print("These are the 20 most related issues:")
+    print(related2)

routers/tool_gpu_checker.py ADDED Viewed

	@@ -0,0 +1,213 @@

+# gpuchecker.py
+import re
+from fastapi import APIRouter
+from fastapi.responses import PlainTextResponse
+router = APIRouter()
+def _check_graphics_card_info(supported_models, unsupported_models, graphics_card_info):
+    for model_pattern, descr in supported_models.items():
+        if match := re.search(model_pattern, graphics_card_info, re.I):
+            return True, match.group(), descr.format(*match.groups())
+    for model_pattern, descr in unsupported_models.items():
+        if match := re.search(model_pattern, graphics_card_info, re.I):
+            return False, match.group(), descr.format(*match.groups())
+    return False, None, None
+def _check_amd(graphics_card_info):
+    supported_models = {
+        r"Radeon\s*6\d{2}([A-Z])?\b": "this model belongs to the RDNA 2 architecture",
+        r"(Radeon\s*)?R9\s*[A-Z0-9]+": "R9 models belongs to the GCN 1st gen or newer architecture",
+        r"(Radeon\s*)?(Pro\s*)?\bW7\d{3}(X)?\b(\s*Duo)?": "Radeon Pro W7000 models belongs to the RDNA 3 architecture",
+        r"(Radeon\s*)?(Pro\s*)?\bW([5-6])\d{3}(X)?\b(\s*Duo)?": "Radeon Pro W{2}000 models belongs to the RDNA 2 architecture",
+        r"(AMD\s*)?6800 XT": "this model belongs to the RDNA 2 architecture",
+        r"Radeon\s*(\(TM\)\s*)?RX Vega(\s*\d{2}\b)": "Radeon RX Vega models belongs to the GCN 5th gen architecture",
+        r"Radeon Pro Vega ((\d{2}(X)?|II)\b)?(\s*Duo)?": "Radeon Pro Vega models belongs to the GCN 5th gen architecture",
+        r"Radeon\s*(\(TM\)\s*)?Pro [4-5]\d{2}(X)?": "Radeon Pro 400/500 series belongs to the GCN 4th gen architecture",
+        r"Radeon VII": "RX models belongs to the GCN 5 architecture",
+        r"Radeon Graphics \(renoir": "this model belongs to the GCN 5th gen architecture (Vega)",
+        r"Radeon\s*(\(TM\)\s*)?Vega 8 (Graphics )?\(raven[^)]+\)": "this model belongs to the GCN 5th gen architecture (Vega)",
+        r"Radeon\s*(\(TM\)\s*)?(Pro\s*)?WX\s*(5\d{3}\b)": "this model belongs to the GCN 4th gen architecture",
+        r"FirePro": "FirePro models belongs to the GCN 1st gen or newer architecture",
+        r"HD\s*(7[7-9]\d{2})": "HD {0} model belongs to the GCN 1st gen or newer architecture",
+        r"(Radeon\s*)?RX\s*([5-7]\d{3})(M|X)?(\s*(XT|Series|S|XTX))?\b": "RX models belongs to the GCN 1st gen or newer architecture",
+        r"(Radeon\s*)?(RX\s*)?6(3|4)0\b": "it has Polaris 23 chip that belongs to GCN 4th gen architecture",
+        r"(Radeon\s*)?62(0|5)\b": "it has Polaris 24 chip that belongs to GCN 3st gen architecture",
+        r"(Radeon\s*)?610\b": "it has Banks chip that belongs to GCN 1st gen architecture",
+        r"(Radeon\s*)?RX\s*580(X?)\b": "it has Polaris 20 XT chip that belongs to GCN 4th gen architecture",
+        r"(Radeon\s*)?RX\s*570\b": "it has Ellesmere Pro chip that belongs to GCN 4th gen architecture",
+        r"(Radeon\s*)?RX\s*560X\b": "it has Polaris 31 XL chip that belongs to GCN 4th gen architecture",
+        r"(Radeon\s*)?RX\s*560\b": "it has Baffin XT chip that belongs to GCN 4th gen architecture",
+        r"(Radeon\s*)?5(40X|50X)\b": "it has Polaris 23 XT chip that belongs to GCN 4th gen architecture",
+        r"(Radeon\s*)?RX\s*5(40|50)\b": "it has Lexa Pro chip that belongs to GCN 4th gen architecture",
+        r"(Radeon\s*)?RX\s*480\b": "it has Arctic Islands chip that belongs to GCN 4th gen architecture",
+        r"(Radeon\s*)?(\(TM\)\s*)?RX\s*4[6-8]0(\b|D)": "it has Ellesmere chip that belongs to GCN 4st gen architecture",
+        r"(Radeon\s*)?5(30X|35)\b": "it has Polaris 24 XT chip that belongs to GCN 3rd gen architecture",
+        r"(Radeon\s*)?530\b": "it has Weston chip that belongs to GCN 3rd gen architecture",
+        r"(Radeon\s*)?520\b": "it has Banks chip that belongs to GCN 1st gen architecture",
+        r"(Radeon\s*)?(\(TM\)\s*)?R4": "Radeon R4 models belongs to the GCN 1st gen or newer architecture",
+        r"(Radeon\s*)?(\(TM\)\s*)?R5 (M)?335": "Radeon R5 M335 belongs to the GCN 1st gen architecture",
+        r"(Radeon\s*)?(\(TM\)\s*)?R7 (M)?2\d{2}(E|X)?\b": "Radeon R7 200 models belongs to GCN 1st or 2nd gen architecture",
+        r"(Radeon\s*)?(\(TM\)\s*)?R5 (M)?24\d(E|X)?\b": "Radeon R5 240 models belongs to GCN 1st gen architecture",
+        # r"Radeon\s*(\(TM\)\s*)?(Pro\s*)?Vega (Pro\s*)?": "this model belongs to the GCN 4th gen architecture",
+        # Add more model-to-architecture mappings as needed
+    }
+    unsupported_models = {
+        r"HD ([5-6])\d{3}": "HD {0}XXX models have TeraScale architecture that is older than GCN 1st gen",
+        r"HD\s*(7[3-6]\d{2})": "HD {0} model has TeraScale 2 architecture that is older than GCN 1st gen",
+        r"Radeon R5 (M)?2(2|3)\d(X)?\b": "Radeon R5 220/230 models belongs to Terascale 2 architecture that is older than GCN 1st gen",
+        r"(AMD\s*ATI\s*)?Radeon\s*680M": "AMD ATI Radeon 680M has TeraScale architecture that is older than GCN 1st gen",
+        # Add more model-to-architecture mappings as needed
+    }
+    return _check_graphics_card_info(supported_models, unsupported_models, graphics_card_info)
+def _check_nvidia(graphics_card_info):
+    supported_models = {
+        r"(GeForce )?(RTX\s*)?(?<!\d)([2-4])0[5-9]\d(\s*(RTX|Ti))?\b": "RTX {2}0 series are newer than GTX 400",
+        r"(GeForce )?(GTX\s*)?(?<!\d)16[5-9]\d(\s*(GTX|Ti))?\b": "GTX 16 series are newer than GTX 400",
+        r"(GeForce )?(GTX\s*)?(?<!\d)10[5-9]\d(\s*(GTX|Ti))?\b": "GTX 10 series are newer than GTX 400",
+        r"(GTX )?TITAN": "GTX TITAN models are newer than GTX 400",
+        r"(RTX )?\bA(\d+)": "RTX A models are newer than GTX 400",
+        r"Quadro FX \d+": "Quadro FX series uses a Quadro-based architecture",
+        r"Quadro RTX \d+": "Quadro RTX series uses a Quadro-based architecture",
+        r"Quadro (K|M|P|GP|GV)?\d+(M)?": "it uses a Quadro-based architecture",
+        r"NVS 8\d{2}(s)?\b": "it uses a Maxwell based architecture",
+        r"(Quadro )?NVS 110M\b": "it uses a Maxwell based architecture",
+        r"(GeForce )?GT 730\b": "GeForce from 700 series are newer than GTX 400. It also has 2 or 4 GB",
+        r"(GeForce )?GTX ([4-9])\d{2}(\s*(GTX|Ti))?\b": "GPUs from GTX {1}00 series are newer than GTX 400",
+        r"(GeForce )?\bMX\d{3}\b": "MX models are newer than GTX 400",
+        r"Tesla (.+)": "it has a Tesla architecture",
+        # Add more model-to-architecture mappings as needed
+    }
+    unsupported_models = {
+        r"(GeForce )(GTX )?3\d{2}": "GTX 3XX models are older than GeForce 400",
+        r"(Quadro )?NVS 50\b": "although quadro, it only supports opengl 1.3 and is older than 10 years",
+        r"(Quadro )?NVS \d{3}(s)?\b": "it is older than 10 years",
+        r"(Quadro )?NVS 1[1-2]0M\b": "it is Curie-based and older than 10 years",
+        r"(Quadro )?NVS 1\d{2}M\b": "although it is Tesla-based it is older than 10 years",
+        r"(Quadro )?NVS 4200M\b": "although it has Fermi architecture (newer than Tesla) it is older than 10 years",
+        # Add unsupported model-to-architecture mappings if needed
+    }
+    return _check_graphics_card_info(supported_models, unsupported_models, graphics_card_info)
+def _check_intel(graphics_card_info):
+    supported_models = {
+        r"HD (Graphics )?5\d{3}\b": "it has the Broadwell architecture and is less than 10 years old (from 2015)",
+        r"(Mesa\s*)?(Iris\s*)?Xe Graphics": "Tiger Lake is newer than Broadwell architecture",
+        r"Iris Plus Graphics G7": "Ice Lake is newer than Broadwell architecture",
+        r"UHD\s*(Graphics )?6[3-5]\d\b": "Coffee Lake or Comet Lake are newer than Broadwell architecture",
+        r"UHD\s*(Graphics )?62\d\b": "Kaby Lake is newer than Broadwell architecture",
+        r"HD\s*(Graphics )?(P)?6[1-3]\d\b": "Kaby Lake is newer than Broadwell architecture",
+        r"UHD\s*(Graphics )?60\d": "Gemini Lake is newer than Broadwell architecture",
+        r"UHD Graphics": "Kaby Lake, Coffee Lake or Comet Lake are newer than Broadwell architecture",
+        r"Iris": "Coffee Lake is newer than Broadwell architecture",
+        r"HD (Graphics )?5\d{2}\b": "Skylake is newer than Broadwell architecture",
+        r"Iris (Graphics )?6\d{3}\b": "it has the Broadwell architecture",
+        r"Intel(\(R\))? (Arc(\(TM\))?\s*)?(A)?7\d{2}\b": "the A770 model is based on the Intel Arc architecture that is newer than Broadwell",
+        r"Intel\s*(Arc\s*)?(A)?7\d{2}\b": "the A770 model is based on the Intel Arc architecture that is newer than Broadwell",
+        # Add more model-to-architecture mappings as needed
+    }
+    unsupported_models = {
+        r"HD (Graphics )?4\d{3}\b": "it has the Haswell architecture that is older than Broadwell architecture. Also [Due to driver issues, support for Intel HD4000 series GPUs has been dropped](https://wiki.blender.org/wiki/Reference/Release_Notes/4.0#:~:text=Due%20to%20driver%20issues%2C%20support%20for%20Intel%20HD4000%20series%20GPUs%20has%20been%20dropped)",
+        r"HD Graphics 3\d{3}\b": "Sandy Bridge is older than Broadwell architecture"
+        # Add unsupported model-to-architecture mappings if needed
+    }
+    return _check_graphics_card_info(supported_models, unsupported_models, graphics_card_info)
+def _check_apple(graphics_card_info):
+    supported_models = {
+        r"(Apple\s*)?(`)?\bM1(`)?(\s*Max)?": "it is one of the new ARM-based system designed by Apple Inc",
+        r"(Apple\s*)?(`)?\bM2(`)?(\s*Max)?": "it is one of the new ARM-based system designed by Apple Inc",
+        # Add more model-to-architecture mappings as needed
+    }
+    unsupported_models = {
+        # Add unsupported model-to-architecture mappings if needed
+    }
+    return _check_graphics_card_info(supported_models, unsupported_models, graphics_card_info)
+def _check_apple_os_version(os_version, is_apple_silicon):
+    major, minor = map(int, os_version.split(".")[:2])
+    if is_apple_silicon:
+        if major >= 11:
+            return True
+    else:
+        if major >= 10 and minor >= 15:
+            return True
+    return False
+def gpu_checker_get_message(text):
+    is_supported = False
+    vendor = None
+    model = None
+    descr = None
+    if "nvidia" in text.lower() or "rtx" in text.lower() or "gtx" in text.lower() or "geforce" in text.lower():
+        vendor = 'NVIDIA'
+        is_supported, model, descr = _check_nvidia(text)
+    elif "amd " in text.lower() or "ati " in text.lower() or "radeon" in text.lower():
+        vendor = 'AMD'
+        is_supported, model, descr = _check_amd(text)
+    elif "intel" in text.lower():
+        vendor = 'Intel'
+        is_supported, model, descr = _check_intel(text)
+    elif "apple" in text.lower() or re.search(r'\bM1\b', text):
+        vendor = 'Apple'
+        is_supported, model, descr = _check_apple(text)
+    else:
+        for func in {_check_nvidia, _check_amd, _check_intel, _check_apple}:
+            is_supported, model, descr = func(text)
+            if model:
+                vendor = 'GPU'
+                break
+    if not vendor:
+        return "Could not find graphics card information"
+    elif not model:
+        return f"Could not determine the card model from {vendor}"
+    message = f"The {vendor} card {model} is {'supported' if is_supported else 'not supported'} as {descr}"
+    if not is_supported:
+        message += """
+This GPU is below the minimum requirements for Blender, so Blender no longer provide support for it. https://www.blender.org/download/requirements/
+Installing the latest graphics driver sometimes helps to make such GPUs work, see here for more information. https://docs.blender.org/manual/en/dev/troubleshooting/gpu/index.html
+If that doesn't help, you can use Blender 2.79: https://www.blender.org/download/previous-versions/
+"""
+    return message
+@router.get("/gpu_checker", response_class=PlainTextResponse)
+def gpu_checker(gpu_info: str = ""):
+    message = gpu_checker_get_message(gpu_info)
+    return message
+if __name__ == "__main__":
+    gpu_info = "AMD Radeon HD 7660D"
+    message = gpu_checker_get_message(gpu_info)
+    print(message)

routers/tool_wiki_search.py ADDED Viewed

	@@ -0,0 +1,201 @@

+# routers/tool_wiki_search.py
+import base64
+import os
+import pickle
+import re
+import torch
+from enum import Enum
+from fastapi import APIRouter, Query, params
+from fastapi.responses import PlainTextResponse
+from heapq import nlargest
+from sentence_transformers import util
+from typing import Dict, List, Tuple, Set, LiteralString
+try:
+    from .rag import SplitDocs, EMBEDDING_CTX
+    from .utils_gitea import gitea_wiki_page_get, gitea_wiki_pages_get
+except:
+    from rag import SplitDocs, EMBEDDING_CTX
+    from utils_gitea import gitea_wiki_page_get, gitea_wiki_pages_get
+MANUAL_DIR = "D:/BlenderDev/blender-manual/manual"
+DOCS_DIR = "D:/BlenderDev/blender-developer-docs/docs"
+class Group(str, Enum):
+    dev_docs = "dev_docs"
+    # wiki = "wiki"
+    manual = "manual"
+GROUPS_DEFAULT = {Group.dev_docs, Group.manual}
+class _Data(dict):
+    cache_path = "routers/rag/embeddings_{}.pkl"
+    def __init__(self):
+        for grp in list(Group):
+            cache_path = self.cache_path.format(grp.name)
+            if os.path.exists(cache_path):
+                with open(cache_path, 'rb') as file:
+                    self[grp.name] = pickle.load(file)
+                continue
+            # Generate
+            print("Embedding Texts for", grp.name)
+            self[grp.name] = {}
+            # Create a list to store the text files
+            if grp is Group.dev_docs:
+                texts = self.docs_get_texts_to_embed()
+            # elif grp is Group.wiki:
+                # texts = self.wiki_get_texts_to_embed()
+            else:
+                texts = self.manual_get_texts_to_embed()
+            self[grp]['texts'] = texts
+            self[grp]['embeddings'] = EMBEDDING_CTX.encode(texts)
+            with open(cache_path, "wb") as file:
+                # Converting the embeddings to be CPU compatible, as the virtual machine in use currently only supports the CPU.
+                self[grp]['embeddings'] = self[grp]['embeddings'].to(
+                    torch.device('cpu'))
+                pickle.dump(self[grp], file, protocol=pickle.HIGHEST_PROTOCOL)
+    @classmethod
+    def manual_get_texts_to_embed(cls):
+        class SplitManual(SplitDocs):
+            def reduce_text(_self, text):
+                # Remove repeated characters
+                text = re.sub(r'\^{3,}', '', text)
+                text = re.sub(r'-{3,}', '', text)
+                text = text.replace('.rst', '.html')
+                text = super().reduce_text(text)
+                return text
+            def embedding_header(self, rel_path, titles):
+                rel_path = rel_path.replace('.rst', '.html')
+                return super().embedding_header(rel_path, titles)
+        # Remove patterns ".. word::" and ":word:"
+        pattern_content_sub = r'\.\. [^\n]+\n+(?: {3,}[^\n]*\n)*|:\w+:'
+        patterns_titles = (
+            r'[\*#%]{3,}\n\s*(.+)\n[\*#%]{3,}', r'(?:[=+]{3,}\n)?\s*(.+)\n[=+]{3,}\n')
+        return SplitManual().split_for_embedding(
+            MANUAL_DIR,
+            pattern_content_sub=pattern_content_sub,
+            patterns_titles=patterns_titles,
+        )
+    @staticmethod
+    def wiki_get_texts_to_embed():
+        class SplitWiki(SplitDocs):
+            def split_in_topics(_self,
+                                filedir: LiteralString = None,
+                                *,
+                                pattern_filename=None,
+                                pattern_content_sub=None,
+                                patterns_titles=None):
+                owner = "blender"
+                repo = "blender"
+                pages = gitea_wiki_pages_get(owner, repo)
+                for page_name in pages:
+                    page_name_title = page_name["title"]
+                    page = gitea_wiki_page_get(owner, repo, page_name_title)
+                    rel_dir = f'/{owner}/{repo}/{page["sub_url"]}'
+                    titles = [page_name_title]
+                    text = base64.b64decode(
+                        page["content_base64"]).decode('utf-8')
+                    yield (rel_dir, titles, text)
+            def reduce_text(_self, text):
+                text = super().reduce_text(text)
+                text = text.replace('https://projects.blender.org', '')
+                return text
+        return SplitWiki().split_for_embedding()
+    @staticmethod
+    def docs_get_texts_to_embed():
+        class SplitBlenderDocs(SplitDocs):
+            def reduce_text(_self, text):
+                text = super().reduce_text(text)
+                # Remove .md or index.md
+                text = re.sub(r'(index)?.md', '', text)
+                return text
+            def embedding_header(_self, rel_path, titles):
+                rel_path = re.sub(r'(index)?.md', '', rel_path)
+                return super().embedding_header(rel_path, titles)
+        return SplitBlenderDocs().split_for_embedding(DOCS_DIR)
+    def _sort_similarity(
+            self,
+            text_to_search: str,
+            groups: Set[Group] = Query(
+                default={Group.dev_docs, Group.manual}),
+            limit: int = 5) -> List[str]:
+        base_url: Dict[Group, str] = {
+            Group.dev_docs: "https://developer.blender.org/docs",
+            # Group.wiki: "https://projects.blender.org",
+            Group.manual: "https://docs.blender.org/manual/en/dev"
+        }
+        query_emb = EMBEDDING_CTX.encode([text_to_search])
+        results: List[Tuple[float, str, Group]] = []
+        for grp in groups:
+            if grp not in self:
+                continue
+            search_results = util.semantic_search(
+                query_emb, self[grp]['embeddings'], top_k=limit, score_function=util.dot_score)
+            for score in search_results[0]:
+                corpus_id = score['corpus_id']
+                text = self[grp]['texts'][corpus_id]
+                results.append((score['score'], text, grp))
+        # Keep only the top `limit` results
+        top_results = nlargest(limit, results, key=lambda x: x[0])
+        # Extract sorted texts with base URL
+        sorted_texts = [base_url[grp] + text for _, text, grp in top_results]
+        return sorted_texts
+G_data = _Data()
+router = APIRouter()
+@router.get("/wiki_search", response_class=PlainTextResponse)
+def wiki_search(
+    query: str = "",
+    groups: Set[Group] = Query(default=GROUPS_DEFAULT)
+) -> str:
+    try:
+        groups = GROUPS_DEFAULT.intersection(groups)
+        if len(groups) == 0:
+            raise
+    except:
+        groups = GROUPS_DEFAULT
+    texts = G_data._sort_similarity(query, groups)
+    result: str = ''
+    for text in texts:
+        result += f'\n---\n{text}'
+    return result
+if __name__ == '__main__':
+    tests = ["Set Snap Base", "Building the Manual",
+             "Bisect Object", "Who are the Triagers", "4.3 Release Notes Motion Paths"]
+    result = wiki_search(tests[0])
+    print(result)

routers/utils_gitea.py ADDED Viewed

	@@ -0,0 +1,126 @@

+# utils_gitea.py
+import json
+import urllib.error
+import urllib.parse
+import urllib.request
+from concurrent.futures import ThreadPoolExecutor, as_completed
+BASE_API_URL = "https://projects.blender.org/api/v1"
+def url_json_get(url, data=None):
+    try:
+        if data:
+            data = json.dumps(data).encode('utf-8')
+            request = urllib.request.Request(url, data=data, method='POST')
+            request.add_header('Content-Type', 'application/json')
+        else:
+            request = urllib.request.Request(url)
+        response = urllib.request.urlopen(request)
+        response_data = json.loads(response.read())
+        return response_data
+    except urllib.error.URLError as ex:
+        print("Error making HTTP request:", ex)
+        return None
+def url_json_get_all_pages(url, item_filter=None, limit=50, exclude=set(), verbose=False):
+    assert limit <= 50, "50 is the maximum limit of items per page"
+    url_for_page = f"{url}&limit={limit}&page="
+    with urllib.request.urlopen(url_for_page + '1') as response:
+        headers_first = response.info()
+        json_data_first = json.loads(response.read())
+    total_count = int(headers_first.get('X-Total-Count'))
+    total_pages = (total_count + limit - 1) // limit
+    def fetch_page(page):
+        if page == 1:
+            json_data = json_data_first
+        else:
+            json_data = url_json_get(url_for_page + str(page))
+        if verbose:
+            print(f"Fetched page {page}")
+        data = []
+        for item in json_data:
+            if exclude and int(item["number"]) in exclude:
+                continue
+            data.append({k: item[k] for k in item_filter}
+                        if item_filter else item)
+        return data
+    with ThreadPoolExecutor() as executor:
+        futures = [executor.submit(fetch_page, page)
+                   for page in range(1, total_pages + 1)]
+        all_results = [future.result() for future in as_completed(futures)]
+    return [item for sublist in all_results for item in sublist]
+def gitea_json_issue_get(owner, repo, number):
+    """
+    Get issue/pull JSON data.
+    """
+    url = f"{BASE_API_URL}/repos/{owner}/{repo}/issues/{number}"
+    return url_json_get(url)
+def gitea_fetch_issues(owner, repo, state='all', labels='', issue_attr_filter=None, since=None, exclude=set()):
+    query_params = {
+        'labels': labels,
+        'state': state,
+        'type': 'issues'}
+    if since:
+        query_params['since'] = since
+    base_url = f"{BASE_API_URL}/repos/{owner}/{repo}/issues"
+    encoded_query_params = urllib.parse.urlencode(query_params)
+    issues_url = f"{base_url}?{encoded_query_params}"
+    return url_json_get_all_pages(issues_url, item_filter=issue_attr_filter, exclude=exclude, verbose=True)
+def gitea_issues_body_updated_at_get(issues, verbose=True):
+    def fetch_issue(issue):
+        number = issue['number']
+        if verbose:
+            print(f"Fetched issue #{number}")
+        json_data = url_json_get(
+            f"https://projects.blender.org/blender/blender/issues/{number}/content-history/list")
+        # Verify that the response contains the expected data before trying to access it
+        if json_data and json_data['results']:
+            return json_data['results'][0]['name'].split('datetime="')[1].split('"')[0]
+        else:
+            return issue['created_at']
+    with ThreadPoolExecutor() as executor:
+        futures = [executor.submit(fetch_issue, issue) for issue in issues]
+        all_results = [future.result() for future in as_completed(futures)]
+    return all_results
+def gitea_wiki_page_get(owner, repo, page_name, verbose=True):
+    """
+    Get a wiki page.
+    """
+    encoded_page_name = urllib.parse.quote(page_name, safe='')
+    base_url = f"{BASE_API_URL}/repos/{owner}/{repo}/wiki/page/{encoded_page_name}"
+    return url_json_get(base_url)
+def gitea_wiki_pages_get(owner, repo, verbose=True):
+    """
+    Get all wiki pages.
+    """
+    base_url = f"{BASE_API_URL}/repos/{owner}/{repo}/wiki/pages"
+    return url_json_get(base_url)

static/favicon.ico ADDED Viewed

static/privace.txt ADDED Viewed

	@@ -0,0 +1,35 @@

+**Privacy Policy for Blender Assistant**
+Effective Date: November 24, 2023
+This Privacy Policy outlines the information we do not collect and provides an overview of the usage of the Blender Assistant and endpoints accessible at `https://mano-wii-function-calling.hf.space/api/v1`.
+### Information We Do Not Collect
+Blender Assistant does not collect the following information:
+- **Conversations**: The chat API used connects directly to OpenAI's servers, so only OpenAI has access to the conversations.
+- **Personally Identifiable Information**: The API does not gather personally identifiable information which includes IP or emails addresses.
+- **Cookies**: The API does not utilize cookies.
+### API Description
+The API provide a chat and tools to provide assistance to Blender users.
+## Tools:
+ "/gpu_checker": Determine if a GPU model is supported by Blender.
+ "/bpy_doc": Returns the documentation for a bpy python object or error description if not supported.
+ "/get_issue": Get the title, body, user and assets of the current report on the page.
+ "/get_messages": Get a list of all messages in the report.
+### Your Consent
+By using the Blender Assistant API, you consent to our privacy policy as described herein.
+### Changes to Our Privacy Policy
+If we decide to modify our privacy policy, we will update it and provide the revised version.
+This document is CC-BY-SA. It was last updated on November 19, 2023.
+For any questions or concerns regarding this privacy policy or the Blender 3D Information API, please contact us at grmncv@gmail.com.

utils/generate_bpy_doc.py ADDED Viewed

	@@ -0,0 +1,224 @@

+import bpy
+import inspect
+import time
+import pickle
+import mathutils
+import os
+import bpy_types
+import addon_utils
+import sys
+INFO_MEMBER = "__info"
+def get_info(name="", descr="", bases=None):
+    return {"name": name,
+            "descr":  descr,
+            "bases": bases}
+##################################################################
+g_bpy_types = {}
+def doc_from_bpy_struct(bl_rna):
+    bases = []
+    try:
+        base = bl_rna.base
+        while base:
+            bases.append(type(base).__name__)
+            base = base.base
+    except:
+        if not bases:
+            bases = None
+    return get_info(name=bl_rna.name, descr=bl_rna.description, bases=bases)
+def bpy_type_first_step(bpy_type):
+    def is_member_from_base_class(bpy_type, identifier):
+        if identifier in bpy.types.ID.bl_rna.properties:
+            return True
+        bases = bpy_type.mro()[1:]
+        for base in bases:
+            if not hasattr(base, "bl_rna"):
+                continue
+            if identifier in base.bl_rna.properties:
+                return True
+        return False
+    info = doc_from_bpy_struct(bpy_type.bl_rna)
+    data = {INFO_MEMBER: info}
+    for prop in bpy_type.bl_rna.properties:
+        identifier = prop.identifier
+        if is_member_from_base_class(bpy_type, identifier):
+            continue
+        if prop.type == 'POINTER':
+            srna_type = prop.fixed_type.identifier
+            try:
+                pointer_type = getattr(bpy.types, srna_type)
+                data[identifier] = pointer_type
+            except Exception:
+                pass
+            continue
+        if prop.type == 'COLLECTION':
+            if prop.srna:
+                srna_type = prop.srna.identifier
+                pointer_type = getattr(bpy.types, srna_type)
+                data[identifier] = pointer_type
+            elif srna_type := prop.fixed_type.identifier:
+                pointer_type = getattr(bpy.types, srna_type)
+                data[identifier] = [pointer_type]
+            continue
+        info_member = doc_from_bpy_struct(prop)
+        data[identifier] = {INFO_MEMBER: info_member}
+    return data
+def bpy_types_first_step():
+    global g_bpy_types
+    for bpy_type_name in dir(bpy.types):
+        bpy_type = getattr(bpy.types, bpy_type_name)
+        if not hasattr(bpy_type, "bl_rna"):
+            continue
+        g_bpy_types[bpy_type] = bpy_type_first_step(bpy_type)
+def bpy_types_second_step():
+    global g_bpy_types
+    for bpy_type, map in g_bpy_types.items():
+        for key, val in map.items():
+            if hasattr(val, "bl_rna"):
+                map[key] = g_bpy_types[val]
+            elif isinstance(val, list):
+                val[0] = g_bpy_types[val[0]]
+##################################################################
+bases_builtin = {int, bool, float, str, bytes, tuple, list,
+                 set, dict, mathutils.Vector, mathutils.Color, type(None)}
+def is_member_inherited(obj, member):
+    mro_bases = inspect.getmro(type(obj))
+    mro_bases_set = set(mro_bases)
+    intersection = mro_bases_set.intersection(bases_builtin)
+    for base in intersection:
+        if hasattr(base, member):
+            return True
+    return False
+def get_doc_recursive(parent, member):
+    ob = getattr(parent, member)
+    member_info = getattr(type(parent), member, ob)
+    if type(member_info) in bases_builtin or member == "bpy_func":
+        descr = type(member_info).__name__
+        return {INFO_MEMBER: get_info(descr=descr)}
+    if hasattr(type(ob), "bl_rna"):
+        return g_bpy_types[type(ob)]
+    if "bl_rna" in dir(ob):
+        return g_bpy_types[ob]
+    result = {}
+    descr = member_info.__doc__ if member_info.__doc__ else type(ob).__name__
+    result[INFO_MEMBER] = get_info(descr=descr)
+    for name in dir(ob):
+        if name.startswith("_"):
+            continue
+        if is_member_inherited(ob, name):
+            continue
+        ob_member = getattr(ob, name, None)
+        if ob_member == parent:
+            descr = type(parent).__name__
+            result[name] = {INFO_MEMBER: get_info(descr=descr)}
+            continue
+        if ob_member == os:
+            continue
+        if ob_member == bpy:
+            continue
+        if ob_member == bpy_types:
+            continue
+        if ob_member == addon_utils:
+            continue
+        if ob_member == sys:
+            continue
+        if name == "addon_install":
+            # This raises a Error
+            continue
+        result[name] = get_doc_recursive(ob, name)
+    return result
+##################################################################
+def print_doc_recursive(map, indent, name, max_step=3):
+    time.sleep(.5)
+    prefix = indent * '|'
+    print(prefix + name)
+    for key, val in map.items():
+        if key == INFO_MEMBER:
+            print(prefix + val.replace('\n', '\n' + prefix) + '\n' + prefix)
+        elif indent < max_step:
+            name_next = name + '.' + key
+            if isinstance(val, list):
+                print_doc_recursive(val[0], indent + 1,
+                                    name_next + "[0]", max_step=max_step)
+            else:
+                print_doc_recursive(
+                    val, indent + 1, name_next, max_step=max_step)
+def main():
+    print("-------------------------------------------------------------")
+    bpy_types_first_step()
+    bpy_types_second_step()
+    members = (
+        "app",
+        "context",
+        "data",
+        "msgbus",
+        "ops",
+        "path",
+        "props",
+        "types",
+        "utils",
+    )
+    result = {
+        "bpy": {INFO_MEMBER: get_info(descr=bpy.__doc__)},
+        "__info": {"bases": None},
+    }
+    for member in members:
+        result["bpy"][member] = get_doc_recursive(bpy, member)
+    # Reference some types at the beginning
+    result["bpy_struct"] = result["bpy"]["types"]["bpy_struct"]
+    result["bpy_types"] = result["bpy"]["types"]
+    if False:
+        print(result["bpy"]["props"]["BoolProperty"])
+        return
+    # print_doc_recursive(result, 1, "bpy")
+    bpy_doc_dir = "D:/Dev/function-calling/routersbpy_doc_v41.pkl"
+    with open(bpy_doc_dir, "wb") as file:
+        # print(result["types"]["bpy_func"])
+        pickle.dump(result, file, protocol=pickle.HIGHEST_PROTOCOL)
+    print(f"File '{bpy_doc_dir}' has been updated.")
+if __name__ == '__main__':
+    main()