concurrent github requests

api/concurrent_external_services.py

@@ -0,0 +1,113 @@
+import re
+import os
+from fastapi import HTTPException
+from dotenv import load_dotenv
+from llama_index import download_loader
+from llama_hub.github_repo import GithubRepositoryReader, GithubClient
+from llama_index import VectorStoreIndex
+from llama_index.vector_stores import DeepLakeVectorStore
+from llama_index.storage.storage_context import StorageContext
+import yaml
+
+load_dotenv()
+
+# Fetch and set API keys
+openai_api_key = os.getenv("OPENAI_API_KEY")
+
+
+# Check for OpenAI API key
+if not openai_api_key:
+    raise EnvironmentError("OpenAI API key not found in environment variables")
+
+
+def get_validate_token(token_name):
+    token = os.getenv(token_name)
+    if not token:
+        raise EnvironmentError(f"{token_name} not found in environment variables")
+    return token
+
+
+class InitiazlizeGithubService:
+    def __init__(self):
+        self.owner = None
+        self.repo = None
+        self.github_token = get_validate_token("GITHUB_TOKEN")  # Check for GitHub Token
+        self.github_client = self.initialize_github_client(self.github_token)
+        download_loader("GithubRepositoryReader")
+
+    def initialize_github_client(self, github_token):
+        return GithubClient(github_token)
+
+    def parse_github_url(self, url):
+        pattern = r"https://github\.com/([^/]+)/([^/]+)"
+        match = re.match(pattern, url)
+        return match.groups() if match else (None, None)
+
+    def validate_owner_repo(self, owner, repo):
+        if bool(owner) and bool(repo):
+            self.owner = owner
+            self.repo = repo
+            return True
+
+        return False
+
+    def load_repo_data(self, owner, repo, file_type):
+        if self.validate_owner_repo(owner, repo):
+            loader = GithubRepositoryReader(
+                self.github_client,
+                owner=self.owner,
+                repo=self.repo,
+                filter_file_extensions=(
+                    [file_type],
+                    GithubRepositoryReader.FilterType.INCLUDE,
+                ),
+                verbose=False,
+                concurrent_requests=25,
+            )
+
+            print(
+                f"Loading {self.repo} repository by {self.owner}, file type: {file_type}"
+            )
+
+            docs = loader.load_data(branch="main")
+            print("Documents uploaded:")
+            for doc in docs:
+                print(doc.metadata)
+
+            return docs
+
+        else:
+            raise HTTPException(
+                status_code=400,
+                detail="Invalid GitHub URL. Please enter a valid GitHub URL",
+            )
+
+
+class InitiazlizeActiveloopService:
+    def __init__(self):
+        self.active_loop_token = get_validate_token(
+            "ACTIVELOOP_TOKEN"
+        )  # Check for Activeloop Token
+        self.dataset_path = self.get_user_info("dataset_path")
+        self.vector_store = DeepLakeVectorStore(
+            dataset_path=f"hub://{self.dataset_path}",
+            overwrite=True,
+            runtime={"tensor_db": True},
+        )
+
+        self.storage_context = StorageContext.from_defaults(
+            vector_store=self.vector_store
+        )
+
+    def upload_to_activeloop(self, docs):
+        self.index = VectorStoreIndex.from_documents(
+            docs, storage_context=self.storage_context
+        )
+        self.query_engine = self.index.as_query_engine()
+
+    def get_user_info(self, user_info):
+        with open("resources.yaml", "r") as file:
+            yaml_data = yaml.safe_load(file)
+
+        retrieved_info = yaml_data["info"][user_info]
+        return retrieved_info

api/concurrent_requests.py

@@ -0,0 +1,78 @@
+import textwrap
+from fastapi import FastAPI
+from pydantic import BaseModel
+from dotenv import load_dotenv
+
+from api.concurrent_external_services import (
+    InitiazlizeGithubService,
+    InitiazlizeActiveloopService,
+)
+import asyncio
+
+# Load environment variables
+load_dotenv()
+
+github_service = InitiazlizeGithubService()
+activeloop_service = InitiazlizeActiveloopService()
+
+app = FastAPI()
+
+
+class GitHubRepoRequest(BaseModel):
+    githubRepoUrl: str
+
+
+class UserCodeRequest(BaseModel):
+    userCode: str
+
+
+async def process_file(owner, repo, file_type):
+    docs = github_service.load_repo_data(owner, repo, file_type)
+    activeloop_service.upload_to_activeloop(docs)
+    return {
+        "status": "success",
+        "message": f"File type {file_type} processed successfully",
+    }
+
+
+@app.post("/upload")
+async def scrape_and_upload_to_activeloop(repo_request: GitHubRepoRequest):
+    # Add logic to scrape and upload to ActiveLoop
+    # Example: Scrape GitHub repo and upload to ActiveLoop
+    # Implement your scraping and upload logic here
+
+    print(f"repo from user: {repo_request.githubRepoUrl}")
+
+    owner, repo = github_service.parse_github_url(repo_request.githubRepoUrl)
+    file_types = [".py", ".js", ".ts", ".md", "ipynb"]
+    tasks = []
+
+    for file_type in file_types:
+        task = process_file(owner, repo, file_type)
+        tasks.append(task)
+
+    results = await asyncio.gather(*tasks)
+    # docs = github_service.load_repo_data(owner, repo, file_type)
+    # activeloop_service.upload_to_activeloop(docs)
+    print(results)
+    return {"status": "success", "message": "Repo processed successfully"}
+
+
+@app.post("/retrieve")
+async def find_similar_code_and_explain(code_request: UserCodeRequest):
+    # Add logic to find similar code and provide explanations or improvements
+    # Example: Search in ActiveLoop DB
+    # Implement your search and analysis logic here
+
+    print(f"code from user: {code_request.userCode}")
+
+    # intro_question = "What is the repository about?"
+    intro_question = code_request.userCode
+    print(f"Test question: {intro_question}")
+    print("=" * 50)
+
+    answer = activeloop_service.query_engine.query(intro_question)
+    print(f"Answer: {answer.__dict__}\n")
+    return {
+        "answer": answer,
+    }

api/external_services.py

@@ -51,23 +51,21 @@ class InitiazlizeGithubService:
 
         return False
 
-    def load_repo_data(self, owner, repo, file_type):
+    def load_repo_data(self, owner, repo):
         if self.validate_owner_repo(owner, repo):
             loader = GithubRepositoryReader(
                 self.github_client,
                 owner=self.owner,
                 repo=self.repo,
                 filter_file_extensions=(
-                    [file_type],
+                    [".py", ".js", ".ts", ".md", ".ipynb"],
                     GithubRepositoryReader.FilterType.INCLUDE,
                 ),
                 verbose=False,
                 concurrent_requests=25,
             )
 
-            print(
-                f"Loading {self.repo} repository by {self.owner}, file type: {file_type}"
-            )
+            print(f"Loading {self.repo} repository by {self.owner}")
 
             docs = loader.load_data(branch="main")
             print("Documents uploaded:")

api/main.py

@@ -4,7 +4,6 @@ from pydantic import BaseModel
 from dotenv import load_dotenv
 
 from api.external_services import InitiazlizeGithubService, InitiazlizeActiveloopService
-import asyncio
 
 # Load environment variables
 load_dotenv()
@@ -23,15 +22,6 @@ class UserCodeRequest(BaseModel):
     userCode: str
 
 
-async def process_file(owner, repo, file_type):
-    docs = github_service.load_repo_data(owner, repo, file_type)
-    activeloop_service.upload_to_activeloop(docs)
-    return {
-        "status": "success",
-        "message": f"File type {file_type} processed successfully",
-    }
-
-
 @app.post("/upload")
 async def scrape_and_upload_to_activeloop(repo_request: GitHubRepoRequest):
     # Add logic to scrape and upload to ActiveLoop
@@ -41,17 +31,9 @@ async def scrape_and_upload_to_activeloop(repo_request: GitHubRepoRequest):
     print(f"repo from user: {repo_request.githubRepoUrl}")
 
     owner, repo = github_service.parse_github_url(repo_request.githubRepoUrl)
-    file_types = [".py", ".js", ".ts", ".md", "ipynb"]
-    tasks = []
-
-    for file_type in file_types:
-        task = process_file(owner, repo, file_type)
-        tasks.append(task)
+    docs = github_service.load_repo_data(owner, repo)
+    activeloop_service.upload_to_activeloop(docs)
 
-    results = await asyncio.gather(*tasks)
-    # docs = github_service.load_repo_data(owner, repo, file_type)
-    # activeloop_service.upload_to_activeloop(docs)
-    print(results)
     return {"status": "success", "message": "Repo processed successfully"}