Spaces:
Runtime error
Runtime error
File size: 2,694 Bytes
2e7f3da |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 |
from fastapi import FastAPI
from typing import List, Dict
import pandas as pd
import requests
from bs4 import BeautifulSoup
app = FastAPI()
# Global variable to store the dataset
kali_tools_df = None
def scrape_kali_tools(base_url: str = "https://www.kali.org/tools/") -> pd.DataFrame:
"""
Scrapes the Kali Linux tools documentation page and returns a structured dataset.
Parameters:
- base_url: The URL of the Kali Linux tools documentation.
Returns:
- Pandas DataFrame containing tool names, descriptions, and links.
"""
response = requests.get(base_url)
response.raise_for_status()
soup = BeautifulSoup(response.text, "html.parser")
# Extract tool names and descriptions
tools = []
for tool in soup.select(".tools--index__item"):
name = tool.select_one(".tools--index__title").get_text(strip=True)
description = tool.select_one(".tools--index__description").get_text(strip=True)
link = tool.find("a", href=True)["href"]
tools.append({"name": name, "description": description, "link": link})
# Convert to DataFrame
return pd.DataFrame(tools)
@app.get("/scrape_kali_tools/")
def scrape_kali_tools_endpoint():
"""
Scrapes the Kali Linux tools documentation and stores it in memory.
Returns:
- Message indicating the dataset has been created.
"""
global kali_tools_df
kali_tools_df = scrape_kali_tools()
return {"message": f"Scraped {len(kali_tools_df)} tools from Kali Linux documentation."}
@app.get("/get_kali_tools/")
def get_kali_tools(start: int = 0, limit: int = 10) -> List[Dict]:
"""
Fetches a chunk of the Kali tools dataset.
Parameters:
- start: Starting index of the tools to fetch.
- limit: Number of tools to return.
Returns:
- A list of tools with their names, descriptions, and links.
"""
if kali_tools_df is None:
return {"error": "Dataset not yet scraped. Call /scrape_kali_tools first."}
return kali_tools_df.iloc[start:start + limit].to_dict(orient="records")
@app.get("/search_kali_tools/")
def search_kali_tools(keyword: str) -> List[Dict]:
"""
Searches the Kali tools dataset for a specific keyword.
Parameters:
- keyword: Keyword to search in tool names or descriptions.
Returns:
- A list of tools matching the keyword.
"""
if kali_tools_df is None:
return {"error": "Dataset not yet scraped. Call /scrape_kali_tools first."}
results = kali_tools_df[
kali_tools_df["name"].str.contains(keyword, case=False) |
kali_tools_df["description"].str.contains(keyword, case=False)
]
return results.to_dict(orient="records") |