|
from fastapi import FastAPI, UploadFile, File |
|
from fastapi.responses import HTMLResponse |
|
import pandas as pd |
|
import io |
|
import time |
|
|
|
app = FastAPI() |
|
|
|
@app.get("/", response_class=HTMLResponse) |
|
async def analyze_logs(): |
|
return """ |
|
<html> |
|
<body> |
|
<form action="/upload/" enctype="multipart/form-data" method="post"> |
|
<input name="file" type="file"> |
|
<input type="submit"> |
|
</form> |
|
</body> |
|
</html> |
|
""" |
|
|
|
@app.post("/upload/") |
|
async def upload_file(file: UploadFile = File(...)): |
|
contents = await file.read() |
|
logs_df = pd.read_parquet(io.BytesIO(contents)) |
|
|
|
processing_message = "\n\n Processing files...\n\n" |
|
time.sleep(3) |
|
|
|
logs_df['datetime'] = pd.to_datetime(logs_df['datetime'], format='%d/%m/%Y:%H:%M:%S') |
|
logs_df['day'] = logs_df['datetime'].apply(lambda x: x.day) |
|
logs_df['hour'] = logs_df['datetime'].apply(lambda x: x.hour) |
|
logs_df['minute'] = logs_df['datetime'].apply(lambda x: x.minute) |
|
|
|
ip_address_count_df = ( |
|
logs_df.groupby(['method', 'client'], as_index=False) |
|
.size() |
|
.rename(columns={'size': 'count'}) |
|
.sort_values('count', ascending=False) |
|
) |
|
|
|
ip_address_count_df = ip_address_count_df.assign( |
|
perc=ip_address_count_df['count'].div(ip_address_count_df['count'].sum()), |
|
cum_perc=lambda df: df['perc'].cumsum(), |
|
) |
|
|
|
dataframe_styled = ( |
|
ip_address_count_df.head(1000) |
|
.style.background_gradient(subset=['count', 'perc', 'cum_perc'], cmap='cividis') |
|
.format({'count': '{:,}', 'perc': '{:.1%}', 'cum_perc': '{:.1%}'}) |
|
) |
|
|
|
|
|
styled_html = dataframe_styled.render() |
|
|
|
result = ( |
|
"<h1>Redundant IP Requests....</h1>" |
|
"<p>The Total API Requests from the sample logs are : {total_requests}</p>" |
|
"<p>The Redundant API Requests from the sample logs are : {redundant_requests}</p>" |
|
"<p>The percentage of Redundant API Requests from the sample logs is : {redundant_percentage:.2f}%</p>" |
|
"{dataframe_html}" |
|
).format( |
|
total_requests=logs_df.shape[0], |
|
redundant_requests=ip_address_count_df.shape[0], |
|
redundant_percentage=(ip_address_count_df.shape[0] / logs_df.shape[0]) * 100, |
|
dataframe_html=styled_html, |
|
) |
|
|
|
return HTMLResponse(content=result) |
|
|
