Last Update

app/routes.py

@@ -5,11 +5,70 @@ import numpy as np
 import requests
 from bs4 import BeautifulSoup
 import os
+from dotenv import load_dotenv
 
+load_dotenv()
 router = APIRouter()
-data = pd.read_csv('app/data-formatted.csv')
 HF_API_KEY = os.getenv("HF_API_KEY")
 
+from bs4 import BeautifulSoup
+import requests
+import csv
+import pandas as pd
+
+
+def getLatestStatistics(symbol):
+    all_company_data = []
+    file_path = 'output_file_update.csv'
+
+    url = f'https://www.mse.mk/mk/stats/symbolhistory/{symbol}'
+    response = requests.get(url)
+    content = BeautifulSoup(response.text, 'html.parser')
+    table_of_data = content.select("#resultsTable tbody tr")
+
+    for row in table_of_data:
+        data = row.text.split('\n')
+        data.pop()
+        data.remove('')
+        if data.__contains__(''):
+            continue
+        data.insert(0, symbol)
+        all_company_data.append(data)
+
+    dataframe = pd.DataFrame(all_company_data, columns=['COMPANY', 'DATE', 'PRICE OF LAST TRANSACTION', 'MAX', 'MIN', 'AVERAGE PRICE', '% PERCENT', 'QUANTITY', 'REVENUE IN BEST DENARS', 'TOTAL REVENUE IN DENARS'])
+
+    def string_to_float(column):
+        if column == 'DATE':
+            return dataframe[column]
+        if column == 'TOTAL REVENUE IN DENARS':
+            dataframe[column] = dataframe[column].str.replace('.', '').astype(float)
+            return dataframe[column]
+        if column == 'COMPANY':
+            return dataframe[column]
+        try:
+            # Replace commas with dots, then remove extra dots and convert to float
+            dataframe[column] = dataframe[column].str.replace(',', '.').str.replace('.', '', 1)
+            dataframe[column] = dataframe[column].astype(float)
+        except Exception as e:
+            print(f"Error processing column {column}: {e}")
+        return dataframe[column]
+
+    # Apply the transformation to each column
+    for col in dataframe.columns:
+        dataframe[col] = string_to_float(col)
+
+    dataframe['DATE'] = pd.to_datetime(dataframe['DATE'])
+
+    current_data = pd.read_csv('app/data-formatted.csv')
+    current_data['DATE'] = pd.to_datetime(current_data['DATE'])
+
+    merged_df = pd.concat([current_data, dataframe], ignore_index=True)
+    updated_df = merged_df.drop_duplicates(keep='first')
+
+    updated_df.to_csv('app/data-formatted.csv', index=False)
+
+    return updated_df
+
 
 def getBerzaNews(symbol):
     url = f'https://www.mse.mk/en/symbol/{symbol}'
@@ -34,7 +93,7 @@ def getBerzaNews(symbol):
     return news
 
 
-
+# Load a multilingual model
 def analyzeSentiment(symbol):
     API_URL = "https://api-inference.huggingface.co/models/mrm8488/distilroberta-finetuned-financial-news-sentiment-analysis"
     headers = {"Authorization": f"Bearer {HF_API_KEY}"}
@@ -110,6 +169,9 @@ def predict_future_price(input_data):
 # Function to resample data for timeframes
 def resample_data(data, timeframe):
     data["DATE"] = pd.to_datetime(data["DATE"])  # Ensure DATE is in datetime format
+
+    data = data.drop_duplicates(subset="DATE", keep="first")  # Drop duplicate dates
+
     data = data.set_index("DATE")  # Set DATE as the index
 
     # Select only numeric columns for resampling
@@ -125,7 +187,9 @@ def resample_data(data, timeframe):
     else:
         raise ValueError("Invalid timeframe. Choose '1D', '1W', or '1M'.")
 
-    # Reset the index to bring DATE back as a column
+    print(f"Resampled data for {timeframe} timeframe:")
+    print(resampled_data)
+
     resampled_data = resampled_data.reset_index()
 
     # Reattach non-numeric columns (e.g., COMPANY)
@@ -182,8 +246,8 @@ def calculate_technical_indicators(data, column="PRICE OF LAST TRANSACTION"):
 @router.get("/stock-data/{ticker}")
 async def get_stock_data(ticker: str):
     print(f"Fetching data for ticker: {ticker}")
-
-    stock_data = data[data["COMPANY"] == ticker]
+    latest_data = getLatestStatistics(ticker)
+    stock_data = latest_data[latest_data["COMPANY"] == ticker]
     if stock_data.empty:
         print("No data found for the given ticker.")
         return {"error": "Ticker not found"}