univariate reg model

getDailyData.py

@@ -135,6 +135,8 @@ def get_daily(mode='daily', periods_30m=None):
 
     # Target for clf -- whether tomorrow will close above or below today's close
     data['Target_clf'] = data['Close'] > data['PrevClose']
+    data['ClosePct'] = (data['Close'] / data['PrevClose']) - 1
+    data['ClosePct'] =  data['ClosePct'].shift(-1)
     data['Target_clf'] = data['Target_clf'].shift(-1)
     data['DayOfWeek'] = pd.to_datetime(data.index)
     data['Quarter'] = data['DayOfWeek'].dt.quarter
@@ -297,8 +299,9 @@ def get_daily(mode='daily', periods_30m=None):
 
     elif mode=='intra':
         from intraCols import model_cols
+        from regrCols import model_cols as regr_cols
 
-    df_final = data.loc[:final_row, model_cols + ['Target', 'Target_clf']]
+    df_final = data.loc[:final_row, model_cols + ['Target', 'Target_clf', 'ClosePct']]
     df_final = df_final.dropna(subset=['Target','Target_clf'])
     # df_final = df_final.dropna(subset=['Target','Target_clf','Perf5Day_n1'])
     return data, df_final, final_row

lambda_function.py

@@ -4,6 +4,8 @@
 from getDailyData import get_daily
 from model_intra_v3 import walk_forward_validation
 from model_day_v2 import walk_forward_validation_seq as walk_forward_validation_daily
+from model_regr_v2 import walk_forward_validation as walk_forward_validation_regr
+from model_regr_v2 import calc_upper_lower
 import pandas as pd
 import json
 from dbConn import connection, engine, insert_dataframe_to_sql
@@ -64,13 +66,19 @@ def is_refresh_time():
 def lambda_handler(periods_30m):
     if periods_30m > 0:
         data, df_final, final_row = get_daily(mode='intra', periods_30m=periods_30m)
+        # Regression model
         res, _ = walk_forward_validation(df_final.drop(columns=['Target']).dropna(), 'Target_clf', 1, mode='single')
+        regr_res, _ = walk_forward_validation_regr(df_final[['CurrentClose30toClose','ClosePct']].dropna(), 'ClosePct', 1, mode='single')
+        df_regr_results = pd.read_sql_query(f'select * from reg_results where ModelNum = {str(periods_30m)}', con = engine)
+        regr_pct = regr_res['Predicted'].iloc[-1]
+        upper, lower = calc_upper_lower(regr_pct, df_regr_results, alpha=0.05)
 
     elif periods_30m == 0:
         data, df_final, final_row = get_daily()
         res, _, _ = walk_forward_validation_daily(df_final.dropna(), 'Target_clf', 'Target', 200, 1)
-    
+
     # Get results, run calibration and pvalue    
+
     df_results = pd.read_sql_query(f'select * from results where ModelNum = {str(periods_30m)}', con = engine)
 
     # Calibrate Probabilities
@@ -103,6 +111,20 @@ def lambda_handler(periods_30m):
     df_write = pd.DataFrame.from_dict({k:[v] for k, v in blob.items()})
     cursor = connection.cursor()
     insert_dataframe_to_sql('results', df_write, cursor)
+
+    if periods_30m > 0:
+        regr_blob = {
+            'Datetime': str(res.index[-1]),
+            'IsTrue':df_final['ClosePct'].iloc[-1],
+            'Predicted': regr_pct,
+            'Upper': upper,
+            'Lower':lower,
+            'ModelNum':periods_30m,
+            'AsOf':str(asof)
+        }
+        df_write_reg = pd.DataFrame.from_dict({k:[v] for k, v in regr_blob.items()})
+        insert_dataframe_to_sql('reg_results', df_write_reg, cursor)
+
     # cursor.close()
     # connection.close()
 
@@ -110,15 +132,15 @@ def lambda_handler(periods_30m):
 
 if __name__ == '__main__':
     # Code that, based on the time of the day, return which data/model to run
-    game_time = False # is_trading_day_and_time()
-    refresh_time = True # is_refresh_time()
+    game_time = is_trading_day_and_time()
+    refresh_time = is_refresh_time()
     if game_time:
         now = datetime.datetime.now()
         # Change this for debugging -- should be EST
         morning_start = datetime.datetime.combine(now.date(), time(9, 30))
         delta = now - morning_start
         print(delta)
-        intervals = 7 # max(0,min((delta.total_seconds() / 60 / 30) // 1, 12))
+        intervals = max(0,min((delta.total_seconds() / 60 / 30) // 1, 12))
         print(f'running for {str(intervals)}')
         j = lambda_handler(intervals)
     elif refresh_time:

model_intra_v2.py

@@ -9,7 +9,6 @@ def walk_forward_validation(df, target_column, num_periods, mode='full'):
     
     df = df[model_cols + [target_column]]
     df[target_column] = df[target_column].astype(bool)
-
     tscv = TimeSeriesSplit(n_splits=len(df)-1, max_train_size=None, test_size=num_periods)  # num_splits is the number of splits you want
 
     if mode == 'full':

model_regr_v2.py

@@ -0,0 +1,91 @@
+import pandas as pd
+import numpy as np
+from tqdm import tqdm
+from sklearn.linear_model import LinearRegression
+from sklearn.model_selection import TimeSeriesSplit
+from regrCols import model_cols
+
+def walk_forward_validation(df, target_column, num_periods, mode='full'):
+    
+    df = df[model_cols + [target_column]]
+    df[target_column] = df[target_column].astype(float)
+
+    tscv = TimeSeriesSplit(n_splits=len(df)-1, max_train_size=None, test_size=num_periods)  # num_splits is the number of splits you want
+
+    if mode == 'full':
+        overall_results = []
+        # Iterate over the rows in the DataFrame, one step at a time
+        # Split the time series data using TimeSeriesSplit
+        for train_index, test_index in tqdm(tscv.split(df), total=tscv.n_splits):
+            # Extract the training and testing data for the current split
+            X_train = df.drop(target_column, axis=1).iloc[train_index]
+            y_train = df[target_column].iloc[train_index]
+            X_test = df.drop(target_column, axis=1).iloc[test_index]
+            y_test = df[target_column].iloc[test_index]
+        
+            y_train = y_train.astype(float)
+            model = LinearRegression()
+            model.fit(X_train, y_train)
+            # Make a prediction on the test data
+            predictions = model.predict(X_test)
+                
+            # Create a DataFrame to store the true and predicted values
+            result_df = pd.DataFrame({'IsTrue': y_test, 'Predicted': predictions}, index=y_test.index)
+            overall_results.append(result_df)
+
+        df_results = pd.concat(overall_results)
+        
+        uppers = []
+        lowers = []
+        alpha = 0.05
+        for i, pct in tqdm(enumerate(df_results['Predicted']), desc='Calibrating Probas',total=len(df_results)):
+            try:
+                
+                df_q = df_results.iloc[:i]
+                pred = df_results['Predicted'].iloc[-1]
+                errors = df_q['IsTrue'] - df_q['Predicted']
+                positive_errors = errors[errors >= 0]
+                negative_errors = errors[errors < 0]
+
+                # Calculate bounds
+                upper_bound = pred + np.quantile(positive_errors, 1 - alpha)
+                lower_bound = pred + np.quantile(negative_errors, alpha)
+                
+            except:
+                upper_bound = None
+                lower_bound = None
+
+            uppers.append(upper_bound)
+            lowers.append(lower_bound)
+
+        df_results['Upper'] = uppers
+        df_results['Lower'] = lowers
+
+    elif mode == 'single':
+        X_train = df.drop(target_column, axis=1).iloc[:-1]
+        y_train = df[target_column].iloc[:-1]
+        X_test = df.drop(target_column, axis=1).iloc[-1]
+        y_test = df[target_column].iloc[-1]
+        y_train = y_train.astype(float)
+        model = LinearRegression()
+        model.fit(X_train, y_train)
+        predictions = model.predict(X_test.values.reshape(1, -1))
+        df_results = pd.DataFrame({'IsTrue': y_test, 'Predicted': predictions}, index=[df.index[-1]])
+
+    return df_results, model
+        
+def calc_upper_lower(pred, df_hist, alpha=0.05):
+    errors = df_hist['IsTrue'] - df_hist['Predicted']
+    positive_errors = errors[errors >= 0]
+    negative_errors = errors[errors < 0]
+
+    # Calculate bounds
+    upper_bound = pred + np.quantile(positive_errors, 1 - alpha)
+    lower_bound = pred + np.quantile(negative_errors, alpha)
+
+    return upper_bound, lower_bound
+
+
+def seq_predict_proba(df, trained_clf_model):
+    clf_pred_proba = trained_clf_model.predict_proba(df[model_cols])[:,-1]
+    return clf_pred_proba

regrCols.py

@@ -0,0 +1,3 @@
+model_cols = [
+    'CurrentClose30toClose'
+]