Update: add datasets in container (docker) environment

.github/workflows/pipeline.yaml

@@ -7,8 +7,8 @@ on:
     tags:
       - '*'   
   schedule:
-    - cron: "0 3 * * *"
-    # 8 - 7 = 1
+    - cron: "0 13 * * *"
+    # 20 - 7 = 13
 
 jobs:
   extraction_train_modeling:
@@ -116,6 +116,19 @@ jobs:
           name: posttrained.zip
           overwrite: "true"
 
+      - name: Remove Temporarary Files and Directories
+        if: env.match != 'true'
+        run: |
+          rm models.zip
+          rm pickles.zip
+          rm datasets.zip
+          rm posttrained.zip
+
+          rm -rf models
+          rm -rf pickles
+          rm -rf datasets
+          rm -rf posttrained
+
       - name: Commit changes
         if: env.match != 'true'
         run: |

Dockerfile

@@ -32,6 +32,10 @@ RUN --mount=type=secret,id=PICKLES_ID,mode=0444,required=true \
 	gdown https://drive.google.com/uc?id=$(cat /run/secrets/PICKLES_ID) && \
     unzip pickles.zip && rm pickles.zip
 
+RUN --mount=type=secret,id=DATASETS_ID,mode=0444,required=true \
+	gdown https://drive.google.com/uc?id=$(cat /run/secrets/DATASETS_ID) && \
+    unzip datasets.zip && rm datasets.zip
+
 RUN --mount=type=secret,id=POSTTRAINED_ID,mode=0444,required=true \
 	gdown https://drive.google.com/uc?id=$(cat /run/secrets/POSTTRAINED_ID) && \
     unzip posttrained.zip && rm posttrained.zip

pipeline_schedule.ctl

@@ -1 +1 @@
-2024-07-27
+2024-07-28

training.py

@@ -138,7 +138,8 @@ def main():
         dataframe = pd.read_csv(os.path.join(datasets_path, dataset), index_col='Date')[['Close']]
         model_file = os.path.join(models_path, f"{dataset.replace('.csv', '')}.keras")
 
-        dataframe = data_processor.preprocess_data(dataframe)
+        # dataframe = data_processor.preprocess_data(dataframe)
+        dataframe.dropna(inplace = True)
         standard_scaler, dataframe = data_processor.scale_data(dataframe, StandardScaler)
         minmax_scaler, dataframe = data_processor.scale_data(dataframe, MinMaxScaler)