added mods

.gitattributes

@@ -32,3 +32,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.blend filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -1,4 +1,7 @@
 # Byte-compiled / optimized / DLL files
+*.mp4
+*.xz
+*.gif
 *.zip
 *.png
 Data-*

app.py

@@ -1,22 +1,60 @@
+import torch
 import gradio as gr
 
+from phate import PHATEAE
+from funcs.som import ClusterSOM
+
 from funcs.processor import process_data
 from funcs.plot_func import plot_sensor_data_from_json
+from funcs.dataloader import BaseDataset2, read_json_files
+
+DEVICE = torch.device("cpu")
+reducer10d = PHATEAE(epochs=30, n_components=10, lr=.0001, batch_size=128, t='auto', knn=8, relax=True, metric='euclidean')
+reducer10d.load('models/r10d_2.pth')
+
+cluster_som = ClusterSOM()
+cluster_som.load("models/cluster_som2.pkl")
+
+# ml inference
+def get_som_mp4(file, slice_select=1, reducer=reducer10d, cluster=cluster_som):
+
+    try:
+        train_x, train_y  = read_json_files(file)
+    except:
+        train_x, train_y  = read_json_files(file.name)
+
+    # Convert tensors to numpy arrays if necessary
+    if isinstance(train_x, torch.Tensor):
+        train_x = train_x.numpy()
+    if isinstance(train_y, torch.Tensor):
+        train_y = train_y.numpy()
+
+    # load the time series slices of the data 4*3*2*64 (feeds+axis*sensor*samples) + 5 for time diff
+    data = BaseDataset2(train_x.reshape(len(train_x), -1) / 32768, train_y)
+
+    #compute the 10 dimensional embeding vector
+    embedding10d = reducer.transform(data)
+
+    # prediction = cluster_som.predict(embedding10d)
+    fig = cluster.plot_activation_v2(embedding10d, slice_select)
+
+    return fig
 
 with gr.Blocks(title='Cabasus') as cabasus_sensor:
     title = gr.Markdown("<h2><center>Data gathering and processing</center></h2>")
     with gr.Tab("Convert"):
-        csv_file_box = gr.File(label='Upload CSV File') 
         with gr.Row():
-            processed_file_box = gr.File(label='Processed CSV File') 
-            json_file_box = gr.File(label='Generated Json file')
+            csv_file_box = gr.File(label='Upload CSV File') 
+            with gr.Column():
+                processed_file_box = gr.File(label='Processed CSV File') 
+                json_file_box = gr.File(label='Generated Json file')
 
         with gr.Row():
-            slice_size_slider = gr.inputs.Slider(16, 512, 1, 64, label="Slice Size")
-            sample_rate = gr.inputs.Slider(1, 199, 1, 20, label="Sample rate")     
+            slice_size_slider = gr.Slider(minimum=16, maximum=512, step=1, value=64, label="Slice Size", visible=False)
+            sample_rate = gr.Slider(minimum=1, maximum=199, step=1, value=20, label="Sample rate", visible=False)     
         with gr.Row():
-            window_size_slider = gr.inputs.Slider(0, 100, 2, 10, label="Window Size")
-            repeat_process = gr.Button('Restart process')  
+            window_size_slider = gr.Slider(minimum=0, maximum=100, step=2, value=10, label="Window Size", visible=False)
+            repeat_process = gr.Button('Restart process', visible=False)  
         with gr.Row():
             leg_dropdown = gr.Dropdown(choices=['GZ1', 'GZ2', 'GZ3', 'GZ4'], label='select leg', value='GZ1')
             slice_slider = gr.Slider(minimum=1, maximum=300, label='Current slice', step=1)
@@ -28,14 +66,21 @@ with gr.Blocks(title='Cabasus') as cabasus_sensor:
             plot_slice_leg = gr.Plot(label="Sliced Signal Plot")
             get_all_slice = gr.Plot(label="Real Signal Plot")
         
+        som_create = gr.Button('generate som')
         with gr.Row():
-            animation = gr.PlayableVideo(label="Animated horse steps")
+            som_figures = gr.Plot(label="som activations")
         
         slices_per_leg = gr.Textbox(label="Number of slices found per LEG")
         
-        csv_file_box.change(process_data, inputs=[csv_file_box, slice_size_slider, sample_rate, window_size_slider], outputs=[processed_file_box, json_file_box, slices_per_leg, plot_box_leg, plot_box_overlay, slice_slider, plot_slice_leg, get_all_slice])
-        leg_dropdown.change(plot_sensor_data_from_json, inputs=[json_file_box, leg_dropdown, slice_slider], outputs=[plot_box_leg, plot_slice_leg, get_all_slice])
-        repeat_process.click(process_data, inputs=[csv_file_box, slice_size_slider, sample_rate, window_size_slider], outputs=[processed_file_box, json_file_box, slices_per_leg, plot_box_leg, plot_box_overlay, slice_slider, plot_slice_leg, get_all_slice])
-        slice_slider.change(plot_sensor_data_from_json, inputs=[json_file_box, leg_dropdown, slice_slider], outputs=[plot_box_leg, plot_slice_leg, get_all_slice])
+        csv_file_box.change(process_data, inputs=[csv_file_box, slice_size_slider, sample_rate, window_size_slider], 
+                            outputs=[processed_file_box, json_file_box, slices_per_leg, plot_box_leg, plot_box_overlay, slice_slider, plot_slice_leg, get_all_slice])
+        leg_dropdown.change(plot_sensor_data_from_json, inputs=[json_file_box, leg_dropdown, slice_slider], 
+                            outputs=[plot_box_leg, plot_slice_leg, get_all_slice])
+        repeat_process.click(process_data, inputs=[csv_file_box, slice_size_slider, sample_rate, window_size_slider], 
+                             outputs=[processed_file_box, json_file_box, slices_per_leg, plot_box_leg, plot_box_overlay, slice_slider, plot_slice_leg, get_all_slice])
+        slice_slider.change(plot_sensor_data_from_json, inputs=[json_file_box, leg_dropdown, slice_slider], 
+                            outputs=[plot_box_leg, plot_slice_leg, get_all_slice])
+        
+        som_create.click(get_som_mp4, inputs=[json_file_box, slice_slider], outputs=[som_figures])
 
 cabasus_sensor.queue(concurrency_count=2).launch(debug=True)

funcs/ml_inference.py

@@ -0,0 +1,26 @@
+import torch
+from funcs.dataloader import BaseDataset2, read_json_files
+
+def get_som_mp4(file, reducer10d, cluster_som, slice_select=1):
+
+    try:
+        train_x, train_y  = read_json_files(file)
+    except:
+        train_x, train_y  = read_json_files(file.name)
+
+    # Convert tensors to numpy arrays if necessary
+    if isinstance(train_x, torch.Tensor):
+        train_x = train_x.numpy()
+    if isinstance(train_y, torch.Tensor):
+        train_y = train_y.numpy()
+
+    # load the time series slices of the data 4*3*2*64 (feeds+axis*sensor*samples) + 5 for time diff
+    data = BaseDataset2(train_x.reshape(len(train_x), -1) / 32768, train_y)
+
+    #compute the 10 dimensional embeding vector
+    embedding10d = reducer10d.transform(data)
+
+    # prediction = cluster_som.predict(embedding10d)
+    fig = cluster_som.plot_activation_v2(embedding10d, slice_select)
+
+    return fig

funcs/plot_func.py

@@ -6,6 +6,8 @@ import pandas as pd
 import seaborn as sns
 import matplotlib.pyplot as plt
 
+from funcs.ml_inference import get_som_mp4
+
 matplotlib.use('Agg')
 plt.style.use('ggplot')
 
@@ -59,7 +61,6 @@ def plot_sensor_data_from_json(json_file, sensor, slice_select=1):
     plt.legend()
     plt.tight_layout()
 
-
     return fig, fig1, plot_other_sensor_with_same_timestamp(json_file, sensor, slice_select)
 
 def plot_overlay_data_from_json(json_file, sensors, use_precise_timestamp=False):

funcs/processor.py

@@ -4,8 +4,9 @@ import gradio as gr
 
 from funcs.convertors import slice_csv_to_json
 from funcs.plot_func import plot_sensor_data_from_json, plot_overlay_data_from_json
+from funcs.ml_inference import get_som_mp4
 
-def process_data(input_file, slice_size=64, min_slice_size=16, sample_rate=20, window_size=40, threshold=1000, span_limit=10000000,):
+def process_data(input_file, slice_size=64, min_slice_size=16, sample_rate=20, window_size=40, threshold=1000, span_limit=10000000):
     # Read the data from the file, including the CRC column
     try:
         if input_file.name is None:

funcs/som.py

@@ -344,8 +344,8 @@ class ClusterSOM:
         images = []
         for sample in tqdm(data[start:end], desc="Visualizing prediction output"):
             prediction = self.predict([sample])[0]
-            if prediction[0] == -1:  # Noise
-                continue
+            # if prediction[0] == -1:  # Noise
+            #     continue
 
             fig, axes = plt.subplots(1, len(self.som_models), figsize=(20, 5), sharex=True, sharey=True)
             fig.suptitle(f"Activation map for SOM {prediction[0]}, node {prediction[1]}", fontsize=16)
@@ -378,7 +378,10 @@ class ClusterSOM:
             # Create a colorbar for each frame
             fig.subplots_adjust(right=0.8)
             cbar_ax = fig.add_axes([0.85, 0.15, 0.05, 0.7])
-            fig.colorbar(im_active, cax=cbar_ax)
+            try:
+                fig.colorbar(im_active, cax=cbar_ax)
+            except:
+                pass
 
             # Save the plot to a buffer
             buf = io.BytesIO()
@@ -422,4 +425,54 @@ class ClusterSOM:
 
         self.hdbscan_model, self.som_models, self.mean_values, self.sigma_values, self.cluster_mapping = model_data[:5]
         if len(model_data) > 5:
-            self.label_centroids, self.label_encodings = model_data[5:]
+            self.label_centroids, self.label_encodings = model_data[5:]
+
+
+    def plot_activation_v2(self, data, slice_select=1):
+        """
+        Generate a GIF visualization of the prediction output using the activation maps of individual SOMs.
+        """
+        if len(self.som_models) == 0:
+            raise ValueError("SOM models not trained yet.")
+        
+        prediction = self.predict([data[int(slice_select)-1]])[0]
+
+        fig, axes = plt.subplots(1, len(self.som_models), figsize=(20, 5), sharex=True, sharey=True)
+        fig.suptitle(f"Activation map for SOM {prediction[0]}, node {prediction[1]}", fontsize=16)
+
+        for idx, (som_key, som) in enumerate(self.som_models.items()):
+            ax = axes[idx]
+            activation_map = np.zeros(som._weights.shape[:2])
+            for x in range(som._weights.shape[0]):
+                for y in range(som._weights.shape[1]):
+                    activation_map[x, y] = np.linalg.norm(data[int(slice_select)-1] - som._weights[x, y])
+
+            winner = som.winner(data[int(slice_select)-1])  # Find the BMU for this SOM
+            activation_map[winner] = 0  # Set the BMU's value to 0 so it will be red in the colormap
+
+            if som_key == prediction[0]:  # Active SOM
+                im_active = ax.imshow(activation_map, cmap='viridis', origin='lower', interpolation='none')
+                ax.plot(winner[1], winner[0], 'r+')  # Mark the BMU with a red plus sign
+                ax.set_title(f"SOM {som_key}", color='blue', fontweight='bold')
+                if hasattr(self, 'label_centroids'):
+                    label_idx = self.label_encodings.inverse_transform([som_key - 1])[0]
+                    ax.set_xlabel(f"Label: {label_idx}", fontsize=12)
+            else:  # Inactive SOM
+                im_inactive = ax.imshow(activation_map, cmap='gray', origin='lower', interpolation='none')
+                ax.set_title(f"SOM {som_key}")
+
+            ax.set_xticks(range(activation_map.shape[1]))
+            ax.set_yticks(range(activation_map.shape[0]))
+            ax.grid(True, linestyle='-', linewidth=0.5)
+
+        # Create a colorbar for each frame
+        # fig.subplots_adjust(right=0.8)
+        # cbar_ax = fig.add_axes([0.85, 0.15, 0.05, 0.7])
+
+        plt.tight_layout()
+        # try:
+        #     fig.colorbar(im_active, cax=cbar_ax)
+        # except:
+        #     pass
+
+        return fig

main.py

@@ -0,0 +1,210 @@
+import csv
+import bpy
+from mathutils import *
+import sys
+
+argv = sys.argv
+argv = argv[argv.index("--") + 1:]  # get all args after "--"
+
+D = bpy.data
+C = bpy.context
+
+def copy_armature_animation(
+    source_armature_name,
+    destination_armature_name,
+    start_frame_source,
+    end_frame_source,
+    start_frame_target,
+):
+    # Get the source and destination armature objects
+    source_armature = bpy.data.objects[source_armature_name]
+    destination_armature = bpy.data.objects[destination_armature_name]
+
+    # Copy the animation data from the source armature to the destination armature
+
+    #print(f"copying animation {start_frame_source}-{end_frame_source} to {start_frame_target} ")
+    for i, frame in enumerate(range(start_frame_source, end_frame_source + 1)):
+        bpy.context.scene.frame_set(frame)
+        # Set the location and rotation of the destination bone at the given frame
+        for bone in source_armature.pose.bones:
+            destination_bone = destination_armature.pose.bones.get(bone.name)
+
+            destination_bone.location = bone.location
+            destination_bone.rotation_quaternion = bone.rotation_quaternion
+            destination_bone.keyframe_insert(
+                data_path="location", frame=start_frame_target + i
+            )
+            destination_bone.keyframe_insert(
+                data_path="rotation_quaternion", frame=start_frame_target + i
+            )
+            # Copy the bone's animation data for each frame in the range
+
+
+def adjust_armature_animation_timing(
+    armature_name, old_start, old_end, new_start, new_end
+):
+
+    # Get the armature object by name
+    armature = bpy.data.objects[armature_name]
+
+    # Calculate the ratio between the old and new keyframe ranges
+    old_range = old_end - old_start
+    new_range = new_end - new_start
+    ratio = new_range / old_range
+    #print(f"Adjsuting timming from {old_start}-{old_end} to {new_start}-{new_end} with ratio: {ratio}")
+
+
+    # Iterate through all the armature bones and modify their animation keyframes
+    for fcurve in armature.animation_data.action.fcurves:
+        if fcurve.array_index < 4:
+            for keyframe in fcurve.keyframe_points:
+                # Adjust the keyframe position based on the ratio
+                if keyframe.co.x in range (old_start, old_end):
+                    #print(f"keyframe move from {keyframe.co.x}")
+                    keyframe.co.x = (keyframe.co.x - old_start) * ratio + new_start
+                    #print(f"to {keyframe.co.x}")
+                else:
+                    continue
+
+def calculate_new_frame_range(frame_rate, timestamp, start_frame):
+
+    adjusted_end_frame = int(timestamp / 1000 * frame_rate + start_frame)
+    return adjusted_end_frame
+
+
+def change_material_property(
+    material_name, node_name, new_value, start_frame, end_frame
+):
+    value = bpy.data.materials[material_name].node_tree.nodes[node_name].outputs[0]
+    value.default_value = new_value
+    value.keyframe_insert(data_path="default_value", frame=start_frame+1)
+    value.keyframe_insert(data_path="default_value", frame=end_frame-1)
+
+
+def set_cues_state(
+    global_state,
+    global_color,
+    c1_state,
+    c1_color,
+    c2_state,
+    c2_color,
+    c3_state,
+    c3_color,
+    c4_state,
+    c4_color,
+    start_frame,
+    end_frame,
+):
+    # cue 1 noise
+    change_material_property(
+        "CUE_MAT_1", "Distortion", float(c1_state), start_frame, end_frame
+    )
+
+    # cue 2 noise
+    change_material_property(
+        "CUE_MAT_2", "Distortion", float(c2_state), start_frame, end_frame
+    )
+
+    # cue 3 noise
+    change_material_property(
+        "CUE_MAT_3", "Distortion", float(c3_state), start_frame, end_frame
+    )
+
+    # cue 4 noise
+    change_material_property(
+        "CUE_MAT_4", "Distortion", float(c4_state), start_frame, end_frame
+    )
+
+    # cue 1 color
+    change_material_property(
+        "CUE_MAT_1", "Color", float(c1_color), start_frame, end_frame
+    )
+
+    # cue 2 color
+    change_material_property(
+        "CUE_MAT_2", "Color", float(c2_color), start_frame, end_frame
+    )
+
+    # cue 3 color
+    change_material_property(
+        "CUE_MAT_3", "Color", float(c3_color), start_frame, end_frame
+    )
+
+    # cue 4 color
+    change_material_property(
+        "CUE_MAT_4", "Color", float(c4_color), start_frame, end_frame
+    )
+
+    # global cue on
+    change_material_property(
+        "GLOBAL_CUE_MAT", "MixFactor", float(global_state), start_frame, end_frame
+    )
+
+    # global cue color
+    change_material_property(
+        "GLOBAL_CUE_MAT", "Color", float(global_color), start_frame, end_frame
+    )
+
+
+def animate_cycle(source_armature, target_armature, csv_file):
+    # Get the current scene
+    scene = bpy.context.scene
+
+    # Get the frame rate of the current scene
+    frame_rate = scene.render.fps / scene.render.fps_base
+
+    gait_to_keyframe_start = {0: 10, 1: 80, 2: 60, 3: 250}
+
+    gait_to_keyframe_end = {0: 49, 1: 99, 2: 74, 3: 310}
+
+    # column_names = ["Gait", "TS", "State", "Condition", "Shape1", "Shape2", "Shape3", "Shape4", "Color1", "Color2", "Color3", "Color4", "Danger1", "Danger2", "Danger3", "Danger4"]
+    previous_end_frame = 0
+    with open(csv_file, "r") as csvfile:
+        datareader = csv.reader(csvfile)
+        for i, row in enumerate(datareader):
+            if i == 0 or row[0] is None:
+                continue
+
+            copy_armature_animation(
+                source_armature,
+                target_armature,
+                gait_to_keyframe_start[int(row[0])],
+                gait_to_keyframe_end[int(row[0])],
+                previous_end_frame + 1,
+            )
+            adjusted_end_frame = calculate_new_frame_range(
+                frame_rate=frame_rate,
+                timestamp=int(float(row[1])),
+                start_frame=previous_end_frame + 1,
+            )
+            adjust_armature_animation_timing(
+                target_armature,
+                previous_end_frame + 1,
+                previous_end_frame + 1 + gait_to_keyframe_end[int(row[0])] - gait_to_keyframe_start[int(row[0])],
+                previous_end_frame + 1,
+                adjusted_end_frame,
+            )
+            set_cues_state(
+                row[3],
+                row[4],
+                row[8],
+                row[5],
+                row[9],
+                row[6],
+                row[10],
+                row[7],
+                row[11],
+                row[2],
+                previous_end_frame,
+                adjusted_end_frame,
+            )
+            #print(previous_end_frame, adjusted_end_frame)
+            previous_end_frame = adjusted_end_frame
+            print(f"Animated row {i}")
+
+        bpy.context.scene.frame_end = previous_end_frame
+        bpy.ops.render.render(animation=True)
+
+print("Cycle_device:")
+print(bpy.context.preferences.addons["cycles"].preferences.has_active_device())
+animate_cycle("Deformation", "Deformation.001", str(argv[0]))

main_template.blend

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7e9e2e198820ce4f0448c667e293537ad9e821017209d5241156937807c40b24
+size 256848708

ml_inference.py

@@ -1,30 +0,0 @@
-import torch
-from phate import PHATEAE
-from funcs.som import ClusterSOM
-
-from funcs.dataloader import BaseDataset2, read_json_files
-
-
-DEVICE = torch.device("cpu")
-
-reducer10d = PHATEAE(epochs=30, n_components=10, lr=.0001, batch_size=128, t='auto', knn=8, relax=True, metric='euclidean')
-reducer10d.load('models/r10d_2.pth')
-
-cluster_som = ClusterSOM()
-cluster_som.load("models/cluster_som2.pkl")
-
-train_x, train_y  = read_json_files('output.json')
-# Convert tensors to numpy arrays if necessary
-if isinstance(train_x, torch.Tensor):
-    train_x = train_x.numpy()
-if isinstance(train_y, torch.Tensor):
-    train_y = train_y.numpy()
-
-# load the time series slices of the data 4*3*2*64 (feeds+axis*sensor*samples) + 5 for time diff
-data = BaseDataset2(train_x.reshape(len(train_x), -1) / 32768, train_y)
-
-#compute the 10 dimensional embeding vector
-embedding10d = reducer10d.transform(data)
-
-prediction = cluster_som.predict(embedding10d)
-cluster_som.plot_activation(embedding10d)