Spaces:
Sleeping
Sleeping
remove unused codes
Browse files- app.py +8 -11
- visualization/et_visualizer.py +3 -19
- visualization/logger.py +3 -6
- visualization/visualizer.py +0 -2
app.py
CHANGED
@@ -3,7 +3,6 @@ from gradio_rerun import Rerun
|
|
3 |
from data.loader import load_simulation_data
|
4 |
from visualization.visualizer import visualize_simulation
|
5 |
from visualization.et_visualizer import visualize_et_data
|
6 |
-
from pathlib import Path
|
7 |
|
8 |
|
9 |
def update_simulation_dropdown(file):
|
@@ -18,7 +17,6 @@ def update_simulation_dropdown(file):
|
|
18 |
def create_app():
|
19 |
with gr.Blocks() as demo:
|
20 |
with gr.Tabs() as tabs:
|
21 |
-
# Camera Simulation Tab
|
22 |
with gr.Tab("Camera Simulation"):
|
23 |
gr.Markdown("""
|
24 |
# Camera Simulation Visualizer
|
@@ -52,7 +50,6 @@ def create_app():
|
|
52 |
outputs=[viewer]
|
53 |
)
|
54 |
|
55 |
-
# E.T. Dataset Tab
|
56 |
with gr.Tab("E.T. Dataset"):
|
57 |
gr.Markdown("""
|
58 |
# E.T. Dataset Visualizer
|
@@ -69,14 +66,6 @@ def create_app():
|
|
69 |
file_types=[".npy"]
|
70 |
)
|
71 |
|
72 |
-
with gr.Row():
|
73 |
-
et_viewer = Rerun(streaming=False)
|
74 |
-
|
75 |
-
def process_et_files(traj_file, char_file):
|
76 |
-
if traj_file is None or char_file is None:
|
77 |
-
return None
|
78 |
-
return visualize_et_data(traj_file.name, char_file.name)
|
79 |
-
|
80 |
with gr.Row():
|
81 |
visualize_btn = gr.Button("Visualize")
|
82 |
visualize_btn.click(
|
@@ -85,6 +74,14 @@ def create_app():
|
|
85 |
outputs=[et_viewer]
|
86 |
)
|
87 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
88 |
return demo
|
89 |
|
90 |
|
|
|
3 |
from data.loader import load_simulation_data
|
4 |
from visualization.visualizer import visualize_simulation
|
5 |
from visualization.et_visualizer import visualize_et_data
|
|
|
6 |
|
7 |
|
8 |
def update_simulation_dropdown(file):
|
|
|
17 |
def create_app():
|
18 |
with gr.Blocks() as demo:
|
19 |
with gr.Tabs() as tabs:
|
|
|
20 |
with gr.Tab("Camera Simulation"):
|
21 |
gr.Markdown("""
|
22 |
# Camera Simulation Visualizer
|
|
|
50 |
outputs=[viewer]
|
51 |
)
|
52 |
|
|
|
53 |
with gr.Tab("E.T. Dataset"):
|
54 |
gr.Markdown("""
|
55 |
# E.T. Dataset Visualizer
|
|
|
66 |
file_types=[".npy"]
|
67 |
)
|
68 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
69 |
with gr.Row():
|
70 |
visualize_btn = gr.Button("Visualize")
|
71 |
visualize_btn.click(
|
|
|
74 |
outputs=[et_viewer]
|
75 |
)
|
76 |
|
77 |
+
with gr.Row():
|
78 |
+
et_viewer = Rerun(streaming=False)
|
79 |
+
|
80 |
+
def process_et_files(traj_file, char_file):
|
81 |
+
if traj_file is None or char_file is None:
|
82 |
+
return None
|
83 |
+
return visualize_et_data(traj_file.name, char_file.name)
|
84 |
+
|
85 |
return demo
|
86 |
|
87 |
|
visualization/et_visualizer.py
CHANGED
@@ -20,21 +20,17 @@ def load_trajectory_data(traj_file: str, char_file: str, num_cams: int = 30) ->
|
|
20 |
raw_trans = torch.clone(matrix_trajectory[:, :3, 3])
|
21 |
raw_rot = matrix_trajectory[:, :3, :3]
|
22 |
|
23 |
-
# Convert to 6D rotation representation
|
24 |
rot6d = raw_rot[:, :, :2].permute(0, 2, 1).reshape(-1, 6)
|
25 |
trajectory_feature = torch.hstack([rot6d, raw_trans]).permute(1, 0)
|
26 |
|
27 |
-
# Pad trajectory features
|
28 |
padded_trajectory_feature = F.pad(
|
29 |
trajectory_feature,
|
30 |
(0, num_cams - trajectory_feature.shape[1])
|
31 |
)
|
32 |
|
33 |
-
# Create padding mask
|
34 |
padding_mask = torch.ones((num_cams))
|
35 |
padding_mask[trajectory_feature.shape[1]:] = 0
|
36 |
|
37 |
-
# Load and pad character features
|
38 |
char_feature = torch.from_numpy(np.load(char_file)).to(torch.float32)
|
39 |
padding_size = num_cams - char_feature.shape[0]
|
40 |
padded_char_feature = F.pad(
|
@@ -55,7 +51,7 @@ class ETLogger(SimulationLogger):
|
|
55 |
super().__init__()
|
56 |
rr.init("et_visualization")
|
57 |
rr.log("world", rr.ViewCoordinates.RIGHT_HAND_Y_UP, timeless=True)
|
58 |
-
|
59 |
self.K = np.array([
|
60 |
[500, 0, 320],
|
61 |
[0, 500, 240],
|
@@ -63,11 +59,9 @@ class ETLogger(SimulationLogger):
|
|
63 |
])
|
64 |
|
65 |
def log_trajectory(self, trajectory: np.ndarray, padding_mask: np.ndarray):
|
66 |
-
"""Log camera trajectory."""
|
67 |
valid_frames = int(padding_mask.sum())
|
68 |
valid_trajectory = trajectory[:valid_frames]
|
69 |
|
70 |
-
# Log trajectory points
|
71 |
positions = valid_trajectory[:, :3, 3]
|
72 |
rr.log(
|
73 |
"world/trajectory/points",
|
@@ -78,7 +72,6 @@ class ETLogger(SimulationLogger):
|
|
78 |
timeless=True
|
79 |
)
|
80 |
|
81 |
-
# Log trajectory line
|
82 |
if len(positions) > 1:
|
83 |
lines = np.stack([positions[:-1], positions[1:]], axis=1)
|
84 |
rr.log(
|
@@ -90,17 +83,14 @@ class ETLogger(SimulationLogger):
|
|
90 |
timeless=True
|
91 |
)
|
92 |
|
93 |
-
# Log cameras
|
94 |
for k in range(valid_frames):
|
95 |
-
|
96 |
rr.set_time_sequence("frame_idx", k)
|
97 |
|
98 |
-
# Get camera pose
|
99 |
translation = valid_trajectory[k, :3, 3]
|
100 |
rotation_q = Rotation.from_matrix(
|
101 |
valid_trajectory[k, :3, :3]).as_quat()
|
102 |
|
103 |
-
# Log camera transform
|
104 |
rr.log(
|
105 |
f"world/camera",
|
106 |
rr.Transform3D(
|
@@ -109,7 +99,6 @@ class ETLogger(SimulationLogger):
|
|
109 |
),
|
110 |
)
|
111 |
|
112 |
-
# Log camera frustum
|
113 |
rr.log(
|
114 |
f"world/camera/image",
|
115 |
rr.Pinhole(
|
@@ -120,7 +109,6 @@ class ETLogger(SimulationLogger):
|
|
120 |
)
|
121 |
|
122 |
def log_character(self, char_feature: np.ndarray, padding_mask: np.ndarray):
|
123 |
-
"""Log character feature visualization."""
|
124 |
valid_frames = int(padding_mask.sum())
|
125 |
valid_char = char_feature[:, :valid_frames]
|
126 |
|
@@ -138,16 +126,13 @@ class ETLogger(SimulationLogger):
|
|
138 |
|
139 |
@spaces.GPU
|
140 |
def visualize_et_data(traj_file: str, char_file: str) -> Optional[str]:
|
141 |
-
"""Visualize E.T. dataset using Rerun."""
|
142 |
try:
|
143 |
-
|
144 |
data = load_trajectory_data(traj_file, char_file)
|
145 |
|
146 |
-
# Create temporary file for RRD
|
147 |
temp_dir = tempfile.mkdtemp()
|
148 |
rrd_path = os.path.join(temp_dir, "et_visualization.rrd")
|
149 |
|
150 |
-
# Initialize logger and log data
|
151 |
logger = ETLogger()
|
152 |
logger.log_trajectory(
|
153 |
data["raw_matrix_trajectory"].numpy(),
|
@@ -158,7 +143,6 @@ def visualize_et_data(traj_file: str, char_file: str) -> Optional[str]:
|
|
158 |
data["padding_mask"].numpy()
|
159 |
)
|
160 |
|
161 |
-
# Save visualization
|
162 |
rr.save(rrd_path)
|
163 |
return rrd_path
|
164 |
|
|
|
20 |
raw_trans = torch.clone(matrix_trajectory[:, :3, 3])
|
21 |
raw_rot = matrix_trajectory[:, :3, :3]
|
22 |
|
|
|
23 |
rot6d = raw_rot[:, :, :2].permute(0, 2, 1).reshape(-1, 6)
|
24 |
trajectory_feature = torch.hstack([rot6d, raw_trans]).permute(1, 0)
|
25 |
|
|
|
26 |
padded_trajectory_feature = F.pad(
|
27 |
trajectory_feature,
|
28 |
(0, num_cams - trajectory_feature.shape[1])
|
29 |
)
|
30 |
|
|
|
31 |
padding_mask = torch.ones((num_cams))
|
32 |
padding_mask[trajectory_feature.shape[1]:] = 0
|
33 |
|
|
|
34 |
char_feature = torch.from_numpy(np.load(char_file)).to(torch.float32)
|
35 |
padding_size = num_cams - char_feature.shape[0]
|
36 |
padded_char_feature = F.pad(
|
|
|
51 |
super().__init__()
|
52 |
rr.init("et_visualization")
|
53 |
rr.log("world", rr.ViewCoordinates.RIGHT_HAND_Y_UP, timeless=True)
|
54 |
+
|
55 |
self.K = np.array([
|
56 |
[500, 0, 320],
|
57 |
[0, 500, 240],
|
|
|
59 |
])
|
60 |
|
61 |
def log_trajectory(self, trajectory: np.ndarray, padding_mask: np.ndarray):
|
|
|
62 |
valid_frames = int(padding_mask.sum())
|
63 |
valid_trajectory = trajectory[:valid_frames]
|
64 |
|
|
|
65 |
positions = valid_trajectory[:, :3, 3]
|
66 |
rr.log(
|
67 |
"world/trajectory/points",
|
|
|
72 |
timeless=True
|
73 |
)
|
74 |
|
|
|
75 |
if len(positions) > 1:
|
76 |
lines = np.stack([positions[:-1], positions[1:]], axis=1)
|
77 |
rr.log(
|
|
|
83 |
timeless=True
|
84 |
)
|
85 |
|
|
|
86 |
for k in range(valid_frames):
|
87 |
+
|
88 |
rr.set_time_sequence("frame_idx", k)
|
89 |
|
|
|
90 |
translation = valid_trajectory[k, :3, 3]
|
91 |
rotation_q = Rotation.from_matrix(
|
92 |
valid_trajectory[k, :3, :3]).as_quat()
|
93 |
|
|
|
94 |
rr.log(
|
95 |
f"world/camera",
|
96 |
rr.Transform3D(
|
|
|
99 |
),
|
100 |
)
|
101 |
|
|
|
102 |
rr.log(
|
103 |
f"world/camera/image",
|
104 |
rr.Pinhole(
|
|
|
109 |
)
|
110 |
|
111 |
def log_character(self, char_feature: np.ndarray, padding_mask: np.ndarray):
|
|
|
112 |
valid_frames = int(padding_mask.sum())
|
113 |
valid_char = char_feature[:, :valid_frames]
|
114 |
|
|
|
126 |
|
127 |
@spaces.GPU
|
128 |
def visualize_et_data(traj_file: str, char_file: str) -> Optional[str]:
|
|
|
129 |
try:
|
130 |
+
|
131 |
data = load_trajectory_data(traj_file, char_file)
|
132 |
|
|
|
133 |
temp_dir = tempfile.mkdtemp()
|
134 |
rrd_path = os.path.join(temp_dir, "et_visualization.rrd")
|
135 |
|
|
|
136 |
logger = ETLogger()
|
137 |
logger.log_trajectory(
|
138 |
data["raw_matrix_trajectory"].numpy(),
|
|
|
143 |
data["padding_mask"].numpy()
|
144 |
)
|
145 |
|
|
|
146 |
rr.save(rrd_path)
|
147 |
return rrd_path
|
148 |
|
visualization/logger.py
CHANGED
@@ -18,10 +18,10 @@ class SimulationLogger:
|
|
18 |
def __init__(self):
|
19 |
rr.init("camera_simulation")
|
20 |
rr.log("world", rr.ViewCoordinates.RIGHT_HAND_Y_UP, timeless=True)
|
21 |
-
|
22 |
self.K = np.array([
|
23 |
-
[500, 0, 960],
|
24 |
-
[0, 500, 540],
|
25 |
[0, 0, 1]
|
26 |
])
|
27 |
|
@@ -93,7 +93,6 @@ class SimulationLogger:
|
|
93 |
timeless=True
|
94 |
)
|
95 |
|
96 |
-
# Add trajectory line
|
97 |
if len(camera_positions) > 1:
|
98 |
lines = np.stack(
|
99 |
[camera_positions[:-1], camera_positions[1:]], axis=1)
|
@@ -120,7 +119,6 @@ class SimulationLogger:
|
|
120 |
position = vector3_to_numpy(camera_frame['position'])
|
121 |
rotation_q = euler_to_quaternion(camera_frame['angle'])
|
122 |
|
123 |
-
# Log camera transform
|
124 |
rr.log(
|
125 |
"world/camera",
|
126 |
rr.Transform3D(
|
@@ -129,7 +127,6 @@ class SimulationLogger:
|
|
129 |
)
|
130 |
)
|
131 |
|
132 |
-
# Log camera image plane with intrinsics
|
133 |
rr.log(
|
134 |
"world/camera/image",
|
135 |
rr.Pinhole(
|
|
|
18 |
def __init__(self):
|
19 |
rr.init("camera_simulation")
|
20 |
rr.log("world", rr.ViewCoordinates.RIGHT_HAND_Y_UP, timeless=True)
|
21 |
+
|
22 |
self.K = np.array([
|
23 |
+
[500, 0, 960],
|
24 |
+
[0, 500, 540],
|
25 |
[0, 0, 1]
|
26 |
])
|
27 |
|
|
|
93 |
timeless=True
|
94 |
)
|
95 |
|
|
|
96 |
if len(camera_positions) > 1:
|
97 |
lines = np.stack(
|
98 |
[camera_positions[:-1], camera_positions[1:]], axis=1)
|
|
|
119 |
position = vector3_to_numpy(camera_frame['position'])
|
120 |
rotation_q = euler_to_quaternion(camera_frame['angle'])
|
121 |
|
|
|
122 |
rr.log(
|
123 |
"world/camera",
|
124 |
rr.Transform3D(
|
|
|
127 |
)
|
128 |
)
|
129 |
|
|
|
130 |
rr.log(
|
131 |
"world/camera/image",
|
132 |
rr.Pinhole(
|
visualization/visualizer.py
CHANGED
@@ -18,11 +18,9 @@ def visualize_simulation(file, simulation_index: Optional[int]) -> Optional[str]
|
|
18 |
print(f"Invalid simulation data or index: {simulation_index}")
|
19 |
return None
|
20 |
|
21 |
-
# Create temporary file for RRD
|
22 |
temp_dir = tempfile.mkdtemp()
|
23 |
rrd_path = os.path.join(temp_dir, "simulation.rrd")
|
24 |
|
25 |
-
# Log selected simulation
|
26 |
simulation = simulations[simulation_index]
|
27 |
|
28 |
logger = SimulationLogger()
|
|
|
18 |
print(f"Invalid simulation data or index: {simulation_index}")
|
19 |
return None
|
20 |
|
|
|
21 |
temp_dir = tempfile.mkdtemp()
|
22 |
rrd_path = os.path.join(temp_dir, "simulation.rrd")
|
23 |
|
|
|
24 |
simulation = simulations[simulation_index]
|
25 |
|
26 |
logger = SimulationLogger()
|