Initial commit

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,12 @@
+---
+title: LensCraft
+emoji: 👀
+colorFrom: red
+colorTo: pink
+sdk: gradio
+sdk_version: 4.44.1
+app_file: app.py
+pinned: false
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,67 @@
+import gradio as gr
+from gradio_rerun import Rerun
+from data.loader import load_simulation_data
+from visualization.visualizer import visualize_simulation
+
+
+def update_simulation_dropdown(file):
+    simulations, descriptions = load_simulation_data(file)
+    return gr.Dropdown(
+        choices=descriptions if descriptions else [],
+        value=None,
+        allow_custom_value=False
+    )
+
+
+def create_app():
+    with gr.Blocks() as demo:
+        gr.Markdown("""
+        # Camera Simulation Visualizer
+        Upload a JSON file containing camera simulation data and select a simulation to visualize.
+        """)
+
+        with gr.Row():
+            file_input = gr.File(
+                label="Upload Simulation JSON",
+                file_types=[".json"]
+            )
+            simulation_dropdown = gr.Dropdown(
+                label="Select Simulation",
+                choices=[],
+                type="index",
+                scale=2
+            )
+
+        frame_input = gr.Textbox(
+            label="Frame Selection",
+            placeholder="E.g. 1-30, 35, 40-50 (leave empty for all frames)"
+        )
+
+        with gr.Row():
+            viewer = Rerun(streaming=False)
+
+        file_input.change(
+            update_simulation_dropdown,
+            inputs=[file_input],
+            outputs=[simulation_dropdown]
+        )
+
+        simulation_dropdown.change(
+            visualize_simulation,
+            inputs=[file_input, simulation_dropdown, frame_input],
+            outputs=[viewer]
+        )
+
+        frame_input.change(
+            visualize_simulation,
+            inputs=[file_input, simulation_dropdown, frame_input],
+            outputs=[viewer]
+        )
+
+
+    return demo
+
+
+if __name__ == "__main__":
+    demo = create_app()
+    demo.queue().launch(share=False)

data/loader.py

@@ -0,0 +1,59 @@
+import json
+from typing import Optional, Dict, Any, List, Tuple
+
+
+def create_instruction_description(instruction: Dict[str, Any]) -> str:
+    parts = []
+
+    movement = instruction.get('cameraMovement')
+    shot_type = instruction.get('initialShotType')
+    frames = instruction.get('frameCount')
+    subject_idx = instruction.get('subjectIndex')
+
+    if movement and shot_type:
+        parts.append(f"{movement} {shot_type}")
+    elif movement:
+        parts.append(f"{movement} shot")
+    elif shot_type:
+        parts.append(f"Static {shot_type}")
+
+    if subject_idx is not None:
+        parts.append(f"of subject {subject_idx}")
+
+    if frames is not None:
+        parts.append(f"lasting {frames} frames")
+
+    if not parts:
+        return "No camera instruction details available"
+
+    return " ".join(parts)
+
+
+def load_simulation_data(file) -> Tuple[Optional[List[Dict[str, Any]]], Optional[List[str]]]:
+    if file is None:
+        return None, None
+
+    try:
+        json_data = json.load(open(file.name))
+        simulations = json_data['simulations']
+
+        descriptions = []
+        for i, sim in enumerate(simulations):
+            header = f"Simulation {i + 1}"
+
+            instruction_texts = []
+            for j, instruction in enumerate(sim['instructions']):
+                inst_desc = create_instruction_description(instruction)
+                instruction_texts.append(f"  {j + 1}. {inst_desc}")
+
+            full_description = f"{header}\n" + "\n".join(instruction_texts)
+
+            subject_count = len(sim['subjects'])
+            full_description = f"{full_description}\n  ({subject_count} subjects)"
+
+            descriptions.append(full_description)
+
+        return simulations, descriptions
+    except Exception as e:
+        print(f"Error loading simulation data: {str(e)}")
+        return None, None

requirements.txt

@@ -0,0 +1,4 @@
+gradio_rerun
+scipy==1.14.1
+evo==1.30.3
+gradio==4.40.0

utils/geometry.py

@@ -0,0 +1,11 @@
+import numpy as np
+from scipy.spatial.transform import Rotation
+from typing import Dict
+
+
+def vector3_to_numpy(vec: Dict[str, float]) -> np.ndarray:
+    return np.array([vec['x'], vec['y'], vec['z']])
+
+
+def euler_to_quaternion(euler: Dict[str, float]) -> np.ndarray:
+    return Rotation.from_euler('xyz', [euler['x'], euler['y'], euler['z']]).as_quat()

visualization/logger.py

@@ -0,0 +1,187 @@
+import rerun as rr
+import numpy as np
+from typing import Dict, Any, List
+from utils.geometry import vector3_to_numpy, euler_to_quaternion
+
+
+def create_subject_box(subject: Dict) -> Dict[str, np.ndarray]:
+    position = vector3_to_numpy(subject['position'])
+    size = vector3_to_numpy(subject['size'])
+
+    return {
+        'center': position,
+        'half_size': size / 2
+    }
+
+
+class SimulationLogger:
+    def __init__(self):
+        rr.init("camera_simulation")
+        rr.log("world", rr.ViewCoordinates.RIGHT_HAND_Y_UP, timeless=True)
+
+        self.K = np.array([
+            [500, 0, 960],
+            [0, 500, 540],
+            [0, 0, 1]
+        ])
+
+    def log_metadata(self, instructions: List[Dict[str, Any]]) -> None:
+        if not instructions:
+            return
+
+        rr.log("metadata/instructions", rr.TextDocument(
+            "\n".join([
+                f"Instruction {i+1}:\n" +
+                f"  Movement: {inst.get('cameraMovement', 'N/A')}\n" +
+                f"  Easing: {inst.get('movementEasing', 'N/A')}\n" +
+                f"  Frames: {inst.get('frameCount', 'N/A')}\n" +
+                f"  Camera Angle: {inst.get('initialCameraAngle', 'N/A')}\n" +
+                f"  Shot Type: {inst.get('initialShotType', 'N/A')}\n" +
+                f"  Subject Index: {inst.get('subjectIndex', 'N/A')}"
+                for i, inst in enumerate(instructions)
+            ])
+        ), timeless=True)
+
+    def log_subjects(self, subjects: List[Dict[str, Any]]) -> None:
+        if not subjects:
+            return
+
+        centers = []
+        half_sizes = []
+        colors = []
+        labels = []
+
+        for subject in subjects:
+            try:
+                box_params = create_subject_box(subject)
+                centers.append(box_params['center'])
+                half_sizes.append(box_params['half_size'])
+                colors.append([0.8, 0.2, 0.2, 1.0])
+                labels.append(subject.get('objectClass', 'Unknown'))
+            except Exception as e:
+                print(f"Error creating box parameters: {str(e)}")
+                continue
+
+        if centers:
+            rr.log(
+                "world/subjects",
+                rr.Boxes3D(
+                    centers=np.array(centers),
+                    half_sizes=np.array(half_sizes),
+                    colors=np.array(colors),
+                    show_labels=False,
+                    fill_mode="solid"
+                ),
+                timeless=True
+            )
+
+    def log_camera_trajectory(self, camera_frames: List[Dict[str, Any]]) -> None:
+        if not camera_frames:
+            return
+
+        try:
+            camera_positions = np.array([
+                vector3_to_numpy(frame['position']) for frame in camera_frames
+            ])
+            rr.log(
+                "world/camera_trajectory",
+                rr.Points3D(
+                    camera_positions,
+                    colors=np.full((len(camera_positions), 4),
+                                   [0.0, 0.8, 0.8, 1.0])
+                ),
+                timeless=True
+            )
+
+            if len(camera_positions) > 1:
+                lines = np.stack(
+                    [camera_positions[:-1], camera_positions[1:]], axis=1)
+                rr.log(
+                    "world/camera_trajectory/line",
+                    rr.LineStrips3D(
+                        lines,
+                        colors=[(0.0, 0.8, 0.8, 1.0)]
+                    ),
+                    timeless=True
+                )
+
+        except Exception as e:
+            print(f"Error logging camera trajectory: {str(e)}")
+
+    def log_camera_frames(self, camera_frames: List[Dict[str, Any]]) -> None:
+        if not camera_frames:
+            return
+
+        for frame_idx, camera_frame in enumerate(camera_frames):
+            try:
+                rr.set_time_sequence("frame_idx", frame_idx)
+
+                position = vector3_to_numpy(camera_frame['position'])
+                rotation_q = euler_to_quaternion(camera_frame['angle'])
+
+                rr.log(
+                    "world/camera",
+                    rr.Transform3D(
+                        translation=position,
+                        rotation=rr.Quaternion(xyzw=rotation_q)
+                    )
+                )
+
+                rr.log(
+                    "world/camera/image",
+                    rr.Pinhole(
+                        image_from_camera=self.K,
+                        width=1920,
+                        height=1080
+                    )
+                )
+
+            except Exception as e:
+                print(f"Error logging camera frame {frame_idx}: {str(e)}")
+
+    def log_helper_keyframes(self, helper_keyframes: List[Dict[str, Any]]) -> None:
+        if not helper_keyframes:
+            return
+
+
+        helper_positions = np.array([
+            vector3_to_numpy(frame['position']) for frame in helper_keyframes
+        ])
+        rr.log(
+            "world/helper_keyframes",
+            rr.Points3D(
+                helper_positions,
+                radii=np.full(len(helper_positions), 0.03),
+                colors=np.full((len(helper_positions), 4), [1.0, 1.0, 0.0, 1.0]),
+            ),
+            timeless=True
+        )
+
+        for keyframe_idx, helper_keyframe in enumerate(helper_keyframes):
+            try:
+                position = vector3_to_numpy(helper_keyframe['position'])
+                rotation_q = euler_to_quaternion(helper_keyframe['angle'])
+
+                rr.log(
+                    f"world/helper_camera_{keyframe_idx}",
+                    rr.Transform3D(
+                        translation=position,
+                        rotation=rr.Quaternion(xyzw=rotation_q),
+                        scale=(.5, .5, .5)
+                    ),
+                    timeless=True
+                )
+
+                rr.log(
+                    f"world/helper_camera_{keyframe_idx}/image",
+                    rr.Pinhole(
+                        image_from_camera=self.K,
+                        width=1920,
+                        height=1080,
+                    ),
+                    timeless=True
+                )
+
+            except Exception as e:
+                print(
+                    f"Error logging helper keyframe {keyframe_idx}: {str(e)}")

visualization/visualizer.py

@@ -0,0 +1,75 @@
+import tempfile
+import os
+import spaces
+from typing import Optional, List
+from data.loader import load_simulation_data
+from visualization.logger import SimulationLogger
+import rerun as rr
+
+
+def parse_frame_selection(selection_str: str, max_frame: int) -> List[int]:
+    if not selection_str:
+        return list(range(max_frame))
+
+    frames = set()
+    segments = selection_str.split(',')
+    for seg in segments:
+        seg = seg.strip()
+        if '-' in seg:
+            # range
+            start_str, end_str = seg.split('-', 1)
+            start, end = int(start_str), int(end_str)
+            # Convert to 0-based indexing
+            for f in range(start - 1, end):
+                if 0 <= f < max_frame:
+                    frames.add(f)
+        else:
+            # single frame
+            f = int(seg) - 1
+            if 0 <= f < max_frame:
+                frames.add(f)
+
+    return sorted(frames)
+
+
+@spaces.GPU
+def visualize_simulation(file, simulation_index: Optional[int], frame_selection: str) -> Optional[str]:
+    if file is None or simulation_index is None:
+        return None
+
+    try:
+        simulations, _ = load_simulation_data(file)
+        if not simulations or not isinstance(simulation_index, int) or simulation_index < 0 or simulation_index >= len(simulations):
+            print(f"Invalid simulation data or index: {simulation_index}")
+            return None
+
+        simulation = simulations[simulation_index]
+
+        camera_frames = simulation.get('cameraFrames', [])
+        max_frame = len(camera_frames)
+        selected_frames = parse_frame_selection(frame_selection, max_frame)
+
+        temp_dir = tempfile.mkdtemp()
+        rrd_path = os.path.join(temp_dir, "simulation.rrd")
+
+        logger = SimulationLogger()
+        logger.log_metadata(simulation['instructions'])
+        logger.log_subjects(simulation['subjects'])
+
+        if 'helper_keyframes' in simulation:
+            logger.log_helper_keyframes(simulation['helper_keyframes'])
+
+        # Filter camera trajectory to selected frames
+        selected_camera_frames = [camera_frames[i] for i in selected_frames]
+
+        # Log the trajectory and frames only for the selected frames
+        logger.log_camera_trajectory(selected_camera_frames)
+        logger.log_camera_frames(selected_camera_frames)
+
+        rr.save(rrd_path)
+
+        return rrd_path
+
+    except Exception as e:
+        print(f"Error processing simulation: {str(e)}")
+        return None