Record3D

Parse and stream Record3D captures. To get demo data, see ../assets/download_assets.sh.

Features:

  • viser.extras Record3D parser for RGBD data

  • Point cloud visualization from depth maps

  • Camera pose trajectory display

  • Temporal playback controls with scrubbing

Note

This example requires external assets. To download them, run:

cd /path/to/viser/examples/assets
./download_assets.sh

Source: examples/04_demos/00_record3d_visualizer.py

Record3D

Code

  1import time
  2from pathlib import Path
  3
  4import numpy as np
  5import tyro
  6from tqdm.auto import tqdm
  7
  8import viser
  9import viser.extras
 10import viser.transforms as tf
 11
 12
 13def main(
 14    data_path: Path = Path(__file__).parent / "../assets/record3d_dance",
 15    downsample_factor: int = 4,
 16    max_frames: int = 100,
 17    share: bool = False,
 18) -> None:
 19    server = viser.ViserServer()
 20    if share:
 21        server.request_share_url()
 22
 23    print("Loading frames!")
 24    loader = viser.extras.Record3dLoader(data_path)
 25    num_frames = min(max_frames, loader.num_frames())
 26
 27    # Initial camera pose.
 28    @server.on_client_connect
 29    def _(client: viser.ClientHandle) -> None:
 30        client.camera.position = (-1.554, -1.013, 1.142)
 31        client.camera.look_at = (-0.005, 2.283, -0.156)
 32
 33    # Add playback UI.
 34    with server.gui.add_folder("Playback"):
 35        gui_point_size = server.gui.add_slider(
 36            "Point size",
 37            min=0.001,
 38            max=0.02,
 39            step=1e-3,
 40            initial_value=0.01,
 41        )
 42        gui_timestep = server.gui.add_slider(
 43            "Timestep",
 44            min=0,
 45            max=num_frames - 1,
 46            step=1,
 47            initial_value=0,
 48            disabled=True,
 49        )
 50        gui_next_frame = server.gui.add_button("Next Frame", disabled=True)
 51        gui_prev_frame = server.gui.add_button("Prev Frame", disabled=True)
 52        gui_playing = server.gui.add_checkbox("Playing", True)
 53        gui_framerate = server.gui.add_slider(
 54            "FPS", min=1, max=60, step=0.1, initial_value=loader.fps
 55        )
 56        gui_framerate_options = server.gui.add_button_group(
 57            "FPS options", ("10", "20", "30", "60")
 58        )
 59
 60    # Frame step buttons.
 61    @gui_next_frame.on_click
 62    def _(_) -> None:
 63        gui_timestep.value = (gui_timestep.value + 1) % num_frames
 64
 65    @gui_prev_frame.on_click
 66    def _(_) -> None:
 67        gui_timestep.value = (gui_timestep.value - 1) % num_frames
 68
 69    # Disable frame controls when we're playing.
 70    @gui_playing.on_update
 71    def _(_) -> None:
 72        gui_timestep.disabled = gui_playing.value
 73        gui_next_frame.disabled = gui_playing.value
 74        gui_prev_frame.disabled = gui_playing.value
 75
 76    # Set the framerate when we click one of the options.
 77    @gui_framerate_options.on_click
 78    def _(_) -> None:
 79        gui_framerate.value = int(gui_framerate_options.value)
 80
 81    prev_timestep = gui_timestep.value
 82
 83    # Toggle frame visibility when the timestep slider changes.
 84    @gui_timestep.on_update
 85    def _(_) -> None:
 86        nonlocal prev_timestep
 87        current_timestep = gui_timestep.value
 88        with server.atomic():
 89            # Toggle visibility.
 90            frame_nodes[current_timestep].visible = True
 91            frame_nodes[prev_timestep].visible = False
 92        prev_timestep = current_timestep
 93        server.flush()  # Optional!
 94
 95    # Load in frames.
 96    server.scene.add_frame(
 97        "/frames",
 98        wxyz=tf.SO3.exp(np.array([np.pi / 2.0, 0.0, 0.0])).wxyz,
 99        position=(0, 0, 0),
100        show_axes=False,
101    )
102    frame_nodes: list[viser.FrameHandle] = []
103    point_nodes: list[viser.PointCloudHandle] = []
104    for i in tqdm(range(num_frames)):
105        frame = loader.get_frame(i)
106        position, color = frame.get_point_cloud(downsample_factor)
107
108        # Add base frame.
109        frame_nodes.append(server.scene.add_frame(f"/frames/t{i}", show_axes=False))
110
111        # Place the point cloud in the frame.
112        point_nodes.append(
113            server.scene.add_point_cloud(
114                name=f"/frames/t{i}/point_cloud",
115                points=position,
116                colors=color,
117                point_size=gui_point_size.value,
118                point_shape="rounded",
119            )
120        )
121
122        # Place the frustum.
123        fov = 2 * np.arctan2(frame.rgb.shape[0] / 2, frame.K[0, 0])
124        aspect = frame.rgb.shape[1] / frame.rgb.shape[0]
125        server.scene.add_camera_frustum(
126            f"/frames/t{i}/frustum",
127            fov=fov,
128            aspect=aspect,
129            scale=0.15,
130            image=frame.rgb[::downsample_factor, ::downsample_factor],
131            wxyz=tf.SO3.from_matrix(frame.T_world_camera[:3, :3]).wxyz,
132            position=frame.T_world_camera[:3, 3],
133        )
134
135        # Add some axes.
136        server.scene.add_frame(
137            f"/frames/t{i}/frustum/axes",
138            axes_length=0.05,
139            axes_radius=0.005,
140        )
141
142    # Hide all but the current frame.
143    for i, frame_node in enumerate(frame_nodes):
144        frame_node.visible = i == gui_timestep.value
145
146    # Playback update loop.
147    prev_timestep = gui_timestep.value
148    while True:
149        # Update the timestep if we're playing.
150        if gui_playing.value:
151            gui_timestep.value = (gui_timestep.value + 1) % num_frames
152
153        # Update point size of both this timestep and the next one! There's
154        # redundancy here, but this will be optimized out internally by viser.
155        #
156        # We update the point size for the next timestep so that it will be
157        # immediately available when we toggle the visibility.
158        point_nodes[gui_timestep.value].point_size = gui_point_size.value
159        point_nodes[
160            (gui_timestep.value + 1) % num_frames
161        ].point_size = gui_point_size.value
162
163        time.sleep(1.0 / gui_framerate.value)
164
165
166if __name__ == "__main__":
167    tyro.cli(main)