Record3D visualizerΒΆ

Parse and stream record3d captures. To get the demo data, see ./assets/download_record3d_dance.sh.

  1import time
  2from pathlib import Path
  3
  4import numpy as np
  5import tyro
  6from tqdm.auto import tqdm
  7
  8import viser
  9import viser.extras
 10import viser.transforms as tf
 11
 12
 13def main(
 14    data_path: Path = Path(__file__).parent / "assets/record3d_dance",
 15    downsample_factor: int = 4,
 16    max_frames: int = 100,
 17    share: bool = False,
 18) -> None:
 19    server = viser.ViserServer()
 20    if share:
 21        server.request_share_url()
 22
 23    print("Loading frames!")
 24    loader = viser.extras.Record3dLoader(data_path)
 25    num_frames = min(max_frames, loader.num_frames())
 26
 27    # Add playback UI.
 28    with server.gui.add_folder("Playback"):
 29        gui_point_size = server.gui.add_slider(
 30            "Point size",
 31            min=0.001,
 32            max=0.02,
 33            step=1e-3,
 34            initial_value=0.01,
 35        )
 36        gui_timestep = server.gui.add_slider(
 37            "Timestep",
 38            min=0,
 39            max=num_frames - 1,
 40            step=1,
 41            initial_value=0,
 42            disabled=True,
 43        )
 44        gui_next_frame = server.gui.add_button("Next Frame", disabled=True)
 45        gui_prev_frame = server.gui.add_button("Prev Frame", disabled=True)
 46        gui_playing = server.gui.add_checkbox("Playing", True)
 47        gui_framerate = server.gui.add_slider(
 48            "FPS", min=1, max=60, step=0.1, initial_value=loader.fps
 49        )
 50        gui_framerate_options = server.gui.add_button_group(
 51            "FPS options", ("10", "20", "30", "60")
 52        )
 53
 54    # Frame step buttons.
 55    @gui_next_frame.on_click
 56    def _(_) -> None:
 57        gui_timestep.value = (gui_timestep.value + 1) % num_frames
 58
 59    @gui_prev_frame.on_click
 60    def _(_) -> None:
 61        gui_timestep.value = (gui_timestep.value - 1) % num_frames
 62
 63    # Disable frame controls when we're playing.
 64    @gui_playing.on_update
 65    def _(_) -> None:
 66        gui_timestep.disabled = gui_playing.value
 67        gui_next_frame.disabled = gui_playing.value
 68        gui_prev_frame.disabled = gui_playing.value
 69
 70    # Set the framerate when we click one of the options.
 71    @gui_framerate_options.on_click
 72    def _(_) -> None:
 73        gui_framerate.value = int(gui_framerate_options.value)
 74
 75    prev_timestep = gui_timestep.value
 76
 77    # Toggle frame visibility when the timestep slider changes.
 78    @gui_timestep.on_update
 79    def _(_) -> None:
 80        nonlocal prev_timestep
 81        current_timestep = gui_timestep.value
 82        with server.atomic():
 83            # Toggle visibility.
 84            frame_nodes[current_timestep].visible = True
 85            frame_nodes[prev_timestep].visible = False
 86        prev_timestep = current_timestep
 87        server.flush()  # Optional!
 88
 89    # Load in frames.
 90    server.scene.add_frame(
 91        "/frames",
 92        wxyz=tf.SO3.exp(np.array([np.pi / 2.0, 0.0, 0.0])).wxyz,
 93        position=(0, 0, 0),
 94        show_axes=False,
 95    )
 96    frame_nodes: list[viser.FrameHandle] = []
 97    point_nodes: list[viser.PointCloudHandle] = []
 98    for i in tqdm(range(num_frames)):
 99        frame = loader.get_frame(i)
100        position, color = frame.get_point_cloud(downsample_factor)
101
102        # Add base frame.
103        frame_nodes.append(server.scene.add_frame(f"/frames/t{i}", show_axes=False))
104
105        # Place the point cloud in the frame.
106        point_nodes.append(
107            server.scene.add_point_cloud(
108                name=f"/frames/t{i}/point_cloud",
109                points=position,
110                colors=color,
111                point_size=gui_point_size.value,
112                point_shape="rounded",
113            )
114        )
115
116        # Place the frustum.
117        fov = 2 * np.arctan2(frame.rgb.shape[0] / 2, frame.K[0, 0])
118        aspect = frame.rgb.shape[1] / frame.rgb.shape[0]
119        server.scene.add_camera_frustum(
120            f"/frames/t{i}/frustum",
121            fov=fov,
122            aspect=aspect,
123            scale=0.15,
124            image=frame.rgb[::downsample_factor, ::downsample_factor],
125            wxyz=tf.SO3.from_matrix(frame.T_world_camera[:3, :3]).wxyz,
126            position=frame.T_world_camera[:3, 3],
127        )
128
129        # Add some axes.
130        server.scene.add_frame(
131            f"/frames/t{i}/frustum/axes",
132            axes_length=0.05,
133            axes_radius=0.005,
134        )
135
136    # Hide all but the current frame.
137    for i, frame_node in enumerate(frame_nodes):
138        frame_node.visible = i == gui_timestep.value
139
140    # Playback update loop.
141    prev_timestep = gui_timestep.value
142    while True:
143        # Update the timestep if we're playing.
144        if gui_playing.value:
145            gui_timestep.value = (gui_timestep.value + 1) % num_frames
146
147        # Update point size of both this timestep and the next one! There's
148        # redundancy here, but this will be optimized out internally by viser.
149        #
150        # We update the point size for the next timestep so that it will be
151        # immediately available when we toggle the visibility.
152        point_nodes[gui_timestep.value].point_size = gui_point_size.value
153        point_nodes[
154            (gui_timestep.value + 1) % num_frames
155        ].point_size = gui_point_size.value
156
157        time.sleep(1.0 / gui_framerate.value)
158
159
160if __name__ == "__main__":
161    tyro.cli(main)