Record3D¶
Parse and stream Record3D captures. To get demo data, see ../assets/download_assets.sh.
Features:
viser.extras
Record3D parser for RGBD dataPoint cloud visualization from depth maps
Camera pose trajectory display
Temporal playback controls with scrubbing
Note
This example requires external assets. To download them, run:
cd /path/to/viser/examples/assets
./download_assets.sh
Source: examples/04_demos/00_record3d_visualizer.py

Code¶
1import time
2from pathlib import Path
3
4import numpy as np
5import tyro
6from tqdm.auto import tqdm
7
8import viser
9import viser.extras
10import viser.transforms as tf
11
12
13def main(
14 data_path: Path = Path(__file__).parent / "../assets/record3d_dance",
15 downsample_factor: int = 4,
16 max_frames: int = 100,
17 share: bool = False,
18) -> None:
19 server = viser.ViserServer()
20 if share:
21 server.request_share_url()
22
23 print("Loading frames!")
24 loader = viser.extras.Record3dLoader(data_path)
25 num_frames = min(max_frames, loader.num_frames())
26
27 # Initial camera pose.
28 @server.on_client_connect
29 def _(client: viser.ClientHandle) -> None:
30 client.camera.position = (-1.554, -1.013, 1.142)
31 client.camera.look_at = (-0.005, 2.283, -0.156)
32
33 # Add playback UI.
34 with server.gui.add_folder("Playback"):
35 gui_point_size = server.gui.add_slider(
36 "Point size",
37 min=0.001,
38 max=0.02,
39 step=1e-3,
40 initial_value=0.01,
41 )
42 gui_timestep = server.gui.add_slider(
43 "Timestep",
44 min=0,
45 max=num_frames - 1,
46 step=1,
47 initial_value=0,
48 disabled=True,
49 )
50 gui_next_frame = server.gui.add_button("Next Frame", disabled=True)
51 gui_prev_frame = server.gui.add_button("Prev Frame", disabled=True)
52 gui_playing = server.gui.add_checkbox("Playing", True)
53 gui_framerate = server.gui.add_slider(
54 "FPS", min=1, max=60, step=0.1, initial_value=loader.fps
55 )
56 gui_framerate_options = server.gui.add_button_group(
57 "FPS options", ("10", "20", "30", "60")
58 )
59
60 # Frame step buttons.
61 @gui_next_frame.on_click
62 def _(_) -> None:
63 gui_timestep.value = (gui_timestep.value + 1) % num_frames
64
65 @gui_prev_frame.on_click
66 def _(_) -> None:
67 gui_timestep.value = (gui_timestep.value - 1) % num_frames
68
69 # Disable frame controls when we're playing.
70 @gui_playing.on_update
71 def _(_) -> None:
72 gui_timestep.disabled = gui_playing.value
73 gui_next_frame.disabled = gui_playing.value
74 gui_prev_frame.disabled = gui_playing.value
75
76 # Set the framerate when we click one of the options.
77 @gui_framerate_options.on_click
78 def _(_) -> None:
79 gui_framerate.value = int(gui_framerate_options.value)
80
81 prev_timestep = gui_timestep.value
82
83 # Toggle frame visibility when the timestep slider changes.
84 @gui_timestep.on_update
85 def _(_) -> None:
86 nonlocal prev_timestep
87 current_timestep = gui_timestep.value
88 with server.atomic():
89 # Toggle visibility.
90 frame_nodes[current_timestep].visible = True
91 frame_nodes[prev_timestep].visible = False
92 prev_timestep = current_timestep
93 server.flush() # Optional!
94
95 # Load in frames.
96 server.scene.add_frame(
97 "/frames",
98 wxyz=tf.SO3.exp(np.array([np.pi / 2.0, 0.0, 0.0])).wxyz,
99 position=(0, 0, 0),
100 show_axes=False,
101 )
102 frame_nodes: list[viser.FrameHandle] = []
103 point_nodes: list[viser.PointCloudHandle] = []
104 for i in tqdm(range(num_frames)):
105 frame = loader.get_frame(i)
106 position, color = frame.get_point_cloud(downsample_factor)
107
108 # Add base frame.
109 frame_nodes.append(server.scene.add_frame(f"/frames/t{i}", show_axes=False))
110
111 # Place the point cloud in the frame.
112 point_nodes.append(
113 server.scene.add_point_cloud(
114 name=f"/frames/t{i}/point_cloud",
115 points=position,
116 colors=color,
117 point_size=gui_point_size.value,
118 point_shape="rounded",
119 )
120 )
121
122 # Place the frustum.
123 fov = 2 * np.arctan2(frame.rgb.shape[0] / 2, frame.K[0, 0])
124 aspect = frame.rgb.shape[1] / frame.rgb.shape[0]
125 server.scene.add_camera_frustum(
126 f"/frames/t{i}/frustum",
127 fov=fov,
128 aspect=aspect,
129 scale=0.15,
130 image=frame.rgb[::downsample_factor, ::downsample_factor],
131 wxyz=tf.SO3.from_matrix(frame.T_world_camera[:3, :3]).wxyz,
132 position=frame.T_world_camera[:3, 3],
133 )
134
135 # Add some axes.
136 server.scene.add_frame(
137 f"/frames/t{i}/frustum/axes",
138 axes_length=0.05,
139 axes_radius=0.005,
140 )
141
142 # Hide all but the current frame.
143 for i, frame_node in enumerate(frame_nodes):
144 frame_node.visible = i == gui_timestep.value
145
146 # Playback update loop.
147 prev_timestep = gui_timestep.value
148 while True:
149 # Update the timestep if we're playing.
150 if gui_playing.value:
151 gui_timestep.value = (gui_timestep.value + 1) % num_frames
152
153 # Update point size of both this timestep and the next one! There's
154 # redundancy here, but this will be optimized out internally by viser.
155 #
156 # We update the point size for the next timestep so that it will be
157 # immediately available when we toggle the visibility.
158 point_nodes[gui_timestep.value].point_size = gui_point_size.value
159 point_nodes[
160 (gui_timestep.value + 1) % num_frames
161 ].point_size = gui_point_size.value
162
163 time.sleep(1.0 / gui_framerate.value)
164
165
166if __name__ == "__main__":
167 tyro.cli(main)