Scene pointer events¶
Capture mouse pointer events to create rays for 3D scene interaction and ray-mesh intersections.
Features:
viser.ViserServer.on_scene_pointer()for mouse ray eventsRay-mesh intersection calculations with trimesh
Dynamic mesh highlighting based on ray hits
Real-time pointer coordinate display
Note
This example requires external assets. To download them, run:
cd /path/to/viser/examples/assets
./download_assets.sh
Source: examples/03_interaction/01_scene_pointer.py
Code¶
1from __future__ import annotations
2
3import time
4from pathlib import Path
5from typing import cast
6
7import numpy as np
8import trimesh
9import trimesh.creation
10import trimesh.ray
11
12import viser
13import viser.transforms as tf
14
15
16def main() -> None:
17 server = viser.ViserServer()
18 server.gui.configure_theme(brand_color=(130, 0, 150))
19 server.scene.set_up_direction("+y")
20
21 mesh = cast(
22 trimesh.Trimesh,
23 trimesh.load_mesh(str(Path(__file__).parent / "../assets/dragon.obj")),
24 )
25 mesh.apply_scale(0.05)
26
27 mesh_handle = server.scene.add_mesh_trimesh(
28 name="/mesh",
29 mesh=mesh,
30 position=(0.0, 0.0, 0.0),
31 )
32
33 hit_pos_handles: list[viser.GlbHandle] = []
34
35 # Buttons + callbacks will operate on a per-client basis, but will modify the global scene! :)
36 @server.on_client_connect
37 def _(client: viser.ClientHandle) -> None:
38 # Set up the camera -- this gives a nice view of the full mesh.
39 client.camera.position = np.array([0.0, 0.0, -10.0])
40 client.camera.wxyz = np.array([0.0, 0.0, 0.0, 1.0])
41
42 # Tests "click" scenepointerevent.
43 click_button_handle = client.gui.add_button(
44 "Add sphere", icon=viser.Icon.POINTER
45 )
46
47 @click_button_handle.on_click
48 def _(_):
49 click_button_handle.disabled = True
50
51 @client.scene.on_pointer_event(event_type="click")
52 def _(event: viser.ScenePointerEvent) -> None:
53 # Check for intersection with the mesh, using trimesh's ray-mesh intersection.
54 # Note that mesh is in the mesh frame, so we need to transform the ray.
55 R_world_mesh = tf.SO3(mesh_handle.wxyz)
56 R_mesh_world = R_world_mesh.inverse()
57 origin = (R_mesh_world @ np.array(event.ray_origin)).reshape(1, 3)
58 direction = (R_mesh_world @ np.array(event.ray_direction)).reshape(1, 3)
59 intersector = trimesh.ray.ray_triangle.RayMeshIntersector(mesh)
60 hit_pos, _, _ = intersector.intersects_location(origin, direction)
61
62 if len(hit_pos) == 0:
63 return
64 client.scene.remove_pointer_callback()
65
66 # Get the first hit position (based on distance from the ray origin).
67 hit_pos = hit_pos[np.argmin(np.sum((hit_pos - origin) ** 2, axis=-1))]
68
69 # Create a sphere at the hit location.
70 hit_pos_mesh = trimesh.creation.icosphere(radius=0.1)
71 hit_pos_mesh.vertices += R_world_mesh @ hit_pos
72 hit_pos_mesh.visual.vertex_colors = (0.5, 0.0, 0.7, 1.0) # type: ignore
73 hit_pos_handle = server.scene.add_mesh_trimesh(
74 name=f"/hit_pos_{len(hit_pos_handles)}", mesh=hit_pos_mesh
75 )
76 hit_pos_handles.append(hit_pos_handle)
77
78 @client.scene.on_pointer_callback_removed
79 def _():
80 click_button_handle.disabled = False
81
82 # Tests "rect-select" scenepointerevent.
83 paint_button_handle = client.gui.add_button("Paint mesh", icon=viser.Icon.PAINT)
84
85 @paint_button_handle.on_click
86 def _(_):
87 paint_button_handle.disabled = True
88
89 @client.scene.on_pointer_event(event_type="rect-select")
90 def _(event: viser.ScenePointerEvent) -> None:
91 client.scene.remove_pointer_callback()
92
93 nonlocal mesh_handle
94 camera = event.client.camera
95
96 # Put the mesh in the camera frame.
97 R_world_mesh = tf.SO3(mesh_handle.wxyz)
98 R_mesh_world = R_world_mesh.inverse()
99 R_camera_world = tf.SE3.from_rotation_and_translation(
100 tf.SO3(camera.wxyz), camera.position
101 ).inverse()
102 vertices = cast(np.ndarray, mesh.vertices)
103 vertices = (R_mesh_world.as_matrix() @ vertices.T).T
104 vertices = (
105 R_camera_world.as_matrix()
106 @ np.hstack([vertices, np.ones((vertices.shape[0], 1))]).T
107 ).T[:, :3]
108
109 # Get the camera intrinsics, and project the vertices onto the image plane.
110 fov, aspect = camera.fov, camera.aspect
111 vertices_proj = vertices[:, :2] / vertices[:, 2].reshape(-1, 1)
112 vertices_proj /= np.tan(fov / 2)
113 vertices_proj[:, 0] /= aspect
114
115 # Move the origin to the upper-left corner, and scale to [0, 1].
116 # ... make sure to match the OpenCV's image coordinates!
117 vertices_proj = (1 + vertices_proj) / 2
118
119 # Select the vertices that lie inside the 2D selected box, once projected.
120 mask = (
121 (vertices_proj > np.array(event.screen_pos[0]))
122 & (vertices_proj < np.array(event.screen_pos[1]))
123 ).all(axis=1)[..., None]
124
125 # Update the mesh color based on whether the vertices are inside the box
126 mesh.visual.vertex_colors = np.where( # type: ignore
127 mask, (0.5, 0.0, 0.7, 1.0), (0.9, 0.9, 0.9, 1.0)
128 )
129 mesh_handle = server.scene.add_mesh_trimesh(
130 name="/mesh",
131 mesh=mesh,
132 position=(0.0, 0.0, 0.0),
133 )
134
135 @client.scene.on_pointer_callback_removed
136 def _():
137 paint_button_handle.disabled = False
138
139 # Button to clear spheres.
140 clear_button_handle = client.gui.add_button("Clear scene", icon=viser.Icon.X)
141
142 @clear_button_handle.on_click
143 def _(_):
144 nonlocal mesh_handle
145 for handle in hit_pos_handles:
146 handle.remove()
147 hit_pos_handles.clear()
148 mesh.visual.vertex_colors = (0.9, 0.9, 0.9, 1.0) # type: ignore
149 mesh_handle = server.scene.add_mesh_trimesh(
150 name="/mesh",
151 mesh=mesh,
152 position=(0.0, 0.0, 0.0),
153 )
154
155 while True:
156 time.sleep(10.0)
157
158
159if __name__ == "__main__":
160 main()