Scene pointer events.#
This example shows how to use scene pointer events to specify rays, and how they can be used to interact with the scene (e.g., ray-mesh intersections).
To get the demo data, see ./assets/download_dragon_mesh.sh
.
1from __future__ import annotations
2
3import time
4from pathlib import Path
5from typing import cast
6
7import numpy as onp
8import trimesh
9import trimesh.creation
10import trimesh.ray
11import viser
12import viser.transforms as tf
13
14server = viser.ViserServer()
15server.gui.configure_theme(brand_color=(130, 0, 150))
16server.scene.set_up_direction("+y")
17
18mesh = cast(
19 trimesh.Trimesh, trimesh.load_mesh(str(Path(__file__).parent / "assets/dragon.obj"))
20)
21mesh.apply_scale(0.05)
22
23mesh_handle = server.scene.add_mesh_trimesh(
24 name="/mesh",
25 mesh=mesh,
26 position=(0.0, 0.0, 0.0),
27)
28
29hit_pos_handles: list[viser.GlbHandle] = []
30
31
32# Buttons + callbacks will operate on a per-client basis, but will modify the global scene! :)
33@server.on_client_connect
34def _(client: viser.ClientHandle) -> None:
35 # Set up the camera -- this gives a nice view of the full mesh.
36 client.camera.position = onp.array([0.0, 0.0, -10.0])
37 client.camera.wxyz = onp.array([0.0, 0.0, 0.0, 1.0])
38
39 # Tests "click" scenepointerevent.
40 click_button_handle = client.gui.add_button("Add sphere", icon=viser.Icon.POINTER)
41
42 @click_button_handle.on_click
43 def _(_):
44 click_button_handle.disabled = True
45
46 @client.scene.on_pointer_event(event_type="click")
47 def _(event: viser.ScenePointerEvent) -> None:
48 # Check for intersection with the mesh, using trimesh's ray-mesh intersection.
49 # Note that mesh is in the mesh frame, so we need to transform the ray.
50 R_world_mesh = tf.SO3(mesh_handle.wxyz)
51 R_mesh_world = R_world_mesh.inverse()
52 origin = (R_mesh_world @ onp.array(event.ray_origin)).reshape(1, 3)
53 direction = (R_mesh_world @ onp.array(event.ray_direction)).reshape(1, 3)
54 intersector = trimesh.ray.ray_triangle.RayMeshIntersector(mesh)
55 hit_pos, _, _ = intersector.intersects_location(origin, direction)
56
57 if len(hit_pos) == 0:
58 return
59 client.scene.remove_pointer_callback()
60
61 # Get the first hit position (based on distance from the ray origin).
62 hit_pos = min(hit_pos, key=lambda x: onp.linalg.norm(x - origin))
63
64 # Create a sphere at the hit location.
65 hit_pos_mesh = trimesh.creation.icosphere(radius=0.1)
66 hit_pos_mesh.vertices += R_world_mesh @ hit_pos
67 hit_pos_mesh.visual.vertex_colors = (0.5, 0.0, 0.7, 1.0) # type: ignore
68 hit_pos_handle = server.scene.add_mesh_trimesh(
69 name=f"/hit_pos_{len(hit_pos_handles)}", mesh=hit_pos_mesh
70 )
71 hit_pos_handles.append(hit_pos_handle)
72
73 @client.scene.on_pointer_callback_removed
74 def _():
75 click_button_handle.disabled = False
76
77 # Tests "rect-select" scenepointerevent.
78 paint_button_handle = client.gui.add_button("Paint mesh", icon=viser.Icon.PAINT)
79
80 @paint_button_handle.on_click
81 def _(_):
82 paint_button_handle.disabled = True
83
84 @client.scene.on_pointer_event(event_type="rect-select")
85 def _(message: viser.ScenePointerEvent) -> None:
86 client.scene.remove_pointer_callback()
87
88 global mesh_handle
89 camera = message.client.camera
90
91 # Put the mesh in the camera frame.
92 R_world_mesh = tf.SO3(mesh_handle.wxyz)
93 R_mesh_world = R_world_mesh.inverse()
94 R_camera_world = tf.SE3.from_rotation_and_translation(
95 tf.SO3(camera.wxyz), camera.position
96 ).inverse()
97 vertices = cast(onp.ndarray, mesh.vertices)
98 vertices = (R_mesh_world.as_matrix() @ vertices.T).T
99 vertices = (
100 R_camera_world.as_matrix()
101 @ onp.hstack([vertices, onp.ones((vertices.shape[0], 1))]).T
102 ).T[:, :3]
103
104 # Get the camera intrinsics, and project the vertices onto the image plane.
105 fov, aspect = camera.fov, camera.aspect
106 vertices_proj = vertices[:, :2] / vertices[:, 2].reshape(-1, 1)
107 vertices_proj /= onp.tan(fov / 2)
108 vertices_proj[:, 0] /= aspect
109
110 # Move the origin to the upper-left corner, and scale to [0, 1].
111 # ... make sure to match the OpenCV's image coordinates!
112 vertices_proj = (1 + vertices_proj) / 2
113
114 # Select the vertices that lie inside the 2D selected box, once projected.
115 mask = (
116 (vertices_proj > onp.array(message.screen_pos[0]))
117 & (vertices_proj < onp.array(message.screen_pos[1]))
118 ).all(axis=1)[..., None]
119
120 # Update the mesh color based on whether the vertices are inside the box
121 mesh.visual.vertex_colors = onp.where( # type: ignore
122 mask, (0.5, 0.0, 0.7, 1.0), (0.9, 0.9, 0.9, 1.0)
123 )
124 mesh_handle = server.scene.add_mesh_trimesh(
125 name="/mesh",
126 mesh=mesh,
127 position=(0.0, 0.0, 0.0),
128 )
129
130 @client.scene.on_pointer_callback_removed
131 def _():
132 paint_button_handle.disabled = False
133
134 # Button to clear spheres.
135 clear_button_handle = client.gui.add_button("Clear scene", icon=viser.Icon.X)
136
137 @clear_button_handle.on_click
138 def _(_):
139 """Reset the mesh color and remove all click-generated spheres."""
140 global mesh_handle
141 for handle in hit_pos_handles:
142 handle.remove()
143 hit_pos_handles.clear()
144 mesh.visual.vertex_colors = (0.9, 0.9, 0.9, 1.0) # type: ignore
145 mesh_handle = server.scene.add_mesh_trimesh(
146 name="/mesh",
147 mesh=mesh,
148 position=(0.0, 0.0, 0.0),
149 )
150
151
152while True:
153 time.sleep(10.0)