Scene pointer events.#

This example shows how to use scene pointer events to specify rays, and how they can be used to interact with the scene (e.g., ray-mesh intersections).

To get the demo data, see ./assets/download_dragon_mesh.sh.

  1from __future__ import annotations
  2
  3import time
  4from pathlib import Path
  5from typing import cast
  6
  7import numpy as onp
  8import trimesh
  9import trimesh.creation
 10import trimesh.ray
 11import viser
 12import viser.transforms as tf
 13from viser.theme import TitlebarConfig
 14
 15server = viser.ViserServer()
 16server.gui.configure_theme(
 17    brand_color=(130, 0, 150),
 18    titlebar_content=TitlebarConfig(buttons=(), image=None),
 19)
 20server.scene.set_up_direction("+y")
 21
 22mesh = cast(
 23    trimesh.Trimesh, trimesh.load_mesh(str(Path(__file__).parent / "assets/dragon.obj"))
 24)
 25mesh.apply_scale(0.05)
 26
 27mesh_handle = server.scene.add_mesh_trimesh(
 28    name="/mesh",
 29    mesh=mesh,
 30    position=(0.0, 0.0, 0.0),
 31)
 32
 33hit_pos_handles: list[viser.GlbHandle] = []
 34
 35
 36# Buttons + callbacks will operate on a per-client basis, but will modify the global scene! :)
 37@server.on_client_connect
 38def _(client: viser.ClientHandle) -> None:
 39    # Set up the camera -- this gives a nice view of the full mesh.
 40    client.camera.position = onp.array([0.0, 0.0, -10.0])
 41    client.camera.wxyz = onp.array([0.0, 0.0, 0.0, 1.0])
 42
 43    # Tests "click" scenepointerevent.
 44    click_button_handle = client.gui.add_button("Add sphere", icon=viser.Icon.POINTER)
 45
 46    @click_button_handle.on_click
 47    def _(_):
 48        click_button_handle.disabled = True
 49
 50        @client.scene.on_pointer_event(event_type="click")
 51        def _(event: viser.ScenePointerEvent) -> None:
 52            # Check for intersection with the mesh, using trimesh's ray-mesh intersection.
 53            # Note that mesh is in the mesh frame, so we need to transform the ray.
 54            R_world_mesh = tf.SO3(mesh_handle.wxyz)
 55            R_mesh_world = R_world_mesh.inverse()
 56            origin = (R_mesh_world @ onp.array(event.ray_origin)).reshape(1, 3)
 57            direction = (R_mesh_world @ onp.array(event.ray_direction)).reshape(1, 3)
 58            intersector = trimesh.ray.ray_triangle.RayMeshIntersector(mesh)
 59            hit_pos, _, _ = intersector.intersects_location(origin, direction)
 60
 61            if len(hit_pos) == 0:
 62                return
 63            client.scene.remove_pointer_callback()
 64
 65            # Get the first hit position (based on distance from the ray origin).
 66            hit_pos = min(hit_pos, key=lambda x: onp.linalg.norm(x - origin))
 67
 68            # Create a sphere at the hit location.
 69            hit_pos_mesh = trimesh.creation.icosphere(radius=0.1)
 70            hit_pos_mesh.vertices += R_world_mesh @ hit_pos
 71            hit_pos_mesh.visual.vertex_colors = (0.5, 0.0, 0.7, 1.0)  # type: ignore
 72            hit_pos_handle = server.scene.add_mesh_trimesh(
 73                name=f"/hit_pos_{len(hit_pos_handles)}", mesh=hit_pos_mesh
 74            )
 75            hit_pos_handles.append(hit_pos_handle)
 76
 77        @client.scene.on_pointer_callback_removed
 78        def _():
 79            click_button_handle.disabled = False
 80
 81    # Tests "rect-select" scenepointerevent.
 82    paint_button_handle = client.gui.add_button("Paint mesh", icon=viser.Icon.PAINT)
 83
 84    @paint_button_handle.on_click
 85    def _(_):
 86        paint_button_handle.disabled = True
 87
 88        @client.scene.on_pointer_event(event_type="rect-select")
 89        def _(message: viser.ScenePointerEvent) -> None:
 90            client.scene.remove_pointer_callback()
 91
 92            global mesh_handle
 93            camera = message.client.camera
 94
 95            # Put the mesh in the camera frame.
 96            R_world_mesh = tf.SO3(mesh_handle.wxyz)
 97            R_mesh_world = R_world_mesh.inverse()
 98            R_camera_world = tf.SE3.from_rotation_and_translation(
 99                tf.SO3(camera.wxyz), camera.position
100            ).inverse()
101            vertices = cast(onp.ndarray, mesh.vertices)
102            vertices = (R_mesh_world.as_matrix() @ vertices.T).T
103            vertices = (
104                R_camera_world.as_matrix()
105                @ onp.hstack([vertices, onp.ones((vertices.shape[0], 1))]).T
106            ).T[:, :3]
107
108            # Get the camera intrinsics, and project the vertices onto the image plane.
109            fov, aspect = camera.fov, camera.aspect
110            vertices_proj = vertices[:, :2] / vertices[:, 2].reshape(-1, 1)
111            vertices_proj /= onp.tan(fov / 2)
112            vertices_proj[:, 0] /= aspect
113
114            # Move the origin to the upper-left corner, and scale to [0, 1].
115            # ... make sure to match the OpenCV's image coordinates!
116            vertices_proj = (1 + vertices_proj) / 2
117
118            # Select the vertices that lie inside the 2D selected box, once projected.
119            mask = (
120                (vertices_proj > onp.array(message.screen_pos[0]))
121                & (vertices_proj < onp.array(message.screen_pos[1]))
122            ).all(axis=1)[..., None]
123
124            # Update the mesh color based on whether the vertices are inside the box
125            mesh.visual.vertex_colors = onp.where(  # type: ignore
126                mask, (0.5, 0.0, 0.7, 1.0), (0.9, 0.9, 0.9, 1.0)
127            )
128            mesh_handle = server.scene.add_mesh_trimesh(
129                name="/mesh",
130                mesh=mesh,
131                position=(0.0, 0.0, 0.0),
132            )
133
134        @client.scene.on_pointer_callback_removed
135        def _():
136            paint_button_handle.disabled = False
137
138    # Button to clear spheres.
139    clear_button_handle = client.gui.add_button("Clear scene", icon=viser.Icon.X)
140
141    @clear_button_handle.on_click
142    def _(_):
143        """Reset the mesh color and remove all click-generated spheres."""
144        global mesh_handle
145        for handle in hit_pos_handles:
146            handle.remove()
147        hit_pos_handles.clear()
148        mesh.visual.vertex_colors = (0.9, 0.9, 0.9, 1.0)  # type: ignore
149        mesh_handle = server.scene.add_mesh_trimesh(
150            name="/mesh",
151            mesh=mesh,
152            position=(0.0, 0.0, 0.0),
153        )
154
155
156while True:
157    time.sleep(10.0)