Scene pointer events.#

This example shows how to use scene pointer events to specify rays, and how they can be used to interact with the scene (e.g., ray-mesh intersections).

To get the demo data, see ./assets/download_dragon_mesh.sh.

  1from __future__ import annotations
  2
  3import time
  4from pathlib import Path
  5from typing import cast
  6
  7import numpy as onp
  8import trimesh
  9import trimesh.creation
 10import trimesh.ray
 11import viser
 12import viser.transforms as tf
 13
 14server = viser.ViserServer()
 15server.gui.configure_theme(brand_color=(130, 0, 150))
 16server.scene.set_up_direction("+y")
 17
 18mesh = cast(
 19    trimesh.Trimesh, trimesh.load_mesh(str(Path(__file__).parent / "assets/dragon.obj"))
 20)
 21mesh.apply_scale(0.05)
 22
 23mesh_handle = server.scene.add_mesh_trimesh(
 24    name="/mesh",
 25    mesh=mesh,
 26    position=(0.0, 0.0, 0.0),
 27)
 28
 29hit_pos_handles: list[viser.GlbHandle] = []
 30
 31
 32# Buttons + callbacks will operate on a per-client basis, but will modify the global scene! :)
 33@server.on_client_connect
 34def _(client: viser.ClientHandle) -> None:
 35    # Set up the camera -- this gives a nice view of the full mesh.
 36    client.camera.position = onp.array([0.0, 0.0, -10.0])
 37    client.camera.wxyz = onp.array([0.0, 0.0, 0.0, 1.0])
 38
 39    # Tests "click" scenepointerevent.
 40    click_button_handle = client.gui.add_button("Add sphere", icon=viser.Icon.POINTER)
 41
 42    @click_button_handle.on_click
 43    def _(_):
 44        click_button_handle.disabled = True
 45
 46        @client.scene.on_pointer_event(event_type="click")
 47        def _(event: viser.ScenePointerEvent) -> None:
 48            # Check for intersection with the mesh, using trimesh's ray-mesh intersection.
 49            # Note that mesh is in the mesh frame, so we need to transform the ray.
 50            R_world_mesh = tf.SO3(mesh_handle.wxyz)
 51            R_mesh_world = R_world_mesh.inverse()
 52            origin = (R_mesh_world @ onp.array(event.ray_origin)).reshape(1, 3)
 53            direction = (R_mesh_world @ onp.array(event.ray_direction)).reshape(1, 3)
 54            intersector = trimesh.ray.ray_triangle.RayMeshIntersector(mesh)
 55            hit_pos, _, _ = intersector.intersects_location(origin, direction)
 56
 57            if len(hit_pos) == 0:
 58                return
 59            client.scene.remove_pointer_callback()
 60
 61            # Get the first hit position (based on distance from the ray origin).
 62            hit_pos = min(hit_pos, key=lambda x: onp.linalg.norm(x - origin))
 63
 64            # Create a sphere at the hit location.
 65            hit_pos_mesh = trimesh.creation.icosphere(radius=0.1)
 66            hit_pos_mesh.vertices += R_world_mesh @ hit_pos
 67            hit_pos_mesh.visual.vertex_colors = (0.5, 0.0, 0.7, 1.0)  # type: ignore
 68            hit_pos_handle = server.scene.add_mesh_trimesh(
 69                name=f"/hit_pos_{len(hit_pos_handles)}", mesh=hit_pos_mesh
 70            )
 71            hit_pos_handles.append(hit_pos_handle)
 72
 73        @client.scene.on_pointer_callback_removed
 74        def _():
 75            click_button_handle.disabled = False
 76
 77    # Tests "rect-select" scenepointerevent.
 78    paint_button_handle = client.gui.add_button("Paint mesh", icon=viser.Icon.PAINT)
 79
 80    @paint_button_handle.on_click
 81    def _(_):
 82        paint_button_handle.disabled = True
 83
 84        @client.scene.on_pointer_event(event_type="rect-select")
 85        def _(message: viser.ScenePointerEvent) -> None:
 86            client.scene.remove_pointer_callback()
 87
 88            global mesh_handle
 89            camera = message.client.camera
 90
 91            # Put the mesh in the camera frame.
 92            R_world_mesh = tf.SO3(mesh_handle.wxyz)
 93            R_mesh_world = R_world_mesh.inverse()
 94            R_camera_world = tf.SE3.from_rotation_and_translation(
 95                tf.SO3(camera.wxyz), camera.position
 96            ).inverse()
 97            vertices = cast(onp.ndarray, mesh.vertices)
 98            vertices = (R_mesh_world.as_matrix() @ vertices.T).T
 99            vertices = (
100                R_camera_world.as_matrix()
101                @ onp.hstack([vertices, onp.ones((vertices.shape[0], 1))]).T
102            ).T[:, :3]
103
104            # Get the camera intrinsics, and project the vertices onto the image plane.
105            fov, aspect = camera.fov, camera.aspect
106            vertices_proj = vertices[:, :2] / vertices[:, 2].reshape(-1, 1)
107            vertices_proj /= onp.tan(fov / 2)
108            vertices_proj[:, 0] /= aspect
109
110            # Move the origin to the upper-left corner, and scale to [0, 1].
111            # ... make sure to match the OpenCV's image coordinates!
112            vertices_proj = (1 + vertices_proj) / 2
113
114            # Select the vertices that lie inside the 2D selected box, once projected.
115            mask = (
116                (vertices_proj > onp.array(message.screen_pos[0]))
117                & (vertices_proj < onp.array(message.screen_pos[1]))
118            ).all(axis=1)[..., None]
119
120            # Update the mesh color based on whether the vertices are inside the box
121            mesh.visual.vertex_colors = onp.where(  # type: ignore
122                mask, (0.5, 0.0, 0.7, 1.0), (0.9, 0.9, 0.9, 1.0)
123            )
124            mesh_handle = server.scene.add_mesh_trimesh(
125                name="/mesh",
126                mesh=mesh,
127                position=(0.0, 0.0, 0.0),
128            )
129
130        @client.scene.on_pointer_callback_removed
131        def _():
132            paint_button_handle.disabled = False
133
134    # Button to clear spheres.
135    clear_button_handle = client.gui.add_button("Clear scene", icon=viser.Icon.X)
136
137    @clear_button_handle.on_click
138    def _(_):
139        """Reset the mesh color and remove all click-generated spheres."""
140        global mesh_handle
141        for handle in hit_pos_handles:
142            handle.remove()
143        hit_pos_handles.clear()
144        mesh.visual.vertex_colors = (0.9, 0.9, 0.9, 1.0)  # type: ignore
145        mesh_handle = server.scene.add_mesh_trimesh(
146            name="/mesh",
147            mesh=mesh,
148            position=(0.0, 0.0, 0.0),
149        )
150
151
152while True:
153    time.sleep(10.0)