Scene pointer events

Capture mouse pointer events to create rays for 3D scene interaction and ray-mesh intersections.

Features:

  • viser.ViserServer.on_scene_pointer() for mouse ray events

  • Ray-mesh intersection calculations with trimesh

  • Dynamic mesh highlighting based on ray hits

  • Real-time pointer coordinate display

Note

This example requires external assets. To download them, run:

git clone -b v1.0.26 https://github.com/viser-project/viser.git
cd viser/examples
./assets/download_assets.sh
python 03_interaction/01_scene_pointer.py  # With viser installed.

Source: examples/03_interaction/01_scene_pointer.py

Scene pointer events

Code

  1from __future__ import annotations
  2
  3import time
  4from pathlib import Path
  5from typing import cast
  6
  7import numpy as np
  8import trimesh
  9import trimesh.creation
 10import trimesh.ray
 11
 12import viser
 13import viser.transforms as tf
 14
 15
 16def main() -> None:
 17    server = viser.ViserServer()
 18    server.gui.configure_theme(brand_color=(130, 0, 150))
 19    server.scene.set_up_direction("+y")
 20    server.initial_camera.position = (0.0, 0.0, -10.0)
 21
 22    mesh = cast(
 23        trimesh.Trimesh,
 24        trimesh.load_mesh(str(Path(__file__).parent / "../assets/dragon.obj")),
 25    )
 26    mesh.apply_scale(0.05)
 27
 28    mesh_handle = server.scene.add_mesh_trimesh(
 29        name="/mesh",
 30        mesh=mesh,
 31        position=(0.0, 0.0, 0.0),
 32    )
 33
 34    hit_pos_handles: list[viser.GlbHandle] = []
 35
 36    # Buttons + callbacks will operate on a per-client basis, but will modify the global scene! :)
 37    @server.on_client_connect
 38    def _(client: viser.ClientHandle) -> None:
 39
 40        # Tests "click" scenepointerevent.
 41        click_button_handle = client.gui.add_button(
 42            "Add sphere", icon=viser.Icon.POINTER
 43        )
 44
 45        @click_button_handle.on_click
 46        def _(_):
 47            click_button_handle.disabled = True
 48
 49            @client.scene.on_pointer_event(event_type="click")
 50            def _(event: viser.ScenePointerEvent) -> None:
 51                # Check for intersection with the mesh, using trimesh's ray-mesh intersection.
 52                # Note that mesh is in the mesh frame, so we need to transform the ray.
 53                R_world_mesh = tf.SO3(mesh_handle.wxyz)
 54                R_mesh_world = R_world_mesh.inverse()
 55                origin = (R_mesh_world @ np.array(event.ray_origin)).reshape(1, 3)
 56                direction = (R_mesh_world @ np.array(event.ray_direction)).reshape(1, 3)
 57                intersector = trimesh.ray.ray_triangle.RayMeshIntersector(mesh)
 58                hit_pos, _, _ = intersector.intersects_location(origin, direction)
 59
 60                if len(hit_pos) == 0:
 61                    return
 62                client.scene.remove_pointer_callback()
 63
 64                # Get the first hit position (based on distance from the ray origin).
 65                hit_pos = hit_pos[np.argmin(np.sum((hit_pos - origin) ** 2, axis=-1))]
 66
 67                # Create a sphere at the hit location.
 68                hit_pos_mesh = trimesh.creation.icosphere(radius=0.1)
 69                hit_pos_mesh.vertices += R_world_mesh @ hit_pos
 70                hit_pos_mesh.visual.vertex_colors = (0.5, 0.0, 0.7, 1.0)  # type: ignore
 71                hit_pos_handle = server.scene.add_mesh_trimesh(
 72                    name=f"/hit_pos_{len(hit_pos_handles)}", mesh=hit_pos_mesh
 73                )
 74                hit_pos_handles.append(hit_pos_handle)
 75
 76            @client.scene.on_pointer_callback_removed
 77            def _():
 78                click_button_handle.disabled = False
 79
 80        # Tests "rect-select" scenepointerevent.
 81        paint_button_handle = client.gui.add_button("Paint mesh", icon=viser.Icon.PAINT)
 82
 83        @paint_button_handle.on_click
 84        def _(_):
 85            paint_button_handle.disabled = True
 86
 87            @client.scene.on_pointer_event(event_type="rect-select")
 88            def _(event: viser.ScenePointerEvent) -> None:
 89                client.scene.remove_pointer_callback()
 90
 91                nonlocal mesh_handle
 92                camera = event.client.camera
 93
 94                # Put the mesh in the camera frame.
 95                R_world_mesh = tf.SO3(mesh_handle.wxyz)
 96                R_mesh_world = R_world_mesh.inverse()
 97                R_camera_world = tf.SE3.from_rotation_and_translation(
 98                    tf.SO3(camera.wxyz), camera.position
 99                ).inverse()
100                vertices = cast(np.ndarray, mesh.vertices)
101                vertices = (R_mesh_world.as_matrix() @ vertices.T).T
102                vertices = (
103                    R_camera_world.as_matrix()
104                    @ np.hstack([vertices, np.ones((vertices.shape[0], 1))]).T
105                ).T[:, :3]
106
107                # Get the camera intrinsics, and project the vertices onto the image plane.
108                fov, aspect = camera.fov, camera.aspect
109                vertices_proj = vertices[:, :2] / vertices[:, 2].reshape(-1, 1)
110                vertices_proj /= np.tan(fov / 2)
111                vertices_proj[:, 0] /= aspect
112
113                # Move the origin to the upper-left corner, and scale to [0, 1].
114                # ... make sure to match the OpenCV's image coordinates!
115                vertices_proj = (1 + vertices_proj) / 2
116
117                # Select the vertices that lie inside the 2D selected box, once projected.
118                mask = (
119                    (vertices_proj > np.array(event.screen_pos[0]))
120                    & (vertices_proj < np.array(event.screen_pos[1]))
121                ).all(axis=1)[..., None]
122
123                # Update the mesh color based on whether the vertices are inside the box
124                mesh.visual.vertex_colors = np.where(  # type: ignore
125                    mask, (0.5, 0.0, 0.7, 1.0), (0.9, 0.9, 0.9, 1.0)
126                )
127                mesh_handle = server.scene.add_mesh_trimesh(
128                    name="/mesh",
129                    mesh=mesh,
130                    position=(0.0, 0.0, 0.0),
131                )
132
133            @client.scene.on_pointer_callback_removed
134            def _():
135                paint_button_handle.disabled = False
136
137        # Button to clear spheres.
138        clear_button_handle = client.gui.add_button("Clear scene", icon=viser.Icon.X)
139
140        @clear_button_handle.on_click
141        def _(_):
142            nonlocal mesh_handle
143            for handle in hit_pos_handles:
144                handle.remove()
145            hit_pos_handles.clear()
146            mesh.visual.vertex_colors = (0.9, 0.9, 0.9, 1.0)  # type: ignore
147            mesh_handle = server.scene.add_mesh_trimesh(
148                name="/mesh",
149                mesh=mesh,
150                position=(0.0, 0.0, 0.0),
151            )
152
153    while True:
154        time.sleep(10.0)
155
156
157if __name__ == "__main__":
158    main()