Browse Source

Add hand laser pointer, passthrough, gaze balls, G1 models, and recenter support

- Hand tracking laser pointer with pinch-to-click gesture for menu interaction
- Meta Quest passthrough (alpha blend + XR_FB_PASSTHROUGH extension)
- Stationary webcam quad (moved from XRCamera3D to XROrigin3D)
- Gaze-activated exit balls (4 balls: exit AR, 2 reserved, quit app)
- Gaze laser beam and ball growth visual feedback
- Spinning G1 3D model flanking the start screen menu
- Recenter support via pose_recentered signal
- Exit AR mode returns to CONFIG phase with UI repositioning

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
master
melancholytron 1 month ago
parent
commit
baffbb7739
  1. 336
      Main.gd
  2. 2
      Main.tscn
  3. 1237690
      models/g1_full.obj
  4. 3
      project.godot
  5. 164
      scripts/vr_ui_pointer.gd

336
Main.gd

@ -10,7 +10,7 @@ enum Phase { CONFIG, AR }
@onready var teleop_client: Node = $TeleopClient @onready var teleop_client: Node = $TeleopClient
@onready var xr_origin: XROrigin3D = $XROrigin3D @onready var xr_origin: XROrigin3D = $XROrigin3D
@onready var xr_camera: XRCamera3D = $XROrigin3D/XRCamera3D @onready var xr_camera: XRCamera3D = $XROrigin3D/XRCamera3D
@onready var webcam_quad: MeshInstance3D = $XROrigin3D/XRCamera3D/WebcamQuad
@onready var webcam_quad: MeshInstance3D = $XROrigin3D/WebcamQuad
@onready var start_screen: Node3D = $XROrigin3D/StartScreen @onready var start_screen: Node3D = $XROrigin3D/StartScreen
@onready var left_controller: XRController3D = $XROrigin3D/LeftController @onready var left_controller: XRController3D = $XROrigin3D/LeftController
@onready var right_controller: XRController3D = $XROrigin3D/RightController @onready var right_controller: XRController3D = $XROrigin3D/RightController
@ -21,6 +21,35 @@ var xr_is_focused: bool = false
var current_phase: Phase = Phase.CONFIG var current_phase: Phase = Phase.CONFIG
var _panel_positioned: bool = false var _panel_positioned: bool = false
# Spinning G1 models flanking the start screen
var _g1_model_left: Node3D
var _g1_model_right: Node3D
var _config_light: DirectionalLight3D
const G1_SPIN_SPEED := 0.5 # radians per second
# Gaze balls: [exit_ar, slot_1, slot_2, quit_app] — left to right
var _gaze_balls: Array = [] # Array of MeshInstance3D
var _gaze_mats: Array = [] # Array of StandardMaterial3D
var _gaze_timers: Array = [0.0, 0.0, 0.0, 0.0]
var _gaze_laser: MeshInstance3D
const GAZE_BALL_COUNT := 4
const GAZE_ACTIVATE_TIME := 5.0
const GAZE_ANGLE_THRESHOLD := 8.0 # degrees
const GAZE_BALL_BASE_SCALE := 1.0
const GAZE_BALL_MAX_SCALE := 1.8
const GAZE_BALL_COLORS: Array = [
Color(1.0, 0.2, 0.2, 0.35), # 0: red — exit AR
Color(0.8, 0.6, 0.2, 0.35), # 1: yellow — reserved
Color(0.2, 0.8, 0.4, 0.35), # 2: green — reserved
Color(0.2, 0.3, 1.0, 0.35), # 3: blue — quit app
]
const GAZE_BALL_BASE_COLORS: Array = [
Color(1.0, 0.2, 0.2),
Color(0.8, 0.6, 0.2),
Color(0.2, 0.8, 0.4),
Color(0.2, 0.3, 1.0),
]
func _ready() -> void: func _ready() -> void:
# Hide webcam quad and start screen until positioned # Hide webcam quad and start screen until positioned
@ -31,11 +60,13 @@ func _ready() -> void:
xr_interface = XRServer.find_interface("OpenXR") xr_interface = XRServer.find_interface("OpenXR")
if xr_interface and xr_interface.is_initialized(): if xr_interface and xr_interface.is_initialized():
print("[Main] OpenXR already initialized") print("[Main] OpenXR already initialized")
xr_interface.connect("pose_recentered", _on_pose_recentered)
_on_openxr_ready() _on_openxr_ready()
elif xr_interface: elif xr_interface:
xr_interface.connect("session_begun", _on_openxr_session_begun) xr_interface.connect("session_begun", _on_openxr_session_begun)
xr_interface.connect("session_focussed", _on_openxr_focused) xr_interface.connect("session_focussed", _on_openxr_focused)
xr_interface.connect("session_stopping", _on_openxr_stopping) xr_interface.connect("session_stopping", _on_openxr_stopping)
xr_interface.connect("pose_recentered", _on_pose_recentered)
if not xr_interface.initialize(): if not xr_interface.initialize():
printerr("[Main] Failed to initialize OpenXR") printerr("[Main] Failed to initialize OpenXR")
get_tree().quit() get_tree().quit()
@ -49,6 +80,11 @@ func _ready() -> void:
# Enable XR on the viewport # Enable XR on the viewport
get_viewport().use_xr = true get_viewport().use_xr = true
# CONFIG phase: keep background opaque so passthrough is hidden
# (blend mode is alpha_blend from project settings, but we render opaque black)
get_viewport().transparent_bg = false
RenderingServer.set_default_clear_color(Color(0, 0, 0, 1))
# Connect start screen signals # Connect start screen signals
start_screen.connect_requested.connect(_on_connect_requested) start_screen.connect_requested.connect(_on_connect_requested)
start_screen.launch_ar_requested.connect(_on_launch_ar_requested) start_screen.launch_ar_requested.connect(_on_launch_ar_requested)
@ -68,7 +104,7 @@ func _ready() -> void:
print("[Main] Starting in CONFIG phase") print("[Main] Starting in CONFIG phase")
func _process(_delta: float) -> void:
func _process(delta: float) -> void:
if not _panel_positioned and current_phase == Phase.CONFIG: if not _panel_positioned and current_phase == Phase.CONFIG:
# Wait until we have valid head tracking data to position the panel # Wait until we have valid head tracking data to position the panel
var hmd := XRServer.get_hmd_transform() var hmd := XRServer.get_hmd_transform()
@ -76,6 +112,16 @@ func _process(_delta: float) -> void:
_position_panel_in_front_of_user(hmd) _position_panel_in_front_of_user(hmd)
_panel_positioned = true _panel_positioned = true
# Spin G1 models in CONFIG phase
if current_phase == Phase.CONFIG:
if _g1_model_left and _g1_model_left.visible:
_g1_model_left.rotate_y(G1_SPIN_SPEED * delta)
if _g1_model_right and _g1_model_right.visible:
_g1_model_right.rotate_y(G1_SPIN_SPEED * delta)
if current_phase == Phase.AR:
_update_gaze_balls(delta)
func _position_panel_in_front_of_user(hmd: Transform3D) -> void: func _position_panel_in_front_of_user(hmd: Transform3D) -> void:
# Place the panel 1.2m in front of the user at their eye height # Place the panel 1.2m in front of the user at their eye height
@ -96,6 +142,59 @@ func _position_panel_in_front_of_user(hmd: Transform3D) -> void:
start_screen.visible = true start_screen.visible = true
print("[Main] Panel positioned at %s (user at %s)" % [panel_pos, hmd.origin]) print("[Main] Panel positioned at %s (user at %s)" % [panel_pos, hmd.origin])
# Position spinning G1 models on each side of the panel
var right := forward.cross(Vector3.UP).normalized()
_setup_g1_models(panel_pos, right, hmd.origin)
func _setup_g1_models(panel_pos: Vector3, right: Vector3, user_pos: Vector3) -> void:
var g1_scene = load("res://models/g1_full.obj")
if g1_scene == null:
printerr("[Main] Failed to load G1 model")
return
if _g1_model_left == null:
_g1_model_left = _create_g1_instance(g1_scene)
add_child(_g1_model_left)
if _g1_model_right == null:
_g1_model_right = _create_g1_instance(g1_scene)
add_child(_g1_model_right)
# Place 0.7m to each side of the panel, slightly below eye height
_g1_model_left.global_position = panel_pos - right * 0.7 + Vector3.DOWN * 0.3
_g1_model_right.global_position = panel_pos + right * 0.7 + Vector3.DOWN * 0.3
# Reset rotation to just the upright correction, then face user
_g1_model_left.rotation = Vector3(-PI / 2.0, 0, 0)
_g1_model_right.rotation = Vector3(-PI / 2.0, 0, 0)
_g1_model_left.visible = true
_g1_model_right.visible = true
# Add a directional light for shading the models
if _config_light == null:
_config_light = DirectionalLight3D.new()
_config_light.light_energy = 1.0
_config_light.rotation_degrees = Vector3(-45, 30, 0)
add_child(_config_light)
_config_light.visible = true
print("[Main] G1 models positioned")
func _create_g1_instance(mesh_resource: Mesh) -> MeshInstance3D:
var inst := MeshInstance3D.new()
inst.mesh = mesh_resource
# Scale down to ~0.3m tall (adjust if needed)
inst.scale = Vector3(0.0015, 0.0015, 0.0015)
# Rotate 90 degrees around X to stand the model upright (OBJ has Z-up)
inst.rotate_x(-PI / 2.0)
inst.visible = false
# Add a shaded material so the model is visible with lighting
var mat := StandardMaterial3D.new()
mat.albedo_color = Color(0.4, 0.6, 0.9, 1.0) # Light blue
inst.material_override = mat
return inst
func _on_openxr_session_begun() -> void: func _on_openxr_session_begun() -> void:
print("[Main] OpenXR session begun") print("[Main] OpenXR session begun")
@ -114,6 +213,16 @@ func _on_openxr_focused() -> void:
print("[Main] OpenXR session focused") print("[Main] OpenXR session focused")
func _on_pose_recentered() -> void:
print("[Main] Pose recentered — repositioning UI")
var hmd := XRServer.get_hmd_transform()
if current_phase == Phase.CONFIG:
_position_panel_in_front_of_user(hmd)
elif current_phase == Phase.AR:
_position_quad_in_front_of_user(webcam_quad, hmd, 1.2, -0.3)
_create_gaze_balls(hmd)
func _on_openxr_stopping() -> void: func _on_openxr_stopping() -> void:
xr_is_focused = false xr_is_focused = false
print("[Main] OpenXR session stopping") print("[Main] OpenXR session stopping")
@ -145,23 +254,226 @@ func _on_launch_ar_requested() -> void:
# Enable passthrough # Enable passthrough
_enable_passthrough() _enable_passthrough()
# Wire body tracker to teleop client
# Wire body tracker to teleop client (only connect once)
if not body_tracker.tracking_data_ready.is_connected(teleop_client._on_tracking_data):
body_tracker.tracking_data_ready.connect(teleop_client._on_tracking_data) body_tracker.tracking_data_ready.connect(teleop_client._on_tracking_data)
# Show webcam quad, hide start screen
# Position webcam quad in front of user (stationary)
var hmd := XRServer.get_hmd_transform()
_position_quad_in_front_of_user(webcam_quad, hmd, 1.2, -0.3)
webcam_quad.visible = true webcam_quad.visible = true
# Hide start screen and G1 models
start_screen.hide_screen() start_screen.hide_screen()
if _g1_model_left:
_g1_model_left.visible = false
if _g1_model_right:
_g1_model_right.visible = false
if _config_light:
_config_light.visible = false
# Create and position gaze exit balls
_create_gaze_balls(hmd)
func _enable_passthrough() -> void: func _enable_passthrough() -> void:
var openxr = xr_interface as OpenXRInterface
if openxr:
var modes = openxr.get_supported_environment_blend_modes()
print("[Main] Supported blend modes: ", modes)
if XRInterface.XR_ENV_BLEND_MODE_ALPHA_BLEND in modes:
openxr.set_environment_blend_mode(XRInterface.XR_ENV_BLEND_MODE_ALPHA_BLEND)
print("[Main] Passthrough enabled (alpha blend)")
# Enable Meta Quest passthrough via XR_FB_PASSTHROUGH extension.
# Requires openxr/extensions/meta/passthrough=true in project settings
# and meta_xr_features/passthrough=2 in export presets.
# Make viewport transparent so passthrough shows through
get_viewport().transparent_bg = true get_viewport().transparent_bg = true
RenderingServer.set_default_clear_color(Color(0, 0, 0, 0)) RenderingServer.set_default_clear_color(Color(0, 0, 0, 0))
# Set blend mode to alpha blend — the Meta vendor plugin intercepts this
# and activates FB passthrough even though it's not in supported modes list.
var openxr = xr_interface as OpenXRInterface
if openxr:
openxr.environment_blend_mode = XRInterface.XR_ENV_BLEND_MODE_ALPHA_BLEND
print("[Main] Passthrough enabled (alpha blend + transparent bg)")
func _disable_passthrough() -> void:
get_viewport().transparent_bg = false
RenderingServer.set_default_clear_color(Color(0, 0, 0, 1))
var openxr = xr_interface as OpenXRInterface
if openxr:
openxr.environment_blend_mode = XRInterface.XR_ENV_BLEND_MODE_OPAQUE
print("[Main] Passthrough disabled")
func _position_quad_in_front_of_user(quad: Node3D, hmd: Transform3D, distance: float, y_offset: float) -> void:
var forward := -hmd.basis.z
forward.y = 0
if forward.length() < 0.01:
forward = Vector3(0, 0, -1)
forward = forward.normalized()
var pos := hmd.origin + forward * distance
pos.y = hmd.origin.y + y_offset
quad.global_position = pos
quad.look_at(hmd.origin, Vector3.UP)
quad.rotate_y(PI)
func _create_gaze_balls(hmd: Transform3D) -> void:
var forward := -hmd.basis.z
forward.y = 0
if forward.length() < 0.01:
forward = Vector3(0, 0, -1)
forward = forward.normalized()
var right := forward.cross(Vector3.UP).normalized()
var base_pos := hmd.origin + forward * 1.0 + Vector3.UP * 0.35
# Create balls if first time, spread evenly left to right
if _gaze_balls.is_empty():
for i in range(GAZE_BALL_COUNT):
var ball := _make_gaze_sphere(GAZE_BALL_COLORS[i])
_gaze_balls.append(ball)
_gaze_mats.append(ball.material_override)
add_child(ball)
# Position: spread from -0.45 to +0.45 across the row
var spread := 0.3 # spacing between balls
var half_width := spread * (GAZE_BALL_COUNT - 1) / 2.0
for i in range(GAZE_BALL_COUNT):
var offset := -half_width + spread * i
_gaze_balls[i].global_position = base_pos + right * offset
_gaze_balls[i].visible = true
_gaze_balls[i].scale = Vector3.ONE
_gaze_timers[i] = 0.0
# Gaze laser beam
if _gaze_laser == null:
_gaze_laser = MeshInstance3D.new()
var cyl := CylinderMesh.new()
cyl.top_radius = 0.002
cyl.bottom_radius = 0.002
cyl.height = 1.0
_gaze_laser.mesh = cyl
var lmat := StandardMaterial3D.new()
lmat.shading_mode = BaseMaterial3D.SHADING_MODE_UNSHADED
lmat.albedo_color = Color(1.0, 1.0, 1.0, 0.4)
lmat.transparency = BaseMaterial3D.TRANSPARENCY_ALPHA
lmat.no_depth_test = true
_gaze_laser.material_override = lmat
add_child(_gaze_laser)
_gaze_laser.visible = false
func _make_gaze_sphere(color: Color) -> MeshInstance3D:
var mesh := MeshInstance3D.new()
var sphere := SphereMesh.new()
sphere.radius = 0.035
sphere.height = 0.07
mesh.mesh = sphere
var mat := StandardMaterial3D.new()
mat.shading_mode = BaseMaterial3D.SHADING_MODE_UNSHADED
mat.albedo_color = color
mat.transparency = BaseMaterial3D.TRANSPARENCY_ALPHA
mat.no_depth_test = true
mesh.material_override = mat
return mesh
func _update_gaze_balls(delta: float) -> void:
if _gaze_balls.is_empty() or not _gaze_balls[0].visible:
return
var hmd := XRServer.get_hmd_transform()
if hmd.origin == Vector3.ZERO:
return
var gaze_dir := (-hmd.basis.z).normalized()
var threshold := deg_to_rad(GAZE_ANGLE_THRESHOLD)
var gazing_at_any := false
for i in range(GAZE_BALL_COUNT):
var ball: MeshInstance3D = _gaze_balls[i]
var to_ball: Vector3 = (ball.global_position - hmd.origin).normalized()
var looking := gaze_dir.angle_to(to_ball) < threshold
if looking:
_gaze_timers[i] += delta
if not gazing_at_any:
gazing_at_any = true
_show_gaze_laser(hmd.origin, ball.global_position)
else: else:
print("[Main] Alpha blend not supported, using opaque mode")
_gaze_timers[i] = 0.0
_update_gaze_visual(_gaze_mats[i] as StandardMaterial3D, GAZE_BALL_BASE_COLORS[i] as Color, _gaze_timers[i], ball)
if _gaze_timers[i] >= GAZE_ACTIVATE_TIME:
_on_gaze_activated(i)
return
if not gazing_at_any and _gaze_laser:
_gaze_laser.visible = false
func _on_gaze_activated(index: int) -> void:
match index:
0: # Exit AR
print("[Main] Gaze exit: returning to CONFIG")
_exit_ar_mode()
3: # Quit app
print("[Main] Gaze exit: quitting app")
get_tree().quit()
_: # Reserved slots — no action yet
print("[Main] Gaze ball %d activated (no action assigned)" % index)
_gaze_timers[index] = 0.0
func _update_gaze_visual(mat: StandardMaterial3D, base_color: Color, timer: float, ball: MeshInstance3D) -> void:
var progress := clampf(timer / GAZE_ACTIVATE_TIME, 0.0, 1.0)
# Opacity goes from 0.35 to 1.0 as gaze progresses
var alpha := lerpf(0.35, 1.0, progress)
# Color brightens toward white at the end
var color := base_color.lerp(Color(1.0, 1.0, 1.0), progress * 0.5)
color.a = alpha
mat.albedo_color = color
# Scale ball up when being gazed at
var s := lerpf(GAZE_BALL_BASE_SCALE, GAZE_BALL_MAX_SCALE, progress)
ball.scale = Vector3(s, s, s)
func _show_gaze_laser(from: Vector3, to: Vector3) -> void:
if _gaze_laser == null:
return
var dir := to - from
var length := dir.length()
var cyl := _gaze_laser.mesh as CylinderMesh
if cyl:
cyl.height = length
# Position at midpoint, orient Y-axis along the ray
_gaze_laser.global_position = from + dir * 0.5
var up := dir.normalized()
var arbitrary := Vector3.UP if abs(up.dot(Vector3.UP)) < 0.99 else Vector3.RIGHT
var right := up.cross(arbitrary).normalized()
var forward := right.cross(up).normalized()
_gaze_laser.global_transform.basis = Basis(right, up, forward)
_gaze_laser.visible = true
func _exit_ar_mode() -> void:
current_phase = Phase.CONFIG
# Disable passthrough
_disable_passthrough()
# Hide gaze balls, laser, and webcam
for ball in _gaze_balls:
ball.visible = false
ball.scale = Vector3.ONE
for i in range(_gaze_timers.size()):
_gaze_timers[i] = 0.0
if _gaze_laser:
_gaze_laser.visible = false
webcam_quad.visible = false
# Show start screen again, reposition in front of user
var hmd := XRServer.get_hmd_transform()
_position_panel_in_front_of_user(hmd)
print("[Main] Returned to CONFIG phase")

2
Main.tscn

@ -14,7 +14,7 @@ script = ExtResource("1")
[node name="XRCamera3D" type="XRCamera3D" parent="XROrigin3D"] [node name="XRCamera3D" type="XRCamera3D" parent="XROrigin3D"]
[node name="WebcamQuad" parent="XROrigin3D/XRCamera3D" instance=ExtResource("5")]
[node name="WebcamQuad" parent="XROrigin3D" instance=ExtResource("5")]
transform = Transform3D(1, 0, 0, 0, 1, 0, 0, 0, 1, 0, -0.3, -1.5) transform = Transform3D(1, 0, 0, 0, 1, 0, 0, 0, 1, 0, -0.3, -1.5)
[node name="StartScreen" parent="XROrigin3D" instance=ExtResource("6")] [node name="StartScreen" parent="XROrigin3D" instance=ExtResource("6")]

1237690
models/g1_full.obj
File diff suppressed because it is too large
View File

3
project.godot

@ -39,5 +39,6 @@ openxr/extensions/hand_tracking=true
openxr/extensions/hand_tracking_unobstructed_data_source=true openxr/extensions/hand_tracking_unobstructed_data_source=true
openxr/extensions/hand_tracking_controller_data_source=true openxr/extensions/hand_tracking_controller_data_source=true
openxr/extensions/hand_interaction_profile=true openxr/extensions/hand_interaction_profile=true
openxr/extensions/meta/body_tracking=true
shaders/enabled=true shaders/enabled=true
openxr/extensions/meta/passthrough=true
openxr/extensions/meta/body_tracking=true

164
scripts/vr_ui_pointer.gd

@ -7,6 +7,9 @@ extends Node3D
@export var poke_threshold: float = 0.03 @export var poke_threshold: float = 0.03
@export var hover_distance: float = 0.15 @export var hover_distance: float = 0.15
@export var laser_color: Color = Color(0.3, 0.6, 1.0, 0.6) @export var laser_color: Color = Color(0.3, 0.6, 1.0, 0.6)
@export var pinch_press_threshold: float = 0.025
@export var pinch_release_threshold: float = 0.035
@export var hand_laser_color: Color = Color(0.8, 0.4, 1.0, 0.5)
var _xr_origin: XROrigin3D var _xr_origin: XROrigin3D
var _camera: XRCamera3D var _camera: XRCamera3D
@ -28,6 +31,11 @@ var _is_pressing: bool = false
var _active_method: String = "" var _active_method: String = ""
var _log_timer: float = 0.0 var _log_timer: float = 0.0
# Hand laser state
var _hand_laser_left: MeshInstance3D
var _hand_laser_right: MeshInstance3D
var _pinch_pressed: Array = [false, false]
const JOINT_COUNT := 26 const JOINT_COUNT := 26
# Fingertip joint indices for XRHandTracker (thumb=5, index=10, middle=15, ring=20, pinky=25) # Fingertip joint indices for XRHandTracker (thumb=5, index=10, middle=15, ring=20, pinky=25)
const TIP_JOINTS := [5, 10, 15, 20, 25] const TIP_JOINTS := [5, 10, 15, 20, 25]
@ -41,6 +49,8 @@ const BODY_HAND_JOINT_COUNT := 25
const BODY_TIP_JOINTS := [4, 9, 14, 19, 24] const BODY_TIP_JOINTS := [4, 9, 14, 19, 24]
# Index finger tip within body tracker hand block (for poke interaction) # Index finger tip within body tracker hand block (for poke interaction)
const BODY_INDEX_TIP := 9 const BODY_INDEX_TIP := 9
# Thumb tip within body tracker hand block (for pinch detection)
const BODY_THUMB_TIP := 4
func setup(xr_origin: XROrigin3D, camera: XRCamera3D, left_ctrl: XRController3D, right_ctrl: XRController3D) -> void: func setup(xr_origin: XROrigin3D, camera: XRCamera3D, left_ctrl: XRController3D, right_ctrl: XRController3D) -> void:
@ -78,6 +88,14 @@ func setup(xr_origin: XROrigin3D, camera: XRCamera3D, left_ctrl: XRController3D,
_reticle.visible = false _reticle.visible = false
add_child(_reticle) add_child(_reticle)
# Hand laser beams (positioned each frame from hand tracking)
_hand_laser_right = _create_hand_laser()
_hand_laser_right.visible = false
add_child(_hand_laser_right)
_hand_laser_left = _create_hand_laser()
_hand_laser_left.visible = false
add_child(_hand_laser_left)
# Hand joint spheres # Hand joint spheres
_create_hand_visuals() _create_hand_visuals()
@ -105,6 +123,21 @@ func _create_laser() -> MeshInstance3D:
return laser return laser
func _create_hand_laser() -> MeshInstance3D:
var laser := MeshInstance3D.new()
var cyl := CylinderMesh.new()
cyl.top_radius = 0.002
cyl.bottom_radius = 0.002
cyl.height = ray_length
laser.mesh = cyl
var mat := StandardMaterial3D.new()
mat.shading_mode = BaseMaterial3D.SHADING_MODE_UNSHADED
mat.albedo_color = hand_laser_color
mat.transparency = BaseMaterial3D.TRANSPARENCY_ALPHA
laser.material_override = mat
return laser
func _create_hand_visuals() -> void: func _create_hand_visuals() -> void:
# Shared meshes - larger for visibility in VR # Shared meshes - larger for visibility in VR
var joint_mesh := SphereMesh.new() var joint_mesh := SphereMesh.new()
@ -219,11 +252,67 @@ func _process(delta: float) -> void:
var panels := _get_ui_panels() var panels := _get_ui_panels()
if panels.is_empty(): if panels.is_empty():
_reticle.visible = false _reticle.visible = false
_hand_laser_left.visible = false
_hand_laser_right.visible = false
return return
var hit := false var hit := false
var controllers_active := l_active or r_active
# Hide hand lasers by default each frame
_hand_laser_left.visible = false
_hand_laser_right.visible = false
# 1. Hand laser ray (highest priority, only when no controllers)
if not controllers_active:
for hand_idx in [1, 0]: # Right hand first
var hand_ray := _get_hand_ray(hand_idx)
if hand_ray.is_empty():
continue
var ray_origin: Vector3 = hand_ray[0]
var ray_dir: Vector3 = hand_ray[1]
var laser: MeshInstance3D = _hand_laser_right if hand_idx == 1 else _hand_laser_left
var method := "hand_laser_%d" % hand_idx
# Show laser beam at full length
laser.visible = true
_update_hand_laser_visual(laser, ray_origin, ray_dir, ray_length)
for panel in panels:
var result := _ray_intersect_panel(ray_origin, ray_dir, panel[0], panel[1])
if result.size() > 0:
hit = true
_reticle.global_position = result[2]
_reticle.visible = true
_current_viewport = panel[1]
_last_viewport_pos = result[1]
_send_mouse_motion(panel[1], result[1])
# Shorten laser to hit point
var dist := ray_origin.distance_to(result[2])
_update_hand_laser_visual(laser, ray_origin, ray_dir, dist)
# Pinch-to-click with hysteresis
var pinch_dist := _get_pinch_distance(hand_idx)
if pinch_dist >= 0.0:
if not _pinch_pressed[hand_idx] and pinch_dist < pinch_press_threshold:
_pinch_pressed[hand_idx] = true
if not _is_pressing:
_is_pressing = true
_active_method = method
_send_mouse_button(panel[1], result[1], true)
elif _pinch_pressed[hand_idx] and pinch_dist > pinch_release_threshold:
_pinch_pressed[hand_idx] = false
if _is_pressing and _active_method == method:
_is_pressing = false
_send_mouse_button(panel[1], result[1], false)
break
if hit:
break
# 1. Hand tracking poke (priority over controller ray)
# 2. Hand tracking poke
if not hit:
for hand_idx in [0, 1]: for hand_idx in [0, 1]:
var tip_pos := _get_fingertip_world_position(hand_idx) var tip_pos := _get_fingertip_world_position(hand_idx)
if tip_pos == Vector3.ZERO: if tip_pos == Vector3.ZERO:
@ -250,7 +339,7 @@ func _process(delta: float) -> void:
if hit: if hit:
break break
# 2. Controller ray pointing
# 3. Controller ray pointing
if not hit: if not hit:
for ctrl_data in [[_right_ctrl, _laser_right, "ray_right"], [_left_ctrl, _laser_left, "ray_left"]]: for ctrl_data in [[_right_ctrl, _laser_right, "ray_right"], [_left_ctrl, _laser_left, "ray_left"]]:
var ctrl: XRController3D = ctrl_data[0] var ctrl: XRController3D = ctrl_data[0]
@ -282,6 +371,7 @@ func _process(delta: float) -> void:
_is_pressing = false _is_pressing = false
if _current_viewport: if _current_viewport:
_send_mouse_button(_current_viewport, _last_viewport_pos, false) _send_mouse_button(_current_viewport, _last_viewport_pos, false)
_pinch_pressed = [false, false]
_current_viewport = null _current_viewport = null
@ -368,6 +458,76 @@ func _get_fingertip_world_position(hand: int) -> Vector3:
return Vector3.ZERO return Vector3.ZERO
func _get_hand_ray(hand: int) -> Array:
var origin_xform := _xr_origin.global_transform
# Try XRHandTracker first
var tracker_name := &"/user/hand_tracker/left" if hand == 0 else &"/user/hand_tracker/right"
var tracker = XRServer.get_tracker(tracker_name)
if tracker:
var hand_tracker := tracker as XRHandTracker
if hand_tracker and hand_tracker.has_tracking_data:
var wrist := hand_tracker.get_hand_joint_transform(XRHandTracker.HAND_JOINT_WRIST)
var middle_meta := hand_tracker.get_hand_joint_transform(XRHandTracker.HAND_JOINT_MIDDLE_FINGER_METACARPAL)
if wrist.origin != Vector3.ZERO and middle_meta.origin != Vector3.ZERO:
var ray_origin := origin_xform * wrist.origin
var ray_target := origin_xform * middle_meta.origin
var ray_dir := (ray_target - ray_origin).normalized()
return [ray_origin, ray_dir]
# Fallback: XRBodyTracker (wrist=offset 1, middle metacarpal=offset 10)
var body_tracker = XRServer.get_tracker(&"/user/body_tracker") as XRBodyTracker
if body_tracker and body_tracker.get_has_tracking_data():
var start_idx := BODY_LEFT_HAND_START if hand == 0 else BODY_RIGHT_HAND_START
var wrist := body_tracker.get_joint_transform(start_idx + 1)
var middle_meta := body_tracker.get_joint_transform(start_idx + 10)
if wrist.origin != Vector3.ZERO and middle_meta.origin != Vector3.ZERO:
var ray_origin := origin_xform * wrist.origin
var ray_target := origin_xform * middle_meta.origin
var ray_dir := (ray_target - ray_origin).normalized()
return [ray_origin, ray_dir]
return []
func _get_pinch_distance(hand: int) -> float:
# Try XRHandTracker first
var tracker_name := &"/user/hand_tracker/left" if hand == 0 else &"/user/hand_tracker/right"
var tracker = XRServer.get_tracker(tracker_name)
if tracker:
var hand_tracker := tracker as XRHandTracker
if hand_tracker and hand_tracker.has_tracking_data:
var thumb := hand_tracker.get_hand_joint_transform(XRHandTracker.HAND_JOINT_THUMB_TIP)
var index := hand_tracker.get_hand_joint_transform(XRHandTracker.HAND_JOINT_INDEX_FINGER_TIP)
if thumb.origin != Vector3.ZERO and index.origin != Vector3.ZERO:
return thumb.origin.distance_to(index.origin)
# Fallback: XRBodyTracker
var body_tracker = XRServer.get_tracker(&"/user/body_tracker") as XRBodyTracker
if body_tracker and body_tracker.get_has_tracking_data():
var start_idx := BODY_LEFT_HAND_START if hand == 0 else BODY_RIGHT_HAND_START
var thumb := body_tracker.get_joint_transform(start_idx + BODY_THUMB_TIP)
var index := body_tracker.get_joint_transform(start_idx + BODY_INDEX_TIP)
if thumb.origin != Vector3.ZERO and index.origin != Vector3.ZERO:
return thumb.origin.distance_to(index.origin)
return -1.0
func _update_hand_laser_visual(laser: MeshInstance3D, ray_origin: Vector3, ray_dir: Vector3, length: float) -> void:
var cyl := laser.mesh as CylinderMesh
if cyl:
cyl.height = length
var midpoint := ray_origin + ray_dir * (length / 2.0)
laser.global_position = midpoint
# Align cylinder Y-axis with ray direction
var up := ray_dir
var arbitrary := Vector3.UP if abs(ray_dir.dot(Vector3.UP)) < 0.99 else Vector3.RIGHT
var right := up.cross(arbitrary).normalized()
var forward := right.cross(up).normalized()
laser.global_transform.basis = Basis(right, up, forward)
func _get_ui_panels() -> Array: func _get_ui_panels() -> Array:
var results := [] var results := []
for screen in get_tree().get_nodes_in_group("start_screen"): for screen in get_tree().get_nodes_in_group("start_screen"):

Loading…
Cancel
Save