Add support for "select" voice command with OpenXR (#10661)

* Add support for "select" voice command with OpenXR

* Update MicrosoftOpenXRGGVHand.cs

Author: MaxWang-MS

Assets/MRTK/Providers/OpenXR/Scripts/MicrosoftOpenXRGGVHand.cs

@@ -0,0 +1,65 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+using Microsoft.MixedReality.Toolkit.Input;
+using Microsoft.MixedReality.Toolkit.Utilities;
+using Microsoft.MixedReality.Toolkit.XRSDK.Input;
+using UnityEngine;
+
+namespace Microsoft.MixedReality.Toolkit.XRSDK.OpenXR
+{
+    /// <summary>
+    /// A GGV (Gaze, Gesture, and Voice) hand instance for OpenXR.
+    /// Used only for the purposes of acting on the select keyword detected by HoloLens 2.
+    /// </summary>
+    [MixedRealityController(
+        SupportedControllerType.GGVHand,
+        new[] { Handedness.Left, Handedness.Right, Handedness.None })]
+    internal class MicrosoftOpenXRGGVHand : GenericXRSDKController
+    {
+        public MicrosoftOpenXRGGVHand(
+            TrackingState trackingState,
+            Handedness controllerHandedness,
+            IMixedRealityInputSource inputSource = null,
+            MixedRealityInteractionMapping[] interactions = null)
+            : base(trackingState, controllerHandedness, inputSource, interactions, new SimpleHandDefinition(controllerHandedness))
+        { }
+
+        internal void UpdateVoiceState(bool isPressed)
+        {
+            MixedRealityInteractionMapping interactionMapping = null;
+
+            for (int i = 0; i < Interactions?.Length; i++)
+            {
+                MixedRealityInteractionMapping currentInteractionMapping = Interactions[i];
+
+                if (currentInteractionMapping.AxisType == AxisType.Digital && currentInteractionMapping.InputType == DeviceInputType.Select)
+                {
+                    interactionMapping = currentInteractionMapping;
+                    break;
+                }
+            }
+
+            if (interactionMapping == null)
+            {
+                return;
+            }
+
+            interactionMapping.BoolData = isPressed;
+
+            // If our value changed raise it.
+            if (interactionMapping.Changed)
+            {
+                // Raise input system event if it's enabled
+                if (interactionMapping.BoolData)
+                {
+                    CoreServices.InputSystem?.RaiseOnInputDown(InputSource, ControllerHandedness, interactionMapping.MixedRealityInputAction);
+                }
+                else
+                {
+                    CoreServices.InputSystem?.RaiseOnInputUp(InputSource, ControllerHandedness, interactionMapping.MixedRealityInputAction);
+                }
+            }
+        }
+    }
+}

Assets/MRTK/Providers/OpenXR/Scripts/MicrosoftOpenXRGGVHand.cs.meta

@@ -16,6 +16,7 @@
 #if MSFT_OPENXR && WINDOWS_UWP
 using Microsoft.MixedReality.OpenXR;
 using Microsoft.MixedReality.Toolkit.Windows.Input;
+using Windows.UI.Input.Spatial;
 #endif // MSFT_OPENXR && WINDOWS_UWP
 
 namespace Microsoft.MixedReality.Toolkit.XRSDK.OpenXR
@@ -80,6 +81,7 @@ public override void Enable()
 
 #if MSFT_OPENXR && WINDOWS_UWP
             CreateGestureRecognizers();
+            SpatialInteractionManager.SourcePressed += SpatialInteractionManager_SourcePressed;
 #endif // MSFT_OPENXR && WINDOWS_UWP
 
             base.Enable();
@@ -114,6 +116,26 @@ public override void Update()
             base.Update();
 
             CheckForGestures();
+
+            if (shouldSendVoiceEvents)
+            {
+                MicrosoftOpenXRGGVHand controller = GetOrAddVoiceController();
+                if (controller != null)
+                {
+                    // RaiseOnInputDown for "select"
+                    controller.UpdateVoiceState(true);
+                    // RaiseOnInputUp for "select"
+                    controller.UpdateVoiceState(false);
+
+                    // On WMR, the voice recognizer does not actually register the phrase 'select'
+                    // when you add it to the speech commands profile. Therefore, simulate
+                    // the "select" voice command running to ensure that we get a select voice command
+                    // registered. This is used by FocusProvider to detect when the select pointer is active.
+                    Service?.RaiseSpeechCommandRecognized(controller.InputSource, RecognitionConfidenceLevel.High, TimeSpan.MinValue, DateTime.Now, new SpeechCommands("select", KeyCode.Alpha1, MixedRealityInputAction.None));
+                }
+
+                shouldSendVoiceEvents = false;
+            }
         }
 
         /// <inheritdoc />
@@ -140,6 +162,14 @@ public override void Disable()
 #endif
             navigationGestureRecognizer = null;
 
+            SpatialInteractionManager.SourcePressed -= SpatialInteractionManager_SourcePressed;
+
+            if (voiceController != null)
+            {
+                RemoveControllerFromScene(voiceController);
+                voiceController = null;
+            }
+
             base.Disable();
         }
 #endif // MSFT_OPENXR && WINDOWS_UWP
@@ -529,5 +559,75 @@ private GenericXRSDKController FindMatchingController(GestureHandedness gestureH
 #endif // MSFT_OPENXR && WINDOWS_UWP
 
         #endregion Gesture implementation
+
+        #region SpatialInteractionManager event and helpers
+
+#if MSFT_OPENXR && WINDOWS_UWP
+        /// <summary>
+        /// SDK Interaction Source Pressed Event handler. Used only for voice.
+        /// </summary>
+        /// <param name="args">SDK source pressed event arguments</param>
+        private void SpatialInteractionManager_SourcePressed(SpatialInteractionManager sender, SpatialInteractionSourceEventArgs args)
+        {
+            if (args.State.Source.Kind == SpatialInteractionSourceKind.Voice)
+            {
+                shouldSendVoiceEvents = true;
+            }
+        }
+
+        private MicrosoftOpenXRGGVHand voiceController = null;
+        private bool shouldSendVoiceEvents = false;
+
+        private MicrosoftOpenXRGGVHand GetOrAddVoiceController()
+        {
+            if (voiceController != null)
+            {
+                return voiceController;
+            }
+
+            IMixedRealityInputSource inputSource = Service?.RequestNewGenericInputSource("Mixed Reality Voice", sourceType: InputSourceType.Voice);
+            MicrosoftOpenXRGGVHand detectedController = new MicrosoftOpenXRGGVHand(TrackingState.NotTracked, Utilities.Handedness.None, inputSource);
+
+            if (!detectedController.Enabled)
+            {
+                // Controller failed to be setup correctly.
+                // Return null so we don't raise the source detected.
+                return null;
+            }
+
+            for (int i = 0; i < detectedController.InputSource?.Pointers?.Length; i++)
+            {
+                detectedController.InputSource.Pointers[i].Controller = detectedController;
+            }
+
+            Service?.RaiseSourceDetected(detectedController.InputSource, detectedController);
+
+            voiceController = detectedController;
+            return voiceController;
+        }
+
+        private SpatialInteractionManager spatialInteractionManager = null;
+
+        /// <summary>
+        /// Provides access to the current native SpatialInteractionManager.
+        /// </summary>
+        private SpatialInteractionManager SpatialInteractionManager
+        {
+            get
+            {
+                if (spatialInteractionManager == null)
+                {
+                    UnityEngine.WSA.Application.InvokeOnUIThread(() =>
+                    {
+                        spatialInteractionManager = SpatialInteractionManager.GetForCurrentView();
+                    }, true);
+                }
+
+                return spatialInteractionManager;
+            }
+        }
+#endif // MSFT_OPENXR && WINDOWS_UWP
+
+        #endregion SpatialInteractionManager events
     }
 }