// ----------------------------------------------------------------------- // // Copyright (c) Microsoft Corporation. All rights reserved. // // ----------------------------------------------------------------------- namespace Microsoft.Samples.Kinect.Webserver.Sensor { using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.Linq; using System.Net; using System.Text.RegularExpressions; using System.Threading.Tasks; using System.Windows; using System.Windows.Media; using Microsoft.Kinect; using Microsoft.Kinect.Toolkit.Interaction; using Microsoft.Samples.Kinect.Webserver.Sensor.Serialization; /// /// Implementation of ISensorStreamHandler that exposes interaction and user viewer /// streams. /// [SuppressMessage("Microsoft.Design", "CA1001:TypesThatOwnDisposableFieldsShouldBeDisposable", Justification = "Disposable interaction stream is disposed when sensor is set to null")] public class InteractionStreamHandler : SensorStreamHandlerBase, IInteractionClient { /// /// JSON name of interaction stream. /// internal const string InteractionStreamName = "interaction"; /// /// JSON name for property representing primary user tracking ID. /// internal const string InteractionPrimaryUserPropertyName = "primaryUser"; /// /// JSON name for property representing user states. /// internal const string InteractionUserStatesPropertyName = "userStates"; /// /// JSON name of user viewer stream. /// internal const string UserViewerStreamName = "userviewer"; /// /// JSON name for property representing user viewer image resolution. /// internal const string UserViewerResolutionPropertyName = "resolution"; /// /// Default width for user viewer image. /// internal const int UserViewerDefaultWidth = 128; /// /// Default height for user viewer image. /// internal const int UserViewerDefaultHeight = 96; /// /// JSON name for property representing default color for users in user viewer image. /// internal const string UserViewerDefaultUserColorPropertyName = "defaultUserColor"; /// /// JSON name for property representing a map between user states and colors that should /// be used to represent those states in user viewer image. /// internal const string UserViewerUserColorsPropertyName = "userColors"; /// /// Sub path for interaction client web-socket RPC endpoint owned by this handler. /// internal const string ClientUriSubpath = "CLIENT"; /// /// Default value for default color for users in user viewer image (light gray). /// internal static readonly Color UserViewerDefaultDefaultUserColor = new Color { R = 0xd3, G = 0xd3, B = 0xd3, A = 0xff }; /// /// Default color for tracked users in user viewer image (Kinect blue). /// internal static readonly Color UserViewerDefaultTrackedUserColor = new Color { R = 0x00, G = 0xbc, B = 0xf2, A = 0xff }; /// /// Default color for engaged users in user viewer image (Kinect purple). /// internal static readonly Color UserViewerDefaultEngagedUserColor = new Color { R = 0x51, G = 0x1c, B = 0x74, A = 0xff }; /// /// Regular expression that matches the user viewer resolution property. /// private static readonly Regex UserViewerResolutionRegex = new Regex(@"^(?i)(\d+)x(\d+)$"); private static readonly Size[] UserViewerSupportedResolutions = { new Size(640, 480), new Size(320, 240), new Size(160, 120), new Size(128, 96), new Size(80, 60) }; /// /// Context that allows this stream handler to communicate with its owner. /// private readonly SensorStreamHandlerContext ownerContext; /// /// Serializable interaction stream message, reused as interaction frames arrive. /// private readonly InteractionStreamMessage interactionStreamMessage = new InteractionStreamMessage { stream = InteractionStreamName }; /// /// Serializable user viewer stream message header, reused as depth frames arrive /// and are colorized into the user viewer image. /// private readonly ImageHeaderStreamMessage userViewerStreamMessage = new ImageHeaderStreamMessage { stream = UserViewerStreamName }; /// /// Ids of users we choose to track. /// private readonly int[] recommendedUserTrackingIds = new int[2]; /// /// A map between user state names and colors that should be used to represent those /// states in user viewer image. /// private readonly Dictionary userViewerUserColors = new Dictionary(); /// /// Width of user viewer image. /// private readonly UserViewerColorizer userViewerColorizer = new UserViewerColorizer(UserViewerDefaultWidth, UserViewerDefaultHeight); /// /// User state manager. /// private readonly IUserStateManager userStateManager = new DefaultUserStateManager(); /// /// Sensor providing data to interaction stream. /// private KinectSensor sensor; /// /// Entry point for interaction stream functionality. /// private InteractionStream interactionStream; /// /// Intermediate storage for the user information received from interaction stream. /// private UserInfo[] userInfos; /// /// true if interaction stream is enabled. /// private bool interactionIsEnabled; /// /// true if user viewer stream is enabled. /// private bool userViewerIsEnabled; /// /// Default color for users in user viewer image, in 32-bit RGBA format. /// private int userViewerDefaultUserColor; /// /// Keep track if we're in the middle of processing an interaction frame. /// private bool isProcessingInteractionFrame; /// /// Keep track if we're in the middle of processing a user viewer image. /// private bool isProcessingUserViewerImage; /// /// Channel used to perform remote procedure calls regarding IInteractionClient state. /// private WebSocketRpcChannel clientRpcChannel; /// /// Initializes a new instance of the class /// and associates it with a context that allows it to communicate with its owner. /// /// /// An instance of class. /// internal InteractionStreamHandler(SensorStreamHandlerContext ownerContext) { this.userViewerDefaultUserColor = GetRgbaColorInt(UserViewerDefaultDefaultUserColor); this.userViewerUserColors[DefaultUserStateManager.TrackedStateName] = GetRgbaColorInt(UserViewerDefaultTrackedUserColor); this.userViewerUserColors[DefaultUserStateManager.EngagedStateName] = GetRgbaColorInt(UserViewerDefaultEngagedUserColor); this.ownerContext = ownerContext; this.userStateManager.UserStateChanged += this.OnUserStateChanged; this.AddStreamConfiguration(InteractionStreamName, new StreamConfiguration(this.GetInteractionStreamProperties, this.SetInteractionStreamProperty)); this.AddStreamConfiguration(UserViewerStreamName, new StreamConfiguration(this.GetUserViewerStreamProperties, this.SetUserViewerStreamProperty)); } /// /// True if we should process interaction data fed into interaction stream and user state manager. /// False otherwise. /// private bool ShouldProcessInteractionData { get { // Check for a null userInfos since we may still get posted events from // the stream after we have unregistered our event handler and deleted // our buffers. // Also we process interaction data even if only user viewer stream is // enabled since data is used by IUserStateManager and UserViewerColorizer // as well as by clients listening directly to interaction stream. return (this.userInfos != null) && (this.interactionIsEnabled || this.userViewerIsEnabled); } } /// /// Lets ISensorStreamHandler know that Kinect Sensor associated with this stream /// handler has changed. /// /// /// New KinectSensor. /// public override void OnSensorChanged(KinectSensor newSensor) { if (this.sensor != null) { try { this.interactionStream.InteractionFrameReady -= this.InteractionFrameReadyAsync; this.interactionStream.Dispose(); this.interactionStream = null; this.sensor.SkeletonStream.AppChoosesSkeletons = false; } catch (InvalidOperationException) { // KinectSensor might enter an invalid state while enabling/disabling streams or stream features. // E.g.: sensor might be abruptly unplugged. } this.userInfos = null; } this.sensor = newSensor; if (newSensor != null) { try { this.interactionStream = new InteractionStream(newSensor, this); this.interactionStream.InteractionFrameReady += this.InteractionFrameReadyAsync; this.sensor.SkeletonStream.AppChoosesSkeletons = true; this.userInfos = new UserInfo[InteractionFrame.UserInfoArrayLength]; } catch (InvalidOperationException) { // KinectSensor might enter an invalid state while enabling/disabling streams or stream features. // E.g.: sensor might be abruptly unplugged. } } this.userStateManager.Reset(); this.userViewerColorizer.ResetColorLookupTable(); } /// /// Process data from one Kinect depth frame. /// /// /// Kinect depth data. /// /// /// from which we obtained depth data. /// public override void ProcessDepth(DepthImagePixel[] depthData, DepthImageFrame depthFrame) { if (depthData == null) { throw new ArgumentNullException("depthData"); } if (depthFrame == null) { throw new ArgumentNullException("depthFrame"); } if (this.ShouldProcessInteractionData) { this.interactionStream.ProcessDepth(depthData, depthFrame.Timestamp); } this.ProcessUserViewerImageAsync(depthData, depthFrame); } /// /// Process data from one Kinect skeleton frame. /// /// /// Kinect skeleton data. /// /// /// from which we obtained skeleton data. /// public override void ProcessSkeleton(Skeleton[] skeletons, SkeletonFrame skeletonFrame) { if (skeletons == null) { throw new ArgumentNullException("skeletons"); } if (skeletonFrame == null) { throw new ArgumentNullException("skeletonFrame"); } if (this.ShouldProcessInteractionData) { this.interactionStream.ProcessSkeleton(skeletons, this.sensor.AccelerometerGetCurrentReading(), skeletonFrame.Timestamp); } this.userStateManager.ChooseTrackedUsers(skeletons, skeletonFrame.Timestamp, this.recommendedUserTrackingIds); try { this.sensor.SkeletonStream.ChooseSkeletons( this.recommendedUserTrackingIds[0], this.recommendedUserTrackingIds[1]); } catch (InvalidOperationException) { // KinectSensor might enter an invalid state while choosing skeletons. // E.g.: sensor might be abruptly unplugged. } } /// /// Handle an http request. /// /// /// Name of stream for which property values should be set. /// /// /// Context containing HTTP request data, which will also contain associated /// response upon return. /// /// /// Request URI path relative to the stream name associated with this sensor stream /// handler in the stream handler owner. /// /// /// Await-able task. /// /// /// Return value should never be null. Implementations should use Task.FromResult(0) /// if function is implemented synchronously so that callers can await without /// needing to check for null. /// public override Task HandleRequestAsync(string streamName, HttpListenerContext requestContext, string subpath) { if (streamName == null) { throw new ArgumentNullException("streamName"); } if (requestContext == null) { throw new ArgumentNullException("requestContext"); } if (subpath == null) { throw new ArgumentNullException("subpath"); } if (!InteractionStreamName.Equals(streamName)) { // Only supported endpoints are related to interaction stream KinectRequestHandler.CloseResponse(requestContext, HttpStatusCode.NotFound); return SharedConstants.EmptyCompletedTask; } var splitPath = KinectRequestHandler.SplitUriSubpath(subpath); if (splitPath == null) { KinectRequestHandler.CloseResponse(requestContext, HttpStatusCode.NotFound); return SharedConstants.EmptyCompletedTask; } var pathComponent = splitPath.Item1; switch (pathComponent) { case ClientUriSubpath: // Only support one client at any one time if (this.clientRpcChannel != null) { if (this.clientRpcChannel.CheckConnectionStatus()) { KinectRequestHandler.CloseResponse(requestContext, HttpStatusCode.Conflict); return SharedConstants.EmptyCompletedTask; } this.clientRpcChannel = null; } WebSocketRpcChannel.TryOpenAsync( requestContext, channel => { // Check again in case another request came in before connection was established if (this.clientRpcChannel != null) { channel.Dispose(); } this.clientRpcChannel = channel; }, channel => { // Only forget the current channel if it matches the channel being closed if (this.clientRpcChannel == channel) { this.clientRpcChannel = null; } }); break; default: KinectRequestHandler.CloseResponse(requestContext, HttpStatusCode.NotFound); break; } return SharedConstants.EmptyCompletedTask; } /// /// Cancel all pending operations. /// public override void Cancel() { if (this.clientRpcChannel != null) { this.clientRpcChannel.Cancel(); } } /// /// Lets handler know that it should clean up resources associated with sensor stream /// handling. /// /// /// Await-able task. /// /// /// Return value should never be null. Implementations should use Task.FromResult(0) /// if function is implemented synchronously so that callers can await without /// needing to check for null. /// public override async Task UninitializeAsync() { if (this.clientRpcChannel != null) { await this.clientRpcChannel.CloseAsync(); } } /// /// Gets interaction information available for a specified location in UI. /// /// /// The skeleton tracking ID for which interaction information is being retrieved. /// /// /// The hand type for which interaction information is being retrieved. /// /// /// X-coordinate of UI location for which interaction information is being retrieved. /// 0.0 corresponds to left edge of interaction region and 1.0 corresponds to right edge /// of interaction region. /// /// /// Y-coordinate of UI location for which interaction information is being retrieved. /// 0.0 corresponds to top edge of interaction region and 1.0 corresponds to bottom edge /// of interaction region. /// /// /// An object instance. /// public InteractionInfo GetInteractionInfoAtLocation(int skeletonTrackingId, InteractionHandType handType, double x, double y) { var interactionInfo = new InteractionInfo { IsPressTarget = false, IsGripTarget = false }; if (this.interactionIsEnabled && (this.clientRpcChannel != null)) { var result = this.clientRpcChannel.CallFunction("getInteractionInfoAtLocation", skeletonTrackingId, handType.ToString(), x, y); if (result.Success) { interactionInfo.IsGripTarget = result.Result.isGripTarget; interactionInfo.IsPressTarget = result.Result.isPressTarget; var elementId = result.Result.pressTargetControlId; interactionInfo.PressTargetControlId = (elementId != null) ? elementId.GetHashCode() : 0; interactionInfo.PressAttractionPointX = result.Result.pressAttractionPointX; interactionInfo.PressAttractionPointY = result.Result.pressAttractionPointY; } } return interactionInfo; } /// /// Converts a color into the corresponding 32-bit integer RGBA representation. /// /// /// Color to convert /// /// /// 32-bit integer RGBA representation of color. /// internal static int GetRgbaColorInt(Color color) { return (color.A << 24) | (color.B << 16) | (color.G << 8) | color.R; } /// /// Event handler for InteractionStream's InteractionFrameReady event /// /// object sending the event /// event arguments internal async void InteractionFrameReadyAsync(object sender, InteractionFrameReadyEventArgs e) { if (!this.ShouldProcessInteractionData) { return; } if (this.isProcessingInteractionFrame) { // Re-entered InteractionFrameReadyAsync while a previous frame is already being processed. // Just ignore new frames until the current one finishes processing. return; } this.isProcessingInteractionFrame = true; try { bool haveFrameData = false; using (var interactionFrame = e.OpenInteractionFrame()) { // Even though we checked value of userInfos above as part of // ShouldProcessInteractionData check, callbacks happening while // opening an interaction frame might have invalidated it, so we // check value again. if ((interactionFrame != null) && (this.userInfos != null)) { // Copy interaction frame data so we can dispose interaction frame // right away, even if data processing/event handling takes a while. interactionFrame.CopyInteractionDataTo(this.userInfos); this.interactionStreamMessage.timestamp = interactionFrame.Timestamp; haveFrameData = true; } } if (haveFrameData) { this.userStateManager.UpdateUserInformation(this.userInfos, this.interactionStreamMessage.timestamp); this.userViewerColorizer.UpdateColorLookupTable(this.userInfos, this.userViewerDefaultUserColor, this.userStateManager.UserStates, this.userViewerUserColors); if (this.interactionIsEnabled) { this.interactionStreamMessage.UpdateHandPointers(this.userInfos, this.userStateManager.PrimaryUserTrackingId); await this.ownerContext.SendStreamMessageAsync(this.interactionStreamMessage); } } } finally { this.isProcessingInteractionFrame = false; } } /// /// Gets all interaction stream property value. /// /// /// Property name->value map where property values should be set. /// internal void GetInteractionStreamProperties(Dictionary propertyMap) { propertyMap.Add(KinectRequestHandler.EnabledPropertyName, this.interactionIsEnabled); propertyMap.Add(InteractionPrimaryUserPropertyName, this.userStateManager.PrimaryUserTrackingId); propertyMap.Add(InteractionUserStatesPropertyName, DefaultUserStateManager.GetStateMappingEntryArray(this.userStateManager.UserStates)); } /// /// Set an interaction stream property value. /// /// /// Name of property to set. /// /// /// Property value to set. /// /// /// null if property setting was successful, error message otherwise. /// internal string SetInteractionStreamProperty(string propertyName, object propertyValue) { bool recognized = true; if (propertyValue == null) { // None of the interaction stream properties accept a null value return Properties.Resources.PropertyValueInvalidFormat; } try { switch (propertyName) { case KinectRequestHandler.EnabledPropertyName: this.interactionIsEnabled = (bool)propertyValue; break; default: recognized = false; break; } if (!recognized) { return Properties.Resources.PropertyNameUnrecognized; } } catch (InvalidCastException) { return Properties.Resources.PropertyValueInvalidFormat; } return null; } /// /// Gets all user viewer stream property value. /// /// /// Property name->value map where property values should be set. /// internal void GetUserViewerStreamProperties(Dictionary propertyMap) { propertyMap.Add(KinectRequestHandler.EnabledPropertyName, this.userViewerIsEnabled); propertyMap.Add(UserViewerResolutionPropertyName, string.Format(CultureInfo.InvariantCulture, @"{0}x{1}", this.userViewerColorizer.Width, this.userViewerColorizer.Height)); propertyMap.Add(UserViewerDefaultUserColorPropertyName, this.userViewerDefaultUserColor); propertyMap.Add(UserViewerUserColorsPropertyName, this.userViewerUserColors); } /// /// Set a user viewer stream property. /// /// /// Name of property to set. /// /// /// Property value to set. /// /// /// null if property setting was successful, error message otherwise. /// internal string SetUserViewerStreamProperty(string propertyName, object propertyValue) { bool recognized = true; try { switch (propertyName) { case KinectRequestHandler.EnabledPropertyName: this.userViewerIsEnabled = (bool)propertyValue; break; case UserViewerResolutionPropertyName: if (propertyValue == null) { return Properties.Resources.PropertyValueInvalidFormat; } var match = UserViewerResolutionRegex.Match((string)propertyValue); if (!match.Success || (match.Groups.Count != 3)) { return Properties.Resources.PropertyValueInvalidFormat; } int width = int.Parse(match.Groups[1].Value, CultureInfo.InvariantCulture); int height = int.Parse(match.Groups[2].Value, CultureInfo.InvariantCulture); if (!IsSupportedUserViewerResolution(width, height)) { return Properties.Resources.PropertyValueUnsupportedResolution; } this.userViewerColorizer.SetResolution(width, height); break; case UserViewerDefaultUserColorPropertyName: this.userViewerDefaultUserColor = (int)propertyValue; this.UpdateColorizerLookupTable(); break; case UserViewerUserColorsPropertyName: if (propertyValue == null) { // Null values just clear the set of user colors this.userViewerUserColors.Clear(); break; } var userColors = (Dictionary)propertyValue; // Verify that all dictionary values are integers bool allIntegers = userColors.Values.Select(color => color as int?).All(colorInt => colorInt != null); if (!allIntegers) { return Properties.Resources.PropertyValueInvalidFormat; } // If property value specified is compatible, copy values over this.userViewerUserColors.Clear(); foreach (var entry in userColors) { this.userViewerUserColors.Add(entry.Key, (int)entry.Value); } this.UpdateColorizerLookupTable(); break; default: recognized = false; break; } if (!recognized) { return Properties.Resources.PropertyNameUnrecognized; } } catch (InvalidCastException) { return Properties.Resources.PropertyValueInvalidFormat; } catch (NullReferenceException) { return Properties.Resources.PropertyValueInvalidFormat; } return null; } /// /// Determine if the specified resolution is supported for user viewer images. /// /// /// Image width. /// /// /// Image height. /// /// /// True if specified resolution is supported, false otherwise. /// private static bool IsSupportedUserViewerResolution(int width, int height) { return UserViewerSupportedResolutions.Any(resolution => ((int)resolution.Width == width) && ((int)resolution.Height == height)); } /// /// Handler for IUserStateManager.UserStateChanged event. /// /// object sending the event /// event arguments private async void OnUserStateChanged(object sender, UserStateChangedEventArgs e) { if (this.interactionIsEnabled) { // If enabled, forward all user state events to client await this.ownerContext.SendEventMessageAsync(e.Message); } } /// /// Update colorizer lookup table from color-related configuration. /// private void UpdateColorizerLookupTable() { if (this.ShouldProcessInteractionData) { this.userViewerColorizer.UpdateColorLookupTable( this.userInfos, this.userViewerDefaultUserColor, this.userStateManager.UserStates, this.userViewerUserColors); } } /// /// Process depth data to obtain user viewer image. /// /// /// Kinect depth data. /// /// /// from which we obtained depth data. /// private async void ProcessUserViewerImageAsync(DepthImagePixel[] depthData, DepthImageFrame depthFrame) { if (this.userViewerIsEnabled) { if (this.isProcessingUserViewerImage) { // Re-entered ProcessUserViewerImageAsync while a previous image is already being processed. // Just ignore new depth frames until the current one finishes processing. return; } this.isProcessingUserViewerImage = true; try { this.userViewerColorizer.ColorizeDepthPixels(depthData, depthFrame.Width, depthFrame.Height); this.userViewerStreamMessage.timestamp = depthFrame.Timestamp; this.userViewerStreamMessage.width = this.userViewerColorizer.Width; this.userViewerStreamMessage.height = this.userViewerColorizer.Height; this.userViewerStreamMessage.bufferLength = this.userViewerColorizer.Buffer.Length; await this.ownerContext.SendTwoPartStreamMessageAsync(this.userViewerStreamMessage, this.userViewerColorizer.Buffer); } finally { this.isProcessingUserViewerImage = false; } } } } }