// ----------------------------------------------------------------------- // // Copyright (c) Microsoft Corporation. All rights reserved. // // ----------------------------------------------------------------------- namespace Microsoft.Samples.Kinect.Webserver.Sensor { using System; using System.Collections.Generic; using System.Collections.ObjectModel; using System.Diagnostics; using System.Globalization; using System.Net; using System.Net.WebSockets; using System.Runtime.Serialization; using System.Threading.Tasks; using Microsoft.Kinect; using Microsoft.Kinect.Toolkit; using Microsoft.Samples.Kinect.Webserver; using Microsoft.Samples.Kinect.Webserver.Sensor.Serialization; /// /// Implementation of IHttpRequestHandler used to handle communication to/from /// a Kinect sensor represented by a specified KinectSensorChooser. /// public class KinectRequestHandler : IHttpRequestHandler { /// /// Property name used to return success status to client. /// public const string SuccessPropertyName = "success"; /// /// Property name used to return a set of property names that encountered errors to client. /// public const string ErrorsPropertyName = "errors"; /// /// Property name used to represent "enabled" property of a stream. /// public const string EnabledPropertyName = "enabled"; /// /// Sub path for REST endpoint owned by this handler. /// public const string StateUriSubpath = "STATE"; /// /// Sub path for stream data web-socket endpoint owned by this handler. /// public const string StreamUriSubpath = "STREAM"; /// /// Sub path for stream data web-socket endpoint owned by this handler. /// public const string EventsUriSubpath = "EVENTS"; /// /// MIME type name for JSON data. /// internal const string JsonContentType = "application/json"; /// /// Reserved names that streams can't have. /// private static readonly HashSet ReservedNames = new HashSet { SuccessPropertyName.ToUpperInvariant(), ErrorsPropertyName.ToUpperInvariant(), StreamUriSubpath.ToUpperInvariant(), EventsUriSubpath.ToUpperInvariant(), StateUriSubpath.ToUpperInvariant() }; /// /// Sensor chooser used to obtain a KinectSensor. /// private readonly KinectSensorChooser sensorChooser; /// /// Array of sensor stream handlers used to process kinect data and deliver /// a data streams ready for web consumption. /// private readonly ISensorStreamHandler[] streamHandlers; /// /// Map of stream handler names to sensor stream handler objects /// private readonly Dictionary streamHandlerMap = new Dictionary(); /// /// Map of uri names (expected to be case-insensitive) corresponding to a stream, to /// their case-sensitive names used in JSON communications. /// private readonly Dictionary uriName2StreamNameMap = new Dictionary(); /// /// Channel used to send event messages that are part of main data stream. /// private readonly List streamChannels = new List(); /// /// Channel used to send event messages that are part of an events stream. /// private readonly List eventsChannels = new List(); /// /// Intermediate storage for the skeleton data received from the Kinect sensor. /// private Skeleton[] skeletons; /// /// Kinect sensor currently associated with this request handler. /// private KinectSensor sensor; /// /// Initializes a new instance of the KinectRequestHandler class. /// /// /// Sensor chooser that will be used to obtain a KinectSensor. /// /// /// Collection of stream handler factories to be used to process kinect data and deliver /// data streams ready for web consumption. /// internal KinectRequestHandler(KinectSensorChooser sensorChooser, Collection streamHandlerFactories) { this.sensorChooser = sensorChooser; this.streamHandlers = new ISensorStreamHandler[streamHandlerFactories.Count]; var streamHandlerContext = new SensorStreamHandlerContext(this.SendStreamMessageAsync, this.SendEventMessageAsync); var normalizedNameSet = new HashSet(); // Associate each of the supported stream names with the corresponding handlers for (int i = 0; i < streamHandlerFactories.Count; ++i) { var handler = streamHandlerFactories[i].CreateHandler(streamHandlerContext); this.streamHandlers[i] = handler; var names = handler.GetSupportedStreamNames(); foreach (var name in names) { if (string.IsNullOrEmpty(name)) { throw new InvalidOperationException(@"Empty stream names are not supported"); } if (name.IndexOfAny(SharedConstants.UriPathComponentDelimiters) >= 0) { throw new InvalidOperationException(@"Stream names can't contain '/' character"); } var normalizedName = name.ToUpperInvariant(); if (ReservedNames.Contains(normalizedName)) { throw new InvalidOperationException(string.Format(CultureInfo.InvariantCulture, "'{0}' is a reserved stream name", normalizedName)); } if (normalizedNameSet.Contains(normalizedName)) { throw new InvalidOperationException(string.Format(CultureInfo.InvariantCulture, "'{0}' is a duplicate stream name", normalizedName)); } normalizedNameSet.Add(normalizedName); this.uriName2StreamNameMap.Add(normalizedName, name); this.streamHandlerMap.Add(name, handler); } } } /// /// Prepares handler to start receiving HTTP requests. /// /// /// Await-able task. /// public Task InitializeAsync() { this.OnKinectChanged(this.sensorChooser.Kinect); this.sensorChooser.KinectChanged += this.SensorChooserKinectChanged; return SharedConstants.EmptyCompletedTask; } /// /// Handle an http request. /// /// /// Context containing HTTP request data, which will also contain associated /// response upon return. /// /// /// Request URI path relative to the URI prefix associated with this request /// handler in the HttpListener. /// /// /// Await-able task. /// public async Task HandleRequestAsync(HttpListenerContext requestContext, string subpath) { if (requestContext == null) { throw new ArgumentNullException("requestContext"); } if (subpath == null) { throw new ArgumentNullException("subpath"); } var splitPath = SplitUriSubpath(subpath); if (splitPath == null) { CloseResponse(requestContext, HttpStatusCode.NotFound); return; } var pathComponent = splitPath.Item1; try { switch (pathComponent) { case StateUriSubpath: await this.HandleStateRequest(requestContext); break; case StreamUriSubpath: this.HandleStreamRequest(requestContext); break; case EventsUriSubpath: this.HandleEventRequest(requestContext); break; default: var remainingSubpath = splitPath.Item2; if (remainingSubpath == null) { CloseResponse(requestContext, HttpStatusCode.NotFound); return; } string streamName; if (!this.uriName2StreamNameMap.TryGetValue(pathComponent, out streamName)) { CloseResponse(requestContext, HttpStatusCode.NotFound); return; } var streamHandler = this.streamHandlerMap[streamName]; await streamHandler.HandleRequestAsync(streamName, requestContext, remainingSubpath); break; } } catch (Exception e) { // If there's any exception while handling a request, return appropriate // error status code rather than propagating exception further. Trace.TraceError("Exception encountered while handling Kinect sensor request:\n{0}", e); CloseResponse(requestContext, HttpStatusCode.InternalServerError); } } /// /// Cancel all pending operations. /// public void Cancel() { foreach (var channel in this.streamChannels.SafeCopy()) { channel.Cancel(); } foreach (var channel in this.eventsChannels.SafeCopy()) { channel.Cancel(); } foreach (var streamHandler in this.streamHandlers) { streamHandler.Cancel(); } } /// /// Lets handler know that no more HTTP requests will be received, so that it can /// clean up resources associated with request handling. /// /// /// Await-able task. /// public async Task UninitializeAsync() { foreach (var channel in this.streamChannels.SafeCopy()) { await channel.CloseAsync(); } foreach (var channel in this.eventsChannels.SafeCopy()) { await channel.CloseAsync(); } this.OnKinectChanged(null); foreach (var handler in this.streamHandlers) { await handler.UninitializeAsync(); } } /// /// Close response stream and associate a status code with response. /// /// /// Context whose response we should close. /// /// /// Status code. /// internal static void CloseResponse(HttpListenerContext context, HttpStatusCode statusCode) { try { context.Response.StatusCode = (int)statusCode; context.Response.Close(); } catch (HttpListenerException e) { Trace.TraceWarning( "Problem encountered while sending response for kinect sensor request. Client might have aborted request. Cause: \"{0}\"", e.Message); } } /// /// Splits a URI sub-path into "first path component" and "rest of sub-path". /// /// /// Uri sub-path. Expected to start with "/" character. /// /// /// /// A tuple containing two elements: /// 1) first sub-path component /// 2) rest of sub-path string /// /// /// May be null if sub-path is badly formed. /// /// /// /// The returned path components will have been normalized to uppercase. /// internal static Tuple SplitUriSubpath(string subpath) { if (!subpath.StartsWith("/", StringComparison.OrdinalIgnoreCase)) { return null; } subpath = subpath.Substring(1).ToUpperInvariant(); int delimiterIndex = subpath.IndexOfAny(SharedConstants.UriPathComponentDelimiters); var firstComponent = (delimiterIndex < 0) ? subpath : subpath.Substring(0, delimiterIndex); var remainingSubpath = (delimiterIndex < 0) ? null : subpath.Substring(delimiterIndex); return new Tuple(firstComponent, remainingSubpath); } /// /// Add response headers that mean that response should not be cached. /// /// /// Http response object. /// /// /// This method needs to be called before starting to write the response output stream, /// or the headers meant to be added will be silently ignored, since they can't be sent /// after content is sent. /// private static void AddNoCacheHeaders(HttpListenerResponse response) { response.Headers.Add("Cache-Control", "no-cache"); response.Headers.Add("Cache-Control", "no-store"); response.Headers.Add("Pragma", "no-cache"); response.Headers.Add("Expires", "Mon, 1 Jan 1990 00:00:00 GMT"); } /// /// Handle Http GET requests for state endpoint. /// /// /// Context containing HTTP GET request data, and which will also contain associated /// response upon return. /// /// /// Await-able task. /// private async Task HandleGetStateRequest(HttpListenerContext requestContext) { // Don't cache results of any endpoint requests AddNoCacheHeaders(requestContext.Response); var responseProperties = new Dictionary(); foreach (var mapEntry in this.streamHandlerMap) { var handlerStatus = mapEntry.Value.GetState(mapEntry.Key); responseProperties.Add(mapEntry.Key, handlerStatus); } requestContext.Response.ContentType = JsonContentType; await responseProperties.DictionaryToJsonAsync(requestContext.Response.OutputStream); CloseResponse(requestContext, HttpStatusCode.OK); } /// /// Handle Http POST requests for state endpoint. /// /// /// Context containing HTTP POST request data, and which will also contain associated /// response upon return. /// /// /// Await-able task. /// private async Task HandlePostStateRequest(HttpListenerContext requestContext) { // Don't cache results of any endpoint requests AddNoCacheHeaders(requestContext.Response); Dictionary requestProperties; try { requestProperties = await requestContext.Request.InputStream.DictionaryFromJsonAsync(); } catch (SerializationException) { requestProperties = null; } if (requestProperties == null) { CloseResponse(requestContext, HttpStatusCode.BadRequest); return; } var responseProperties = new Dictionary(); var errorStreamNames = new List(); foreach (var requestEntry in requestProperties) { ISensorStreamHandler handler; if (!this.streamHandlerMap.TryGetValue(requestEntry.Key, out handler)) { // Don't process unrecognized handlers responseProperties.Add(requestEntry.Key, Properties.Resources.StreamNameUnrecognized); errorStreamNames.Add(requestEntry.Key); continue; } var propertiesToSet = requestEntry.Value as IDictionary; if (propertiesToSet == null) { continue; } var propertyErrors = new Dictionary(); var success = handler.SetState(requestEntry.Key, new ReadOnlyDictionary(propertiesToSet), propertyErrors); if (!success) { responseProperties.Add(requestEntry.Key, propertyErrors); errorStreamNames.Add(requestEntry.Key); } } if (errorStreamNames.Count == 0) { responseProperties.Add(SuccessPropertyName, true); } else { // The only properties returned other than the "success" property are to indicate error, // so if there are other properties present it means that we've encountered at least // one error while trying to change state of the streams. responseProperties.Add(SuccessPropertyName, false); responseProperties.Add(ErrorsPropertyName, errorStreamNames.ToArray()); } requestContext.Response.ContentType = JsonContentType; await responseProperties.DictionaryToJsonAsync(requestContext.Response.OutputStream); CloseResponse(requestContext, HttpStatusCode.OK); } /// /// Handle Http requests for state endpoint. /// /// /// Context containing HTTP request data, and which will also contain associated /// response upon return. /// /// /// Await-able task. /// private async Task HandleStateRequest(HttpListenerContext requestContext) { const string AllowHeader = "Allow"; const string AllowedMethods = "GET, POST, OPTIONS"; switch (requestContext.Request.HttpMethod) { case "GET": await this.HandleGetStateRequest(requestContext); break; case "POST": await this.HandlePostStateRequest(requestContext); break; case "OPTIONS": requestContext.Response.Headers.Set(AllowHeader, AllowedMethods); CloseResponse(requestContext, HttpStatusCode.OK); break; default: requestContext.Response.Headers.Set(AllowHeader, AllowedMethods); CloseResponse(requestContext, HttpStatusCode.MethodNotAllowed); break; } } /// /// Handle Http requests for stream endpoint. /// /// /// Context containing HTTP request data, and which will also contain associated /// response upon return. /// private void HandleStreamRequest(HttpListenerContext requestContext) { WebSocketEventChannel.TryOpenAsync( requestContext, channel => this.streamChannels.Add(channel), channel => this.streamChannels.Remove(channel)); } /// /// Handle Http requests for event endpoint. /// /// /// Context containing HTTP request data, and which will also contain associated /// response upon return. /// private void HandleEventRequest(HttpListenerContext requestContext) { WebSocketEventChannel.TryOpenAsync( requestContext, channel => this.eventsChannels.Add(channel), channel => this.eventsChannels.Remove(channel)); } /// /// Asynchronously send stream message to client(s) of stream handler. /// /// /// Stream message to send. /// /// /// Binary payload of stream message. May be null if message does not require a binary /// payload. /// /// /// Await-able task. /// private async Task SendStreamMessageAsync(StreamMessage message, byte[] binaryPayload) { var webSocketMessage = message.ToTextMessage(); foreach (var channel in this.streamChannels.SafeCopy()) { if (channel == null) { break; } if (binaryPayload != null) { // If binary payload is non-null, send two-part stream message, with the first // part acting as a message header. await channel.SendMessagesAsync( webSocketMessage, new WebSocketMessage(new ArraySegment(binaryPayload), WebSocketMessageType.Binary)); continue; } await channel.SendMessagesAsync(webSocketMessage); } } /// /// Asynchronously send event message to client(s) of stream handler. /// /// /// Event message to send. /// /// /// Await-able task. /// private async Task SendEventMessageAsync(EventMessage message) { foreach (var channel in this.eventsChannels.SafeCopy()) { await channel.SendMessagesAsync(message.ToTextMessage()); } } /// /// Responds to KinectSensor changes. /// /// /// New sensor associated with this KinectRequestHandler. /// private void OnKinectChanged(KinectSensor newSensor) { if (this.sensor != null) { try { this.sensor.ColorFrameReady -= this.SensorColorFrameReady; this.sensor.DepthFrameReady -= this.SensorDepthFrameReady; this.sensor.SkeletonFrameReady -= this.SensorSkeletonFrameReady; this.sensor.DepthStream.Range = DepthRange.Default; this.sensor.SkeletonStream.EnableTrackingInNearRange = false; this.sensor.DepthStream.Disable(); this.sensor.SkeletonStream.Disable(); } catch (InvalidOperationException) { // KinectSensor might enter an invalid state while enabling/disabling streams or stream features. // E.g.: sensor might be abruptly unplugged. } this.sensor = null; this.skeletons = null; } if (newSensor != null) { // Allocate space to put the skeleton and interaction data we'll receive this.sensor = newSensor; try { newSensor.DepthStream.Enable(DepthImageFormat.Resolution640x480Fps30); newSensor.SkeletonStream.Enable(); try { newSensor.DepthStream.Range = DepthRange.Near; newSensor.SkeletonStream.EnableTrackingInNearRange = true; } catch (InvalidOperationException) { // Non Kinect for Windows devices do not support Near mode, so reset back to default mode. newSensor.DepthStream.Range = DepthRange.Default; newSensor.SkeletonStream.EnableTrackingInNearRange = false; } this.skeletons = new Skeleton[newSensor.SkeletonStream.FrameSkeletonArrayLength]; newSensor.ColorFrameReady += this.SensorColorFrameReady; newSensor.DepthFrameReady += this.SensorDepthFrameReady; newSensor.SkeletonFrameReady += this.SensorSkeletonFrameReady; } catch (InvalidOperationException) { // KinectSensor might enter an invalid state while enabling/disabling streams or stream features. // E.g.: sensor might be abruptly unplugged. } } foreach (var handler in this.streamHandlers) { handler.OnSensorChanged(newSensor); } } /// /// Handler for KinectSensorChooser's KinectChanged event. /// /// object sending the event /// event arguments private void SensorChooserKinectChanged(object sender, KinectChangedEventArgs args) { this.OnKinectChanged(args.NewSensor); } /// /// Handler for the Kinect sensor's ColorFrameReady event /// /// object sending the event /// event arguments private void SensorColorFrameReady(object sender, ColorImageFrameReadyEventArgs colorImageFrameReadyEventArgs) { // Even though we un-register all our event handlers when the sensor // changes, there may still be an event for the old sensor in the queue // due to the way the KinectSensor delivers events. So check again here. if (this.sensor != sender) { return; } using (var colorFrame = colorImageFrameReadyEventArgs.OpenColorImageFrame()) { if (null != colorFrame) { try { // Hand data to each handler to be processed foreach (var handler in this.streamHandlers) { handler.ProcessColor(colorFrame.GetRawPixelData(), colorFrame); } } catch (InvalidOperationException) { // ColorFrame functions may throw when the sensor gets // into a bad state. Ignore the frame in that case. } } } } /// /// Handler for the Kinect sensor's DepthFrameReady event /// /// object sending the event /// event arguments private void SensorDepthFrameReady(object sender, DepthImageFrameReadyEventArgs depthImageFrameReadyEventArgs) { // Even though we un-register all our event handlers when the sensor // changes, there may still be an event for the old sensor in the queue // due to the way the KinectSensor delivers events. So check again here. if (this.sensor != sender) { return; } using (var depthFrame = depthImageFrameReadyEventArgs.OpenDepthImageFrame()) { if (null != depthFrame) { var depthBuffer = depthFrame.GetRawPixelData(); try { // Hand data to each handler to be processed foreach (var handler in this.streamHandlers) { handler.ProcessDepth(depthBuffer, depthFrame); } } catch (InvalidOperationException) { // DepthFrame functions may throw when the sensor gets // into a bad state. Ignore the frame in that case. } } } } /// /// Handler for the Kinect sensor's SkeletonFrameReady event /// /// object sending the event /// event arguments private void SensorSkeletonFrameReady(object sender, SkeletonFrameReadyEventArgs skeletonFrameReadyEventArgs) { // Even though we un-register all our event handlers when the sensor // changes, there may still be an event for the old sensor in the queue // due to the way the KinectSensor delivers events. So check again here. if (this.sensor != sender) { return; } using (SkeletonFrame skeletonFrame = skeletonFrameReadyEventArgs.OpenSkeletonFrame()) { if (null != skeletonFrame) { try { // Copy the skeleton data from the frame to an array used for temporary storage skeletonFrame.CopySkeletonDataTo(this.skeletons); foreach (var handler in this.streamHandlers) { handler.ProcessSkeleton(this.skeletons, skeletonFrame); } } catch (InvalidOperationException) { // SkeletonFrame functions may throw when the sensor gets // into a bad state. Ignore the frame in that case. } } } } } /// /// Helper class that adds a SafeCopy extension method to a List of WebSocketEventChannel. /// internal static class ChannelListHelper { /// /// Safely makes a copy of the list of stream channels. /// /// Array containing the stream channels. public static IEnumerable SafeCopy(this List list) { var channels = new WebSocketEventChannel[list.Count]; list.CopyTo(channels); return channels; } } }