// -----------------------------------------------------------------------
//
// Copyright (c) Microsoft Corporation. All rights reserved.
//
// -----------------------------------------------------------------------
namespace Microsoft.Samples.Kinect.Webserver.Sensor
{
using System;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.Diagnostics;
using System.Globalization;
using System.Net;
using System.Net.WebSockets;
using System.Runtime.Serialization;
using System.Threading.Tasks;
using Microsoft.Kinect;
using Microsoft.Kinect.Toolkit;
using Microsoft.Samples.Kinect.Webserver;
using Microsoft.Samples.Kinect.Webserver.Sensor.Serialization;
///
/// Implementation of IHttpRequestHandler used to handle communication to/from
/// a Kinect sensor represented by a specified KinectSensorChooser.
///
public class KinectRequestHandler : IHttpRequestHandler
{
///
/// Property name used to return success status to client.
///
public const string SuccessPropertyName = "success";
///
/// Property name used to return a set of property names that encountered errors to client.
///
public const string ErrorsPropertyName = "errors";
///
/// Property name used to represent "enabled" property of a stream.
///
public const string EnabledPropertyName = "enabled";
///
/// Sub path for REST endpoint owned by this handler.
///
public const string StateUriSubpath = "STATE";
///
/// Sub path for stream data web-socket endpoint owned by this handler.
///
public const string StreamUriSubpath = "STREAM";
///
/// Sub path for stream data web-socket endpoint owned by this handler.
///
public const string EventsUriSubpath = "EVENTS";
///
/// MIME type name for JSON data.
///
internal const string JsonContentType = "application/json";
///
/// Reserved names that streams can't have.
///
private static readonly HashSet ReservedNames = new HashSet
{
SuccessPropertyName.ToUpperInvariant(),
ErrorsPropertyName.ToUpperInvariant(),
StreamUriSubpath.ToUpperInvariant(),
EventsUriSubpath.ToUpperInvariant(),
StateUriSubpath.ToUpperInvariant()
};
///
/// Sensor chooser used to obtain a KinectSensor.
///
private readonly KinectSensorChooser sensorChooser;
///
/// Array of sensor stream handlers used to process kinect data and deliver
/// a data streams ready for web consumption.
///
private readonly ISensorStreamHandler[] streamHandlers;
///
/// Map of stream handler names to sensor stream handler objects
///
private readonly Dictionary streamHandlerMap = new Dictionary();
///
/// Map of uri names (expected to be case-insensitive) corresponding to a stream, to
/// their case-sensitive names used in JSON communications.
///
private readonly Dictionary uriName2StreamNameMap = new Dictionary();
///
/// Channel used to send event messages that are part of main data stream.
///
private readonly List streamChannels = new List();
///
/// Channel used to send event messages that are part of an events stream.
///
private readonly List eventsChannels = new List();
///
/// Intermediate storage for the skeleton data received from the Kinect sensor.
///
private Skeleton[] skeletons;
///
/// Kinect sensor currently associated with this request handler.
///
private KinectSensor sensor;
///
/// Initializes a new instance of the KinectRequestHandler class.
///
///
/// Sensor chooser that will be used to obtain a KinectSensor.
///
///
/// Collection of stream handler factories to be used to process kinect data and deliver
/// data streams ready for web consumption.
///
internal KinectRequestHandler(KinectSensorChooser sensorChooser, Collection streamHandlerFactories)
{
this.sensorChooser = sensorChooser;
this.streamHandlers = new ISensorStreamHandler[streamHandlerFactories.Count];
var streamHandlerContext = new SensorStreamHandlerContext(this.SendStreamMessageAsync, this.SendEventMessageAsync);
var normalizedNameSet = new HashSet();
// Associate each of the supported stream names with the corresponding handlers
for (int i = 0; i < streamHandlerFactories.Count; ++i)
{
var handler = streamHandlerFactories[i].CreateHandler(streamHandlerContext);
this.streamHandlers[i] = handler;
var names = handler.GetSupportedStreamNames();
foreach (var name in names)
{
if (string.IsNullOrEmpty(name))
{
throw new InvalidOperationException(@"Empty stream names are not supported");
}
if (name.IndexOfAny(SharedConstants.UriPathComponentDelimiters) >= 0)
{
throw new InvalidOperationException(@"Stream names can't contain '/' character");
}
var normalizedName = name.ToUpperInvariant();
if (ReservedNames.Contains(normalizedName))
{
throw new InvalidOperationException(string.Format(CultureInfo.InvariantCulture, "'{0}' is a reserved stream name", normalizedName));
}
if (normalizedNameSet.Contains(normalizedName))
{
throw new InvalidOperationException(string.Format(CultureInfo.InvariantCulture, "'{0}' is a duplicate stream name", normalizedName));
}
normalizedNameSet.Add(normalizedName);
this.uriName2StreamNameMap.Add(normalizedName, name);
this.streamHandlerMap.Add(name, handler);
}
}
}
///
/// Prepares handler to start receiving HTTP requests.
///
///
/// Await-able task.
///
public Task InitializeAsync()
{
this.OnKinectChanged(this.sensorChooser.Kinect);
this.sensorChooser.KinectChanged += this.SensorChooserKinectChanged;
return SharedConstants.EmptyCompletedTask;
}
///
/// Handle an http request.
///
///
/// Context containing HTTP request data, which will also contain associated
/// response upon return.
///
///
/// Request URI path relative to the URI prefix associated with this request
/// handler in the HttpListener.
///
///
/// Await-able task.
///
public async Task HandleRequestAsync(HttpListenerContext requestContext, string subpath)
{
if (requestContext == null)
{
throw new ArgumentNullException("requestContext");
}
if (subpath == null)
{
throw new ArgumentNullException("subpath");
}
var splitPath = SplitUriSubpath(subpath);
if (splitPath == null)
{
CloseResponse(requestContext, HttpStatusCode.NotFound);
return;
}
var pathComponent = splitPath.Item1;
try
{
switch (pathComponent)
{
case StateUriSubpath:
await this.HandleStateRequest(requestContext);
break;
case StreamUriSubpath:
this.HandleStreamRequest(requestContext);
break;
case EventsUriSubpath:
this.HandleEventRequest(requestContext);
break;
default:
var remainingSubpath = splitPath.Item2;
if (remainingSubpath == null)
{
CloseResponse(requestContext, HttpStatusCode.NotFound);
return;
}
string streamName;
if (!this.uriName2StreamNameMap.TryGetValue(pathComponent, out streamName))
{
CloseResponse(requestContext, HttpStatusCode.NotFound);
return;
}
var streamHandler = this.streamHandlerMap[streamName];
await streamHandler.HandleRequestAsync(streamName, requestContext, remainingSubpath);
break;
}
}
catch (Exception e)
{
// If there's any exception while handling a request, return appropriate
// error status code rather than propagating exception further.
Trace.TraceError("Exception encountered while handling Kinect sensor request:\n{0}", e);
CloseResponse(requestContext, HttpStatusCode.InternalServerError);
}
}
///
/// Cancel all pending operations.
///
public void Cancel()
{
foreach (var channel in this.streamChannels.SafeCopy())
{
channel.Cancel();
}
foreach (var channel in this.eventsChannels.SafeCopy())
{
channel.Cancel();
}
foreach (var streamHandler in this.streamHandlers)
{
streamHandler.Cancel();
}
}
///
/// Lets handler know that no more HTTP requests will be received, so that it can
/// clean up resources associated with request handling.
///
///
/// Await-able task.
///
public async Task UninitializeAsync()
{
foreach (var channel in this.streamChannels.SafeCopy())
{
await channel.CloseAsync();
}
foreach (var channel in this.eventsChannels.SafeCopy())
{
await channel.CloseAsync();
}
this.OnKinectChanged(null);
foreach (var handler in this.streamHandlers)
{
await handler.UninitializeAsync();
}
}
///
/// Close response stream and associate a status code with response.
///
///
/// Context whose response we should close.
///
///
/// Status code.
///
internal static void CloseResponse(HttpListenerContext context, HttpStatusCode statusCode)
{
try
{
context.Response.StatusCode = (int)statusCode;
context.Response.Close();
}
catch (HttpListenerException e)
{
Trace.TraceWarning(
"Problem encountered while sending response for kinect sensor request. Client might have aborted request. Cause: \"{0}\"", e.Message);
}
}
///
/// Splits a URI sub-path into "first path component" and "rest of sub-path".
///
///
/// Uri sub-path. Expected to start with "/" character.
///
///
///
/// A tuple containing two elements:
/// 1) first sub-path component
/// 2) rest of sub-path string
///
///
/// May be null if sub-path is badly formed.
///
///
///
/// The returned path components will have been normalized to uppercase.
///
internal static Tuple SplitUriSubpath(string subpath)
{
if (!subpath.StartsWith("/", StringComparison.OrdinalIgnoreCase))
{
return null;
}
subpath = subpath.Substring(1).ToUpperInvariant();
int delimiterIndex = subpath.IndexOfAny(SharedConstants.UriPathComponentDelimiters);
var firstComponent = (delimiterIndex < 0) ? subpath : subpath.Substring(0, delimiterIndex);
var remainingSubpath = (delimiterIndex < 0) ? null : subpath.Substring(delimiterIndex);
return new Tuple(firstComponent, remainingSubpath);
}
///
/// Add response headers that mean that response should not be cached.
///
///
/// Http response object.
///
///
/// This method needs to be called before starting to write the response output stream,
/// or the headers meant to be added will be silently ignored, since they can't be sent
/// after content is sent.
///
private static void AddNoCacheHeaders(HttpListenerResponse response)
{
response.Headers.Add("Cache-Control", "no-cache");
response.Headers.Add("Cache-Control", "no-store");
response.Headers.Add("Pragma", "no-cache");
response.Headers.Add("Expires", "Mon, 1 Jan 1990 00:00:00 GMT");
}
///
/// Handle Http GET requests for state endpoint.
///
///
/// Context containing HTTP GET request data, and which will also contain associated
/// response upon return.
///
///
/// Await-able task.
///
private async Task HandleGetStateRequest(HttpListenerContext requestContext)
{
// Don't cache results of any endpoint requests
AddNoCacheHeaders(requestContext.Response);
var responseProperties = new Dictionary();
foreach (var mapEntry in this.streamHandlerMap)
{
var handlerStatus = mapEntry.Value.GetState(mapEntry.Key);
responseProperties.Add(mapEntry.Key, handlerStatus);
}
requestContext.Response.ContentType = JsonContentType;
await responseProperties.DictionaryToJsonAsync(requestContext.Response.OutputStream);
CloseResponse(requestContext, HttpStatusCode.OK);
}
///
/// Handle Http POST requests for state endpoint.
///
///
/// Context containing HTTP POST request data, and which will also contain associated
/// response upon return.
///
///
/// Await-able task.
///
private async Task HandlePostStateRequest(HttpListenerContext requestContext)
{
// Don't cache results of any endpoint requests
AddNoCacheHeaders(requestContext.Response);
Dictionary requestProperties;
try
{
requestProperties = await requestContext.Request.InputStream.DictionaryFromJsonAsync();
}
catch (SerializationException)
{
requestProperties = null;
}
if (requestProperties == null)
{
CloseResponse(requestContext, HttpStatusCode.BadRequest);
return;
}
var responseProperties = new Dictionary();
var errorStreamNames = new List();
foreach (var requestEntry in requestProperties)
{
ISensorStreamHandler handler;
if (!this.streamHandlerMap.TryGetValue(requestEntry.Key, out handler))
{
// Don't process unrecognized handlers
responseProperties.Add(requestEntry.Key, Properties.Resources.StreamNameUnrecognized);
errorStreamNames.Add(requestEntry.Key);
continue;
}
var propertiesToSet = requestEntry.Value as IDictionary;
if (propertiesToSet == null)
{
continue;
}
var propertyErrors = new Dictionary();
var success = handler.SetState(requestEntry.Key, new ReadOnlyDictionary(propertiesToSet), propertyErrors);
if (!success)
{
responseProperties.Add(requestEntry.Key, propertyErrors);
errorStreamNames.Add(requestEntry.Key);
}
}
if (errorStreamNames.Count == 0)
{
responseProperties.Add(SuccessPropertyName, true);
}
else
{
// The only properties returned other than the "success" property are to indicate error,
// so if there are other properties present it means that we've encountered at least
// one error while trying to change state of the streams.
responseProperties.Add(SuccessPropertyName, false);
responseProperties.Add(ErrorsPropertyName, errorStreamNames.ToArray());
}
requestContext.Response.ContentType = JsonContentType;
await responseProperties.DictionaryToJsonAsync(requestContext.Response.OutputStream);
CloseResponse(requestContext, HttpStatusCode.OK);
}
///
/// Handle Http requests for state endpoint.
///
///
/// Context containing HTTP request data, and which will also contain associated
/// response upon return.
///
///
/// Await-able task.
///
private async Task HandleStateRequest(HttpListenerContext requestContext)
{
const string AllowHeader = "Allow";
const string AllowedMethods = "GET, POST, OPTIONS";
switch (requestContext.Request.HttpMethod)
{
case "GET":
await this.HandleGetStateRequest(requestContext);
break;
case "POST":
await this.HandlePostStateRequest(requestContext);
break;
case "OPTIONS":
requestContext.Response.Headers.Set(AllowHeader, AllowedMethods);
CloseResponse(requestContext, HttpStatusCode.OK);
break;
default:
requestContext.Response.Headers.Set(AllowHeader, AllowedMethods);
CloseResponse(requestContext, HttpStatusCode.MethodNotAllowed);
break;
}
}
///
/// Handle Http requests for stream endpoint.
///
///
/// Context containing HTTP request data, and which will also contain associated
/// response upon return.
///
private void HandleStreamRequest(HttpListenerContext requestContext)
{
WebSocketEventChannel.TryOpenAsync(
requestContext,
channel => this.streamChannels.Add(channel),
channel => this.streamChannels.Remove(channel));
}
///
/// Handle Http requests for event endpoint.
///
///
/// Context containing HTTP request data, and which will also contain associated
/// response upon return.
///
private void HandleEventRequest(HttpListenerContext requestContext)
{
WebSocketEventChannel.TryOpenAsync(
requestContext,
channel => this.eventsChannels.Add(channel),
channel => this.eventsChannels.Remove(channel));
}
///
/// Asynchronously send stream message to client(s) of stream handler.
///
///
/// Stream message to send.
///
///
/// Binary payload of stream message. May be null if message does not require a binary
/// payload.
///
///
/// Await-able task.
///
private async Task SendStreamMessageAsync(StreamMessage message, byte[] binaryPayload)
{
var webSocketMessage = message.ToTextMessage();
foreach (var channel in this.streamChannels.SafeCopy())
{
if (channel == null)
{
break;
}
if (binaryPayload != null)
{
// If binary payload is non-null, send two-part stream message, with the first
// part acting as a message header.
await
channel.SendMessagesAsync(
webSocketMessage, new WebSocketMessage(new ArraySegment(binaryPayload), WebSocketMessageType.Binary));
continue;
}
await channel.SendMessagesAsync(webSocketMessage);
}
}
///
/// Asynchronously send event message to client(s) of stream handler.
///
///
/// Event message to send.
///
///
/// Await-able task.
///
private async Task SendEventMessageAsync(EventMessage message)
{
foreach (var channel in this.eventsChannels.SafeCopy())
{
await channel.SendMessagesAsync(message.ToTextMessage());
}
}
///
/// Responds to KinectSensor changes.
///
///
/// New sensor associated with this KinectRequestHandler.
///
private void OnKinectChanged(KinectSensor newSensor)
{
if (this.sensor != null)
{
try
{
this.sensor.ColorFrameReady -= this.SensorColorFrameReady;
this.sensor.DepthFrameReady -= this.SensorDepthFrameReady;
this.sensor.SkeletonFrameReady -= this.SensorSkeletonFrameReady;
this.sensor.DepthStream.Range = DepthRange.Default;
this.sensor.SkeletonStream.EnableTrackingInNearRange = false;
this.sensor.DepthStream.Disable();
this.sensor.SkeletonStream.Disable();
}
catch (InvalidOperationException)
{
// KinectSensor might enter an invalid state while enabling/disabling streams or stream features.
// E.g.: sensor might be abruptly unplugged.
}
this.sensor = null;
this.skeletons = null;
}
if (newSensor != null)
{
// Allocate space to put the skeleton and interaction data we'll receive
this.sensor = newSensor;
try
{
newSensor.DepthStream.Enable(DepthImageFormat.Resolution640x480Fps30);
newSensor.SkeletonStream.Enable();
try
{
newSensor.DepthStream.Range = DepthRange.Near;
newSensor.SkeletonStream.EnableTrackingInNearRange = true;
}
catch (InvalidOperationException)
{
// Non Kinect for Windows devices do not support Near mode, so reset back to default mode.
newSensor.DepthStream.Range = DepthRange.Default;
newSensor.SkeletonStream.EnableTrackingInNearRange = false;
}
this.skeletons = new Skeleton[newSensor.SkeletonStream.FrameSkeletonArrayLength];
newSensor.ColorFrameReady += this.SensorColorFrameReady;
newSensor.DepthFrameReady += this.SensorDepthFrameReady;
newSensor.SkeletonFrameReady += this.SensorSkeletonFrameReady;
}
catch (InvalidOperationException)
{
// KinectSensor might enter an invalid state while enabling/disabling streams or stream features.
// E.g.: sensor might be abruptly unplugged.
}
}
foreach (var handler in this.streamHandlers)
{
handler.OnSensorChanged(newSensor);
}
}
///
/// Handler for KinectSensorChooser's KinectChanged event.
///
/// object sending the event
/// event arguments
private void SensorChooserKinectChanged(object sender, KinectChangedEventArgs args)
{
this.OnKinectChanged(args.NewSensor);
}
///
/// Handler for the Kinect sensor's ColorFrameReady event
///
/// object sending the event
/// event arguments
private void SensorColorFrameReady(object sender, ColorImageFrameReadyEventArgs colorImageFrameReadyEventArgs)
{
// Even though we un-register all our event handlers when the sensor
// changes, there may still be an event for the old sensor in the queue
// due to the way the KinectSensor delivers events. So check again here.
if (this.sensor != sender)
{
return;
}
using (var colorFrame = colorImageFrameReadyEventArgs.OpenColorImageFrame())
{
if (null != colorFrame)
{
try
{
// Hand data to each handler to be processed
foreach (var handler in this.streamHandlers)
{
handler.ProcessColor(colorFrame.GetRawPixelData(), colorFrame);
}
}
catch (InvalidOperationException)
{
// ColorFrame functions may throw when the sensor gets
// into a bad state. Ignore the frame in that case.
}
}
}
}
///
/// Handler for the Kinect sensor's DepthFrameReady event
///
/// object sending the event
/// event arguments
private void SensorDepthFrameReady(object sender, DepthImageFrameReadyEventArgs depthImageFrameReadyEventArgs)
{
// Even though we un-register all our event handlers when the sensor
// changes, there may still be an event for the old sensor in the queue
// due to the way the KinectSensor delivers events. So check again here.
if (this.sensor != sender)
{
return;
}
using (var depthFrame = depthImageFrameReadyEventArgs.OpenDepthImageFrame())
{
if (null != depthFrame)
{
var depthBuffer = depthFrame.GetRawPixelData();
try
{
// Hand data to each handler to be processed
foreach (var handler in this.streamHandlers)
{
handler.ProcessDepth(depthBuffer, depthFrame);
}
}
catch (InvalidOperationException)
{
// DepthFrame functions may throw when the sensor gets
// into a bad state. Ignore the frame in that case.
}
}
}
}
///
/// Handler for the Kinect sensor's SkeletonFrameReady event
///
/// object sending the event
/// event arguments
private void SensorSkeletonFrameReady(object sender, SkeletonFrameReadyEventArgs skeletonFrameReadyEventArgs)
{
// Even though we un-register all our event handlers when the sensor
// changes, there may still be an event for the old sensor in the queue
// due to the way the KinectSensor delivers events. So check again here.
if (this.sensor != sender)
{
return;
}
using (SkeletonFrame skeletonFrame = skeletonFrameReadyEventArgs.OpenSkeletonFrame())
{
if (null != skeletonFrame)
{
try
{
// Copy the skeleton data from the frame to an array used for temporary storage
skeletonFrame.CopySkeletonDataTo(this.skeletons);
foreach (var handler in this.streamHandlers)
{
handler.ProcessSkeleton(this.skeletons, skeletonFrame);
}
}
catch (InvalidOperationException)
{
// SkeletonFrame functions may throw when the sensor gets
// into a bad state. Ignore the frame in that case.
}
}
}
}
}
///
/// Helper class that adds a SafeCopy extension method to a List of WebSocketEventChannel.
///
internal static class ChannelListHelper
{
///
/// Safely makes a copy of the list of stream channels.
///
/// Array containing the stream channels.
public static IEnumerable SafeCopy(this List list)
{
var channels = new WebSocketEventChannel[list.Count];
list.CopyTo(channels);
return channels;
}
}
}