We Keep Coding

Azure App Service- GZip Compression in Request

With this code (without the request compression part) I'm able to get gzip compressed content from Azure App Service (Xamarin.Froms App with offline sync). But when i try to gzip the request http-content i get a "Bad Request".
Any ideas? Is it possible to gzip the request content with Azure App Service?
namespace XXX.XXX.XXX.XXX.XXX
{
public class HttpGZipClientHandler : System.Net.Http.HttpClientHandler
{
long time = 0;
private long _downloadedBytesFromServer;
private long _downloadedProcessedBytes;
private long _intendedUploadedBytesToServer;
private long _uploadedBytesToServer;
private long _additionalTimeOverhead = 0;
public override bool SupportsAutomaticDecompression { get { return true; } }
public long DownloadedBytesFromServer { get { return _downloadedBytesFromServer; } }
public long DownloadedProcessedBytes { get { return _downloadedProcessedBytes; } }
public long IntendedUploadedBytesToServer { get { return _intendedUploadedBytesToServer; } }
public long UploadedBytesToServer { get { return _uploadedBytesToServer; } }
public long AdditionalTimeOverhead { get { return _additionalTimeOverhead; } }
public void ResetStatistics()
{
_downloadedBytesFromServer = 0;
_downloadedProcessedBytes = 0;
_intendedUploadedBytesToServer = 0;
_uploadedBytesToServer = 0;
_additionalTimeOverhead = 0;
}
protected override async Task<System.Net.Http.HttpResponseMessage> SendAsync(System.Net.Http.HttpRequestMessage request, System.Threading.CancellationToken cancellationToken)
{
//Save content headers before compressing
System.Collections.Generic.Dictionary<string, System.Collections.Generic.IEnumerable<string>> savedContentHeaders = new Dictionary<string, IEnumerable<string>>();
foreach (System.Collections.Generic.KeyValuePair<string, System.Collections.Generic.IEnumerable<string>> keyValue in request.Content.Headers)
{
savedContentHeaders.Add(keyValue.Key, keyValue.Value);
}
//Compress request content
System.Diagnostics.Stopwatch sp1 = new System.Diagnostics.Stopwatch();
sp1.Start();
_intendedUploadedBytesToServer += request.Content.Headers.ContentLength.HasValue ? request.Content.Headers.ContentLength.Value : 0;
await request.Content.LoadIntoBufferAsync().ConfigureAwait(false);
request.Content = new HttpGZipContent(await request.Content.ReadAsByteArrayAsync().ConfigureAwait(false), System.IO.Compression.CompressionMode.Compress);
byte[] uploadedBytes = await request.Content.ReadAsByteArrayAsync().ConfigureAwait(false);
_uploadedBytesToServer += uploadedBytes.Length;
sp1.Stop();
_additionalTimeOverhead += sp1.ElapsedMilliseconds;
//Set headers
foreach (System.Collections.Generic.KeyValuePair<string, System.Collections.Generic.IEnumerable<string>> keyValue in savedContentHeaders)
{
request.Content.Headers.Add(keyValue.Key, keyValue.Value);
}
request.Headers.AcceptEncoding.Add(new System.Net.Http.Headers.StringWithQualityHeaderValue("gzip"));
request.Content.Headers.Add("Content-Encoding", "gzip");
//Execute request
System.Net.Http.HttpResponseMessage response = await base.SendAsync(request, cancellationToken).ConfigureAwait(false);
_downloadedBytesFromServer += response.Content.Headers.ContentLength.HasValue ? response.Content.Headers.ContentLength.Value : 0;
//Decompress response content
if (response.Content.Headers.ContentEncoding.Contains("gzip"))
{
System.Diagnostics.Stopwatch sp2 = new System.Diagnostics.Stopwatch();
sp2.Start();
await response.Content.LoadIntoBufferAsync().ConfigureAwait(false);
response.Content = new HttpGZipContent(await response.Content.ReadAsByteArrayAsync().ConfigureAwait(false), System.IO.Compression.CompressionMode.Decompress);
byte[] processedBytes = await response.Content.ReadAsByteArrayAsync().ConfigureAwait(false);
_downloadedProcessedBytes += processedBytes.Length;
sp2.Stop();
_additionalTimeOverhead += sp2.ElapsedMilliseconds;
}
else
_downloadedProcessedBytes += response.Content.Headers.ContentLength.HasValue ? response.Content.Headers.ContentLength.Value : 0;
return response;
}
}
internal sealed class HttpGZipContent : System.Net.Http.HttpContent
{
private readonly byte[] _content;
private readonly System.IO.Compression.CompressionMode _compressionMode;
public HttpGZipContent(byte[] content, System.IO.Compression.CompressionMode compressionMode)
{
_compressionMode = compressionMode;
_content = content;
}
protected override async System.Threading.Tasks.Task SerializeToStreamAsync(System.IO.Stream stream, System.Net.TransportContext context)
{
if (_compressionMode == System.IO.Compression.CompressionMode.Compress)
{
using (System.IO.MemoryStream memoryStream = new System.IO.MemoryStream(_content.Length))
{
using (System.IO.Compression.GZipStream zipStream = new System.IO.Compression.GZipStream(memoryStream, System.IO.Compression.CompressionMode.Compress))
{
zipStream.Write(_content, 0, _content.Length);
zipStream.Flush();
}
byte[] compressed = memoryStream.ToArray();
System.IO.MemoryStream copyStream = new System.IO.MemoryStream(compressed);
await copyStream.CopyToAsync(stream).ConfigureAwait(false);
}
}
else
{
using (System.IO.MemoryStream memoryStream = new System.IO.MemoryStream(_content, 0, _content.Length))
{
using (System.IO.Compression.GZipStream zipStream = new System.IO.Compression.GZipStream(memoryStream, System.IO.Compression.CompressionMode.Decompress))
{
await zipStream.CopyToAsync(stream).ConfigureAwait(false);
}
}
}
}
protected override bool TryComputeLength(out long length)
{
length = _content.Length;
return true;
}
protected override void Dispose(bool disposing)
{
base.Dispose(disposing);
}
}
}

Based on my understanding, you need to implement the request decompression for your mobile app back-end. If you are using the C# backend, you could create your custom ActionFilterAttribute as follows:
public class RequestDeCompressFilter : ActionFilterAttribute
{
public override async Task OnActionExecutingAsync(HttpActionContext actionContext, CancellationToken cancellationToken)
{
var request = actionContext.Request;
if (request.Content.Headers.ContentEncoding.Contains("GZIP"))
{
await request.Content.LoadIntoBufferAsync().ConfigureAwait(false);
request.Content = new HttpGZipContent(await request.Content.ReadAsByteArrayAsync().ConfigureAwait(false), System.IO.Compression.CompressionMode.Decompress);
}
//TODO: compress the response, you could follow http://www.intstrings.com/ramivemula/articles/jumpstart-47-gzipdeflate-compression-in-asp-net-mvc-application/
await base.OnActionExecutingAsync(actionContext, cancellationToken);
}
public override Task OnActionExecutedAsync(HttpActionExecutedContext actionExecutedContext, CancellationToken cancellationToken)
{
//you could also compress the response here
return base.OnActionExecutedAsync(actionExecutedContext, cancellationToken);
}
}
Then, mark your action as follows:
[RequestDeCompressFilter]
public async Task<IHttpActionResult> PostMessage(Message item)
Also, you could follow HTTP Message Handlers in ASP.NET Web API to implement your HTTP message handler.

How to enrich App Insights ExceptionTelemetry with request body in ASP.NET Core

TL;DR Question: Is there a way to get the request body into an existing ExceptionTelemetry instance, in ASP.NET Core, without copying ALL request bodies?
I would like to be able to include the Request Body in the exception telemetry for application insights. I.e. I only want the request when an exception has occurred.
Browsing around for documentation on both ASP.NET Core and Application Insights, it seems the "right" way to enrich telemetry is using TelemetryProcessors or TelemetryInitializers, so I tried getting the request body in a custom telemetryinitializer, only to discover that the request body stream is closed/disposed when I want to read it (rewinding does not help because apparently it has already been disposed when the App Insights telemetryinitializer is being executed).
I ended up solving it by having a middleware that copies the request stream:
public async Task Invoke(HttpContext context)
{
var stream = context.Request.Body;
try
{
using (var buffer = new MemoryStream())
{
// Copy the request stream and rewind the copy
await stream.CopyToAsync(buffer);
buffer.Position = 0L;
// Create another copy and rewind both
var otherBuffer = new MemoryStream();
await buffer.CopyToAsync(otherBuffer);
buffer.Position = 0L;
otherBuffer.Position = 0L;
// Replace the request stream by the first copy
context.Request.Body = buffer;
// Put a separate copy in items collection for other things to use
context.Items["RequestStreamCopy"] = otherBuffer;
context.Response.RegisterForDispose(otherBuffer);
await next(context);
}
}
finally
{
context.Request.Body = stream;
}
}
And my initializer:
public AiExceptionInitializer(IHttpContextAccessor httpContextAccessor)
{
this.httpContextAccessor = httpContextAccessor ?? throw new ArgumentNullException("httpContextAccessor");
}
public void Initialize(ITelemetry telemetry)
{
var context = this.httpContextAccessor.HttpContext;
if (context == null)
{
return;
}
lock (context)
{
var request = context.Features.Get<RequestTelemetry>();
if (request == null)
{
return;
}
this.OnInitializeTelemetry(context, request, telemetry);
}
}
protected void OnInitializeTelemetry(HttpContext platformContext, RequestTelemetry requestTelemetry, ITelemetry telemetry)
{
if (telemetry is ExceptionTelemetry exceptionTelemetry)
{
var stream = platformContext.Items["RequestStreamCopy"] as MemoryStream;
try
{
if (stream?.Length <= 0)
{
return;
}
// Rewind the stream position just to be on the safe side
stream.Position = 0L;
using (var reader = new StreamReader(stream, Encoding.UTF8, true, 1024, true))
{
string requestBody = reader.ReadToEnd();
exceptionTelemetry.Properties.Add("HttpRequestBody", requestBody);
}
}
finally
{
if (stream != null)
{
// Rewind the stream for others to use.
stream.Position = 0L;
}
}
}
}
However this having to copy the request stream (TWICE) for each request, to only have it used on failures seems rather inefficient to me.
So I am wondering if there is any other way to do something like this where I don't have to copy the stream of each and every request just to serialize the ones failing?
I am aware I could "just" write a middleware that would create new ExceptionTelemetry instances, but as far as I know (I might be wrong) it would leave me with two Exception instances in Application Insights (i.e. the one generated by me and the one generated by the AI extensions), instead of just one exception with the added property I need.

Thanks to the comment from #DmitryMatveev I found an alternate solution. I am not sure its the most effective, but it is better than what i had!
The middleware is "reduced" to only tracking exceptions, and then serialising the body right away (you might still have a stream copy, but I don't need it in my case), something like the following:
using System;
using System.IO;
using System.Text;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Http.Internal;
public class ExceptionBodyTrackingMiddleware
{
public const string ExceptionRequestBodyKey = "ExceptionRequestBody";
private readonly RequestDelegate next;
public ExceptionBodyTrackingMiddleware(RequestDelegate next)
{
this.next = next ?? throw new ArgumentNullException(nameof(next));
}
public async Task Invoke(HttpContext context)
{
try
{
context.Request.EnableRewind();
await this.next.Invoke(context);
}
catch (Exception)
{
RegisterRequestBody(context);
throw;
}
}
private static void RegisterRequestBody(HttpContext context)
{
if (context.Request.Body?.CanSeek == false)
{
return;
}
var body = CopyStreamToString(context.Request.Body);
context.Items[ExceptionRequestBodyKey] = body;
}
private static string CopyStreamToString(Stream stream)
{
var originalPosition = stream.Position;
RewindStream(stream);
string requestBody = null;
using (var reader = new StreamReader(stream, Encoding.UTF8, true, 1024, true))
{
requestBody = reader.ReadToEnd();
}
stream.Position = originalPosition;
return requestBody;
}
private static void RewindStream(Stream stream)
{
if (stream != null)
{
stream.Position = 0L;
}
}
}
Likewise the Initializer becomes a whole lot simpler:
public AiExceptionInitializer(IHttpContextAccessor httpContextAccessor)
{
this.httpContextAccessor = httpContextAccessor ?? throw new ArgumentNullException("httpContextAccessor");
}
public void Initialize(ITelemetry telemetry)
{
var context = this.httpContextAccessor.HttpContext;
if (context == null)
{
return;
}
lock (context)
{
var request = context.Features.Get<RequestTelemetry>();
if (request == null)
{
return;
}
this.OnInitializeTelemetry(context, request, telemetry);
}
}
protected void OnInitializeTelemetry(HttpContext platformContext, RequestTelemetry requestTelemetry, ITelemetry telemetry)
{
if (telemetry is ExceptionTelemetry exceptionTelemetry)
{
var requestBody = platformContext.Items[ExceptionBodyTrackingMiddleware.ExceptionRequestBodyKey] as string;
exceptionTelemetry.Properties.Add("HttpRequestBody", requestBody);
}
}

View POST request body in Application Insights

Is it possible to view POST request body in Application Insights?
I can see request details, but not the payload being posted in application insights. Do I have to track this with some coding?
I am building a MVC core 1.1 Web Api.

You can simply implement your own Telemetry Initializer:
For example, below an implementation that extracts the payload and adds it as a custom dimension of the request telemetry:
public class RequestBodyInitializer : ITelemetryInitializer
{
public void Initialize(ITelemetry telemetry)
{
var requestTelemetry = telemetry as RequestTelemetry;
if (requestTelemetry != null && (requestTelemetry.HttpMethod == HttpMethod.Post.ToString() || requestTelemetry.HttpMethod == HttpMethod.Put.ToString()))
{
using (var reader = new StreamReader(HttpContext.Current.Request.InputStream))
{
string requestBody = reader.ReadToEnd();
requestTelemetry.Properties.Add("body", requestBody);
}
}
}
}
Then add it to the configuration either by configuration file or via code:
TelemetryConfiguration.Active.TelemetryInitializers.Add(new RequestBodyInitializer());
Then query it in Analytics:
requests | limit 1 | project customDimensions.body

The solution provided by #yonisha is in my opinion the cleanest one available. However you still need to get your HttpContext in there and for that you need some more code. I have also inserted some comments which are based or taken from code examples above. It is important to reset the position of your request else you will lose its data.
This is my solution that I have tested and gives me the jsonbody:
public class RequestBodyInitializer : ITelemetryInitializer
{
readonly IHttpContextAccessor httpContextAccessor;
public RequestBodyInitializer(IHttpContextAccessor httpContextAccessor)
{
this.httpContextAccessor = httpContextAccessor;
}
public void Initialize(ITelemetry telemetry)
{
if (telemetry is RequestTelemetry requestTelemetry)
{
if ((httpContextAccessor.HttpContext.Request.Method == HttpMethods.Post ||
httpContextAccessor.HttpContext.Request.Method == HttpMethods.Put) &&
httpContextAccessor.HttpContext.Request.Body.CanRead)
{
const string jsonBody = "JsonBody";
if (requestTelemetry.Properties.ContainsKey(jsonBody))
{
return;
}
//Allows re-usage of the stream
httpContextAccessor.HttpContext.Request.EnableRewind();
var stream = new StreamReader(httpContextAccessor.HttpContext.Request.Body);
var body = stream.ReadToEnd();
//Reset the stream so data is not lost
httpContextAccessor.HttpContext.Request.Body.Position = 0;
requestTelemetry.Properties.Add(jsonBody, body);
}
}
}
Then also be sure to add this to your Startup -> ConfigureServices
services.AddSingleton<ITelemetryInitializer, RequestBodyInitializer>();
EDIT:
If you also want to get the response body I found it useful to create a piece of middleware (.NET Core, not sure about Framework). At first I took above approach where you log a response and a request but most of the time you want these together:
public async Task Invoke(HttpContext context)
{
var reqBody = await this.GetRequestBodyForTelemetry(context.Request);
var respBody = await this.GetResponseBodyForTelemetry(context);
this.SendDataToTelemetryLog(reqBody, respBody, context);
}
This awaits both a request and a response. GetRequestBodyForTelemetry is almost identical to the code from the telemetry initializer, except using Task. For the response body I have used the code below, I also excluded a 204 since that leads to a nullref:
public async Task<string> GetResponseBodyForTelemetry(HttpContext context)
{
var originalBody = context.Response.Body;
try
{
using (var memStream = new MemoryStream())
{
context.Response.Body = memStream;
//await the responsebody
await next(context);
if (context.Response.StatusCode == 204)
{
return null;
}
memStream.Position = 0;
var responseBody = new StreamReader(memStream).ReadToEnd();
//make sure to reset the position so the actual body is still available for the client
memStream.Position = 0;
await memStream.CopyToAsync(originalBody);
return responseBody;
}
}
finally
{
context.Response.Body = originalBody;
}
}

Few days back, I got a similar requirement to log the request Body in Application insights with filtering out sensitive input user data from the payload. So sharing my solution. The below solution is developed for ASP.NET Core 2.0 Web API.
ActionFilterAttribute
I've used ActionFilterAttribute from (Microsoft.AspNetCore.Mvc.Filters namespace) which provides the Model via ActionArgument so that by reflection, those properties can be extracted which are marked as sensitive.
public class LogActionFilterAttribute : ActionFilterAttribute
{
private readonly IHttpContextAccessor httpContextAccessor;
public LogActionFilterAttribute(IHttpContextAccessor httpContextAccessor)
{
this.httpContextAccessor = httpContextAccessor;
}
public override async Task OnActionExecutionAsync(ActionExecutingContext context, ActionExecutionDelegate next)
{
if (context.HttpContext.Request.Method == HttpMethods.Post || context.HttpContext.Request.Method == HttpMethods.Put)
{
// Check parameter those are marked for not to log.
var methodInfo = ((Microsoft.AspNetCore.Mvc.Controllers.ControllerActionDescriptor)context.ActionDescriptor).MethodInfo;
var noLogParameters = methodInfo.GetParameters().Where(p => p.GetCustomAttributes(true).Any(t => t.GetType() == typeof(NoLogAttribute))).Select(p => p.Name);
StringBuilder logBuilder = new StringBuilder();
foreach (var argument in context.ActionArguments.Where(a => !noLogParameters.Contains(a.Key)))
{
var serializedModel = JsonConvert.SerializeObject(argument.Value, new JsonSerializerSettings() { ContractResolver = new NoPIILogContractResolver() });
logBuilder.AppendLine($"key: {argument.Key}; value : {serializedModel}");
}
var telemetry = this.httpContextAccessor.HttpContext.Items["Telemetry"] as Microsoft.ApplicationInsights.DataContracts.RequestTelemetry;
if (telemetry != null)
{
telemetry.Context.GlobalProperties.Add("jsonBody", logBuilder.ToString());
}
}
await next();
}
}
The 'LogActionFilterAttribute' is injected in MVC pipeline as Filter.
services.AddMvc(options =>
{
options.Filters.Add<LogActionFilterAttribute>();
});
NoLogAttribute
In above code, NoLogAttribute attribute is used which should be applied on Model/Model's Properties or method parameter to indicate that value should not be logged.
public class NoLogAttribute : Attribute
{
}
NoPIILogContractResolver
Also, NoPIILogContractResolver is used in JsonSerializerSettings during serialization process
internal class NoPIILogContractResolver : DefaultContractResolver
{
protected override IList<JsonProperty> CreateProperties(Type type, MemberSerialization memberSerialization)
{
var properties = new List<JsonProperty>();
if (!type.GetCustomAttributes(true).Any(t => t.GetType() == typeof(NoLogAttribute)))
{
IList<JsonProperty> retval = base.CreateProperties(type, memberSerialization);
var excludedProperties = type.GetProperties().Where(p => p.GetCustomAttributes(true).Any(t => t.GetType() == typeof(NoLogAttribute))).Select(s => s.Name);
foreach (var property in retval)
{
if (excludedProperties.Contains(property.PropertyName))
{
property.PropertyType = typeof(string);
property.ValueProvider = new PIIValueProvider("PII Data");
}
properties.Add(property);
}
}
return properties;
}
}
internal class PIIValueProvider : IValueProvider
{
private object defaultValue;
public PIIValueProvider(string defaultValue)
{
this.defaultValue = defaultValue;
}
public object GetValue(object target)
{
return this.defaultValue;
}
public void SetValue(object target, object value)
{
}
}
PIITelemetryInitializer
To inject the RequestTelemetry object, I've to use ITelemetryInitializer so that RequestTelemetry can be retrieved in LogActionFilterAttribute class.
public class PIITelemetryInitializer : ITelemetryInitializer
{
IHttpContextAccessor httpContextAccessor;
public PIITelemetryInitializer(IHttpContextAccessor httpContextAccessor)
{
this.httpContextAccessor = httpContextAccessor;
}
public void Initialize(ITelemetry telemetry)
{
if (this.httpContextAccessor.HttpContext != null)
{
if (telemetry is Microsoft.ApplicationInsights.DataContracts.RequestTelemetry)
{
this.httpContextAccessor.HttpContext.Items.TryAdd("Telemetry", telemetry);
}
}
}
}
The PIITelemetryInitializer is registered as
services.AddSingleton<ITelemetryInitializer, PIITelemetryInitializer>();
Testing feature
Following code demonstrates the usage of above code
Created a controller
[Route("api/[controller]")]
public class ValuesController : Controller
{
private readonly ILogger _logger;
public ValuesController(ILoggerFactory loggerFactory)
{
_logger = loggerFactory.CreateLogger<ValuesController>();
}
// POST api/values
[HttpPost]
public void Post([FromBody, NoLog]string value)
{
}
[HttpPost]
[Route("user")]
public void AddUser(string id, [FromBody]User user)
{
}
}
Where User Model is defined as
public class User
{
[NoLog]
public string Id { get; set; }
public string Name { get; set; }
public DateTime AnneviseryDate { get; set; }
[NoLog]
public int LinkId { get; set; }
public List<Address> Addresses { get; set; }
}
public class Address
{
public string AddressLine { get; set; }
[NoLog]
public string City { get; set; }
[NoLog]
public string Country { get; set; }
}
So when API is invoked by Swagger tool
The jsonBody is logged in Request without sensitive data. All sensitive data is replaced by 'PII Data' string literal.

Update: I have put the logic below into a ready-to-use NuGet package. You can find more about the package here and about the topic itself here.
I choose the custom middleware path as it made things easier with HttpContext already being there.
public class RequestBodyLoggingMiddleware : IMiddleware
{
public async Task InvokeAsync(HttpContext context, RequestDelegate next)
{
var method = context.Request.Method;
// Ensure the request body can be read multiple times
context.Request.EnableBuffering();
// Only if we are dealing with POST or PUT, GET and others shouldn't have a body
if (context.Request.Body.CanRead && (method == HttpMethods.Post || method == HttpMethods.Put))
{
// Leave stream open so next middleware can read it
using var reader = new StreamReader(
context.Request.Body,
Encoding.UTF8,
detectEncodingFromByteOrderMarks: false,
bufferSize: 512, leaveOpen: true);
var requestBody = await reader.ReadToEndAsync();
// Reset stream position, so next middleware can read it
context.Request.Body.Position = 0;
// Write request body to App Insights
var requestTelemetry = context.Features.Get<RequestTelemetry>();
requestTelemetry?.Properties.Add("RequestBody", requestBody);
}
// Call next middleware in the pipeline
await next(context);
}
}
And this is how I log the response body
public class ResponseBodyLoggingMiddleware : IMiddleware
{
public async Task InvokeAsync(HttpContext context, RequestDelegate next)
{
var originalBodyStream = context.Response.Body;
try
{
// Swap out stream with one that is buffered and suports seeking
using var memoryStream = new MemoryStream();
context.Response.Body = memoryStream;
// hand over to the next middleware and wait for the call to return
await next(context);
// Read response body from memory stream
memoryStream.Position = 0;
var reader = new StreamReader(memoryStream);
var responseBody = await reader.ReadToEndAsync();
// Copy body back to so its available to the user agent
memoryStream.Position = 0;
await memoryStream.CopyToAsync(originalBodyStream);
// Write response body to App Insights
var requestTelemetry = context.Features.Get<RequestTelemetry>();
requestTelemetry?.Properties.Add("ResponseBody", responseBody);
}
finally
{
context.Response.Body = originalBodyStream;
}
}
}
Than add an extension method...
public static class ApplicationInsightExtensions
{
public static IApplicationBuilder UseRequestBodyLogging(this IApplicationBuilder builder)
{
return builder.UseMiddleware<RequestBodyLoggingMiddleware>();
}
public static IApplicationBuilder UseResponseBodyLogging(this IApplicationBuilder builder)
{
return builder.UseMiddleware<ResponseBodyLoggingMiddleware>();
}
}
...that allows for a clean integration inside Startup.cs
public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
{
if (env.IsDevelopment())
{
app.UseDeveloperExceptionPage();
// Enable our custom middleware
app.UseRequestBodyLogging();
app.UseResponseBodyLogging();
}
// ...
}
Don't forget to register the custom middleware components inside ConfigureServices()
public void ConfigureServices(IServiceCollection services)
{
// ...
services.AddApplicationInsightsTelemetry(Configuration["APPINSIGHTS_CONNECTIONSTRING"]);
services.AddTransient<RequestBodyLoggingMiddleware>();
services.AddTransient<ResponseBodyLoggingMiddleware>();
}

I never got #yonisha's answer working so I used a DelegatingHandler instead:
public class MessageTracingHandler : DelegatingHandler
{
protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
// Trace the request
await TraceRequest(request);
// Execute the request
var response = await base.SendAsync(request, cancellationToken);
// Trace the response
await TraceResponse(response);
return response;
}
private async Task TraceRequest(HttpRequestMessage request)
{
try
{
var requestTelemetry = HttpContext.Current?.GetRequestTelemetry();
var requestTraceInfo = request.Content != null ? await request.Content.ReadAsByteArrayAsync() : null;
var body = requestTraceInfo.ToString();
if (!string.IsNullOrWhiteSpace(body) && requestTelemetry != null)
{
requestTelemetry.Properties.Add("Request Body", body);
}
}
catch (Exception exception)
{
// Log exception
}
}
private async Task TraceResponse(HttpResponseMessage response)
{
try
{
var requestTelemetry = HttpContext.Current?.GetRequestTelemetry();
var responseTraceInfo = response.Content != null ? await response.Content.ReadAsByteArrayAsync() : null;
var body = responseTraceInfo.ToString();
if (!string.IsNullOrWhiteSpace(body) && requestTelemetry != null)
{
requestTelemetry.Properties.Add("Response Body", body);
}
}
catch (Exception exception)
{
// Log exception
}
}
}
.GetRequestTelemetry() is an extension method from Microsoft.ApplicationInsights.Web.

In Asp.Net core it looks like we dont have to use ITelemetryInitializer. We can use the middleware to log the requests to application insights. Thanks to #IanKemp https://github.com/microsoft/ApplicationInsights-aspnetcore/issues/686
public async Task Invoke(HttpContext httpContext)
{
var requestTelemetry = httpContext.Features.Get<RequestTelemetry>();
//Handle Request
var request = httpContext.Request;
if (request?.Body?.CanRead == true)
{
request.EnableBuffering();
var bodySize = (int)(request.ContentLength ?? request.Body.Length);
if (bodySize > 0)
{
request.Body.Position = 0;
byte[] body;
using (var ms = new MemoryStream(bodySize))
{
await request.Body.CopyToAsync(ms);
body = ms.ToArray();
}
request.Body.Position = 0;
if (requestTelemetry != null)
{
var requestBodyString = Encoding.UTF8.GetString(body);
requestTelemetry.Properties.Add("RequestBody", requestBodyString);
}
}
}
await _next(httpContext); // calling next middleware
}

I implemented a middleware for this,
Invoke method does,
if (context.Request.Method == "POST" || context.Request.Method == "PUT")
{
var bodyStr = GetRequestBody(context);
var telemetryClient = new TelemetryClient();
var traceTelemetry = new TraceTelemetry
{
Message = bodyStr,
SeverityLevel = SeverityLevel.Verbose
};
//Send a trace message for display in Diagnostic Search.
telemetryClient.TrackTrace(traceTelemetry);
}
Where, GetRequestBody is like,
private static string GetRequestBody(HttpContext context)
{
var bodyStr = "";
var req = context.Request;
//Allows using several time the stream in ASP.Net Core.
req.EnableRewind();
//Important: keep stream opened to read when handling the request.
using (var reader = new StreamReader(req.Body, Encoding.UTF8, true, 1024, true))
{
bodyStr = reader.ReadToEnd();
}
// Rewind, so the core is not lost when it looks the body for the request.
req.Body.Position = 0;
return bodyStr;
}

I can able to log the request message body in Application Insights using #yonisha method but I can't able to log the response message body. I am interested in logging the response message body. I am already logging the Post, Put, Delete Request message body using #yonisha method.
When I tried to access the response body in the TelemetryInitializer I keep getting an exception with an error message saying that "stream was not readable. When I researched more I found that AzureInitializer is running as part of HttpModule(ApplicationInsightsWebTracking) so by the time it gets control response object is disposed.
I got an idea from #Oskar answer. Why not have a delegate handler and record the response since the response object is not disposed at the stage of message handler. The message handler is part of the Web API life cycle i.e. similar to the HTTP module but confined to web API. When I developed and tested this idea, fortunately, It worked I recorded the response in the request message using message handler and retrieved it at the AzureInitializer (HTTP module whose execution happens later than the message handler). Here is the sample code.
public class AzureRequestResponseInitializer : ITelemetryInitializer
{
public void Initialize(ITelemetry telemetry)
{
var requestTelemetry = telemetry as RequestTelemetry;
if (requestTelemetry != null && HttpContext.Current != null && HttpContext.Current.Request != null)
{
if ((HttpContext.Current.Request.HttpMethod == HttpMethod.Post.ToString()
|| HttpContext.Current.Request.HttpMethod == HttpMethod.Put.ToString()) &&
HttpContext.Current.Request.Url.AbsoluteUri.Contains("api"))
using (var reader = new StreamReader(HttpContext.Current.Request.InputStream))
{
HttpContext.Current.Request.InputStream.Position = 0;
string requestBody = reader.ReadToEnd();
if (requestTelemetry.Properties.Keys.Contains("requestbody"))
{
requestTelemetry.Properties["requestbody"] = requestBody;
}
else
{
requestTelemetry.Properties.Add("requestbody", requestBody);
}
}
else if (HttpContext.Current.Request.HttpMethod == HttpMethod.Get.ToString()
&& HttpContext.Current.Response.ContentType.Contains("application/json"))
{
var netHttpRequestMessage = HttpContext.Current.Items["MS_HttpRequestMessage"] as HttpRequestMessage;
if (netHttpRequestMessage.Properties.Keys.Contains("responsejson"))
{
var responseJson = netHttpRequestMessage.Properties["responsejson"].ToString();
if (requestTelemetry.Properties.Keys.Contains("responsebody"))
{
requestTelemetry.Properties["responsebody"] = responseJson;
}
else
{
requestTelemetry.Properties.Add("responsebody", responseJson);
}
}
}
}
}
}
config.MessageHandlers.Add(new LoggingHandler());
public class LoggingHandler : DelegatingHandler
{
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
return base.SendAsync(request, cancellationToken).ContinueWith(task =>
{
var response = task.Result;
StoreResponse(response);
return response;
});
}
private void StoreResponse(HttpResponseMessage response)
{
var request = response.RequestMessage;
(response.Content ?? new StringContent("")).ReadAsStringAsync().ContinueWith(x =>
{
var ctx = request.Properties["MS_HttpContext"] as HttpContextWrapper;
if (request.Properties.ContainsKey("responseJson"))
{
request.Properties["responsejson"] = x.Result;
}
else
{
request.Properties.Add("responsejson", x.Result);
}
});
}
}

The solution provided by yonisha is clean, but it does not work for me in .Net Core 2.0. This works if you have a JSON body:
public IActionResult MyAction ([FromBody] PayloadObject payloadObject)
{
//create a dictionary to store the json string
var customDataDict = new Dictionary<string, string>();
//convert the object to a json string
string activationRequestJson = JsonConvert.SerializeObject(
new
{
payloadObject = payloadObject
});
customDataDict.Add("body", activationRequestJson);
//Track this event, with the json string, in Application Insights
telemetryClient.TrackEvent("MyAction", customDataDict);
return Ok();
}

I am sorry, #yonisha's solution does not seem to work in .NET 4.7. The Application Insights part works OK, but there is actually no simple way to get the request body inside the telemetry initializer in .NET 4.7. .NET 4.7 uses GetBufferlessInputStream() to get the stream, and this stream is "read once". One potential code is like this:
private static void LogRequestBody(ISupportProperties requestTelemetry)
{
var requestStream = HttpContext.Current?.Request?.GetBufferlessInputStream();
if (requestStream?.Length > 0)
using (var reader = new StreamReader(requestStream))
{
string body = reader.ReadToEnd();
requestTelemetry.Properties["body"] = body.Substring(0, Math.Min(body.Length, 8192));
}
}
But the return from GetBufferlessInputStream() is already consumed, and does not support seeking. Therefore, the body will always be an empty string.

How to play a video stream that requires authentication?

I have a Windows Store app (C#/XAML) which communicates with a REST service. At some point, I need to play a video stream provided by this service.
If I just assign the stream URI to the MediaElement.Source property, it doesn't work, because the request needs to be authenticated. I need to customize the request sent by the MediaElement control in order to add cookies, credentials and some other custom headers, but I can't find any method or property to do this.
How can I do it? Is it even possible?

OK, I got it working. Basically, the solution has 2 parts:
make the HTTP request manually (with any required credentials or headers)
wrap the response stream in a custom IRandomAccessStream that implements Seek by making another request to the server, using the Range header to specify which part of the stream I need.
Here's the RandomAccessStream implementation:
delegate Task<Stream> AsyncRangeDownloader(ulong start, ulong? end);
class StreamingRandomAccessStream : IRandomAccessStream
{
private readonly AsyncRangeDownloader _downloader;
private readonly ulong _size;
public StreamingRandomAccessStream(Stream startStream, AsyncRangeDownloader downloader, ulong size)
{
if (startStream != null)
_stream = startStream.AsInputStream();
_downloader = downloader;
_size = size;
}
private IInputStream _stream;
private ulong _requestedPosition;
public void Dispose()
{
if (_stream != null)
_stream.Dispose();
}
public IAsyncOperationWithProgress<IBuffer, uint> ReadAsync(IBuffer buffer, uint count, InputStreamOptions options)
{
return AsyncInfo.Run<IBuffer, uint>(async (cancellationToken, progress) =>
{
progress.Report(0);
if (_stream == null)
{
var netStream = await _downloader(_requestedPosition, null);
_stream = netStream.AsInputStream();
}
var result = await _stream.ReadAsync(buffer, count, options).AsTask(cancellationToken, progress);
return result;
});
}
public void Seek(ulong position)
{
if (_stream != null)
_stream.Dispose();
_requestedPosition = position;
_stream = null;
}
public bool CanRead { get { return true; } }
public bool CanWrite { get { return false; } }
public ulong Size { get { return _size; } set { throw new NotSupportedException(); } }
public IAsyncOperationWithProgress<uint, uint> WriteAsync(IBuffer buffer) { throw new NotSupportedException(); }
public IAsyncOperation<bool> FlushAsync() { throw new NotSupportedException(); }
public IInputStream GetInputStreamAt(ulong position) { throw new NotSupportedException(); }
public IOutputStream GetOutputStreamAt(ulong position) { throw new NotSupportedException(); }
public IRandomAccessStream CloneStream() { throw new NotSupportedException(); }
public ulong Position { get { throw new NotSupportedException(); } }
}
It can be used like this:
private HttpClient _client;
private void InitClient()
{
_client = new HttpClient();
// Configure the client as needed with CookieContainer, Credentials, etc
// ...
}
private async Task StartVideoStreamingAsync(Uri uri)
{
var request = new HttpRequestMessage(HttpMethod.Get, uri);
// Add required headers
// ...
var response = await _client.SendAsync(request);
ulong length = (ulong)response.Content.Headers.ContentLength;
string mimeType = response.Content.Headers.ContentType.MediaType;
Stream responseStream = await response.Content.ReadAsStreamAsync();
// Delegate that will fetch a stream for the specified range
AsyncRangeDownloader downloader = async (start, end) =>
{
var request2 = new HttpRequestMessage();
request2.Headers.Range = new RangeHeaderValue((long?)start, (long?)end);
// Add other required headers
// ...
var response2 = await _client.SendAsync(request2);
return await response2.Content.ReadAsStreamAsync();
};
var videoStream = new StreamingRandomAccessStream(responseStream, downloader, length);
_mediaElement.SetSource(videoStream, mimeType);
}
The user can seek to an arbitrary position in the video, and the stream will issue another request to get the stream at the specified position.
It's still more complex than I think it should be, but it works...
Note that the server must support the Range header in requests, and must issue the Content-Length header in the initial response.

HttpResponseMessage called twice for content type image/jpeg

My machine is hosting a RESTful service on MS WebAPI (.net4) and is also connected to a camera. I can make a request to have the camera take a snapshot (jpg) and show the captured image.
Strange thing is, the request is always called 2x - 2 images are captured but only the last image is returned as output. (using google chrome postman to test)
in my server:
var config = new HttpSelfHostConfiguration(string.Format("http://localhost:{0}/", port));
config.MessageHandlers.Add(new SimpleFileHandler());
config.Formatters.Add(new JpegTypeFormatter());
config.Routes.MapHttpRoute( /* the route map */
in SimpleFileHandler
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
return base.SendAsync(request, cancellationToken);
}
A jpeg content type formatter called JpegMediaFormatter:
public class JpegTypeFormatter : MediaTypeFormatter
{
private static Type _supportedType = typeof(MemoryStream);
public JpegTypeFormatter()
{
SupportedMediaTypes.Add(new MediaTypeHeaderValue(MediaTypeNames.Image.Jpeg));
}
public override bool CanReadType(Type type)
{
return type == _supportedType;
}
public override bool CanWriteType(Type type)
{
return type == _supportedType;
}
public override Task<object> ReadFromStreamAsync(Type type, Stream readStream, HttpContent content, IFormatterLogger formatterLogger)
{
throw new NotImplementedException();
}
private Task GetWriteTask(Stream stream, MemoryStream data)
{
return new Task(() =>
{
var ms = new MemoryStream(data.ToArray());
ms.CopyTo(stream);
});
}
public override Task WriteToStreamAsync(Type type, object value, Stream writeStream, HttpContent content, TransportContext transportContext)
{
if (value == null)
{
//value = new byte[0];
value = new MemoryStream();
}
Task writeTask = GetWriteTask(writeStream, (MemoryStream)value);
writeTask.Start();
return writeTask;
}
}
Camera capture RESTful call:
[HttpGet]
public HttpResponseMessage Capture(int width)
{ // call camera API, capture image and save as img
var memoryStream = new MemoryStream();
img.Save(memoryStream, ImageFormat.Jpeg);
return Request.CreateResponse(HttpStatusCode.OK, memoryStream,
new MediaTypeHeaderValue(MediaTypeNames.Image.Jpeg));
}
Somehow calling new MediaTypeHeaderValue() with JPEG content will cause the GET to be called twice.
If I change to call some other MediaType i.e. application/json, the GET is not called 2x.
Any ideas why this is happening?
Edit: additional details
server code:
public Server : IDisposable
{
HttpSelfHostServer _server;
public Server()
{
var config = new HttpSelfHostConfiguration("http://localhost:30019");
config.Routes.MapHttpRoute("capture", "camera/capture/");
_server = new HttpSelfHostServer(config);
_server.OpenAsync().Wait();
}
...
}
Camera controller:
public class CameraController : ApiController
{
// camera -> variable to camera h/w
var camera = HW.Camera;
var img = camera.Capture();
if(img != null)
{
var memoryStream = new MemoryStream();
img.Save(memoryStream, ImageFormat.Jpeg);
var resp = new HttpResponseMessage
{
StatusCode = HttpStatusCode.OK;
Content = new PushStreamContent((respStream, cnt, ctx) =>
{
using(respStream)
{
memoryStream.WriteTo(respStream);
}
});
};
resp.Content.Headers.ContentType = new MediaTypeHeaderValue("image/jpeg");
return resp;
}
}

I am trying to understand your requirement...what type is 'img'? and also i think you can avoid creating a custom formatter..example below:
[HttpGet]
public HttpResponseMessage Capture(int width)
{
HttpResponseMessage response = new HttpResponseMessage();
response.Content = new PushStreamContent((responseStream, httpContent, transportContext) =>
{
using (responseStream)
{
img.Save(responseStream, ImageFormat.Jpeg);
}//closing this important!
});
response.Content.Headers.ContentType = new MediaTypeHeaderValue("image/jpeg");
return response;
}
if you have an image already as a file, then you could just do the following to respond back with an image:
[HttpGet]
public HttpResponseMessage Capture(int width)
{
HttpResponseMessage response = new HttpResponseMessage();
response.Content = new StreamContent(File.OpenRead(#"C:\Images\Car.jpg"));
response.Content.Headers.ContentType = new MediaTypeHeaderValue("image/jpeg");
return response;
}