We Keep Coding

Natural Linking of Request and Response logs .NET Core app

I am trying to link my request and response logs by GUID. Either i get the same GUID for every request and response, or i get a completely different one fore every request and response. What i need is for the linked Requests and responses to have the Same GUID, but be different to all other requests an responses. Is it even possible to do this with my current structure?
public class RequestResponseLoggingMiddleware
{
private readonly RequestDelegate _next;
private readonly ILogger _logger;
private readonly RecyclableMemoryStreamManager _recyclableMemoryStreamManager;
public RequestResponseLoggingMiddleware(RequestDelegate next, ILoggerFactory loggerFactory)
{
_next = next;
_logger = loggerFactory.CreateLogger<RequestResponseLoggingMiddleware>();
_recyclableMemoryStreamManager = new RecyclableMemoryStreamManager();
}
public async Task Invoke(HttpContext context)
{
await LogRequest(context);
await LogResponse(context);
}
private async Task LogRequest(HttpContext context)
{
context.Request.EnableBuffering();
await using var requestStream = _recyclableMemoryStreamManager.GetStream();
await context.Request.Body.CopyToAsync(requestStream);
var Request = new Dictionary<string, string>(){
{"Http Request Information", $"{DateTime.UtcNow}"},
{"ID", Guid.NewGuid().ToString()},
{"IP", $"{context.Request.HttpContext.Connection.RemoteIpAddress}" },
{"Schema", context.Request.Scheme},
{"Path", context.Request.Path},
{"QueryString", $"{context.Request.QueryString}"},
{"Request Body", ReadStreamInChunks(requestStream)}
};
var requestJson = JsonConvert.SerializeObject(Request, Formatting.None);
_logger.LogInformation(requestJson);
context.Request.Body.Position = 0;
}
private async Task LogResponse(HttpContext context)
{
var originalBodyStream = context.Response.Body;
await using var responseBody = _recyclableMemoryStreamManager.GetStream();
context.Response.Body = responseBody;
await _next(context);
context.Response.Body.Seek(0, SeekOrigin.Begin);
var text = await new StreamReader(context.Response.Body).ReadToEndAsync();
context.Response.Body.Seek(0, SeekOrigin.Begin);
var Response = new Dictionary<string, string>(){
{"Http Response Information", $"{DateTime.UtcNow}"},
{"ID", Guid.NewGuid().ToString()},
{"IP", $"{context.Request.HttpContext.Connection.RemoteIpAddress}" },
{"Schema", context.Request.Scheme},
{"Path", context.Request.Path },
{"QueryString", $"{context.Request.QueryString}"},
{"Response Body", text}
};
var responseJson = JsonConvert.SerializeObject(Response, Formatting.None);
_logger.LogInformation(responseJson);
await responseBody.CopyToAsync(originalBodyStream);
}
private static string ReadStreamInChunks(Stream stream)
{
const int readChunkBufferLength = 4096;
stream.Seek(0, SeekOrigin.Begin);
using var textWriter = new StringWriter();
using var reader = new StreamReader(stream);
var readChunk = new char[readChunkBufferLength];
int readChunkLength;
do
{
readChunkLength = reader.ReadBlock(readChunk, 0, readChunkBufferLength);
textWriter.Write(readChunk, 0, readChunkLength);
} while (readChunkLength > 0);
return textWriter.ToString();
}

I'd extend request and response json objects and add there some ID fields and fill the with the same Guid:
public class RequestResponseLoggingMiddleware
{
private readonly RequestDelegate _next;
private readonly ILogger _logger;
private readonly RecyclableMemoryStreamManager _recyclableMemoryStreamManager;
public RequestResponseLoggingMiddleware(RequestDelegate next, ILoggerFactory loggerFactory)
{
_next = next;
_logger = loggerFactory.CreateLogger<RequestResponseLoggingMiddleware>();
_recyclableMemoryStreamManager = new RecyclableMemoryStreamManager();
}
public async Task Invoke(HttpContext context)
{
var guid = Guid.NewGuid();
await LogRequest(context, guid);
await LogResponse(context, guid);
}
private async Task LogRequest(HttpContext context, Guid guid)
{
context.Request.EnableBuffering();
await using var requestStream = _recyclableMemoryStreamManager.GetStream();
await context.Request.Body.CopyToAsync(requestStream);
var Request = new Dictionary<string, string>(){
{"Request ID", guid.ToString()}
{"Http Request Information", $"{DateTime.UtcNow}"},
{"ID", Guid.NewGuid().ToString()},
{"IP", $"{context.Request.HttpContext.Connection.RemoteIpAddress}" },
{"Schema", context.Request.Scheme},
{"Path", context.Request.Path},
{"QueryString", $"{context.Request.QueryString}"},
{"Request Body", ReadStreamInChunks(requestStream)}
};
var requestJson = JsonConvert.SerializeObject(Request, Formatting.None);
_logger.LogInformation(requestJson);
context.Request.Body.Position = 0;
}
private async Task LogResponse(HttpContext context, Guid guid)
{
var originalBodyStream = context.Response.Body;
await using var responseBody = _recyclableMemoryStreamManager.GetStream();
context.Response.Body = responseBody;
await _next(context);
context.Response.Body.Seek(0, SeekOrigin.Begin);
var text = await new StreamReader(context.Response.Body).ReadToEndAsync();
context.Response.Body.Seek(0, SeekOrigin.Begin);
var Response = new Dictionary<string, string>(){
{"Request ID", guid.ToString()}
{"Http Response Information", $"{DateTime.UtcNow}"},
{"ID", Guid.NewGuid().ToString()},
{"IP", $"{context.Request.HttpContext.Connection.RemoteIpAddress}" },
{"Schema", context.Request.Scheme},
{"Path", context.Request.Path },
{"QueryString", $"{context.Request.QueryString}"},
{"Response Body", text}
};
var responseJson = JsonConvert.SerializeObject(Response, Formatting.None);
_logger.LogInformation(responseJson);
await responseBody.CopyToAsync(originalBodyStream);
}
private static string ReadStreamInChunks(Stream stream)
{
const int readChunkBufferLength = 4096;
stream.Seek(0, SeekOrigin.Begin);
using var textWriter = new StringWriter();
using var reader = new StreamReader(stream);
var readChunk = new char[readChunkBufferLength];
int readChunkLength;
do
{
readChunkLength = reader.ReadBlock(readChunk, 0, readChunkBufferLength);
textWriter.Write(readChunk, 0, readChunkLength);
} while (readChunkLength > 0);
return textWriter.ToString();
}

I am not sure if there's a specific principle you "link" your requests and responses, but assuming it's stock standard asp.net core app and you work within user request, I believe it is possible to inject scoped services into your middleware.
So, suppose you create a service with something these lines:
class GuidProvider {
public string CorrelationId {get; private set;}
public GuidProvider() {
CorrelationId = Guid.NewGuid().ToString()
}
}
you'd register it in Startup:
services.AddScoped(typeof(GuidProvider));
and inject into your middleware constructor:
public RequestResponseLoggingMiddleware(..., GuidProvider guidProvider)
{
guidProvider.CorrelationId; // should stay the same within request
...
}```

how to retrieve actual error message in case of Bad Request using OwinContext middleware in .Net Web Api 2?

When .Net web API returns Badrequest("this is an example of Badrequest".), IOwinContext object in my middleware only contains context.Response.StatusCode 400 and context.Response.ReasonPhrase as "Bad Request". I want actual error message so that I can log it somewhere. Is it possible to get actual error message from IOwinContext without writing any custom class?
EDIT:
Startup.cs
public class Startup
{
public void Configuration(IAppBuilder app)
{
ICoreLogger dv_logger = new CoreLogger();
app.Use<InvalidAuthenticationMiddleware>(dv_logger);
ConfigureOAuth(app);
//register log4net
XmlConfigurator.Configure();
// configure log4net variables
GlobalContext.Properties["processId"] = "dv_logger";
System.Web.Http.GlobalConfiguration.Configure(WebApiConfig.Register);
RouteConfig.RegisterRoutes(RouteTable.Routes);
//entity framework
DbContext.Intialize();
AuthContext.Intialize();
.....
}
Middleware
public class InvalidAuthenticationMiddleware : OwinMiddleware
{
public override async Task Invoke(IOwinContext context)
{
var stream = context.Response.Body;
using (var buffer = new MemoryStream())
{
context.Response.Body = buffer;
await Next.Invoke(context);
buffer.Seek(0, SeekOrigin.Begin);
using (var reader = new StreamReader(buffer))
{
string responseBody = await reader.ReadToEndAsync();
if (context.Response.StatusCode == (int)HttpStatusCode.BadRequest)
{
var definition = new { Message = "" };
var error = JsonConvert.DeserializeAnonymousType(responseBody, definition);
Debug.WriteLine(error.Message);
}
buffer.Seek(0, SeekOrigin.Begin);
await buffer.CopyToAsync(stream);
}
}
}
}
Controller
public class LoginController : BaseController
{
[Route("")]
public IHttpActionResult Get(string email)
{
return BadRequest("this is an example of bad request!");
}
}

You will have to read the response's body. BadRequest("this is an example of BadRequest") sets an object in the body of the response with a property Message containing that message. I am assuming you are using JSON as a serialization format:
{
"Message": "this is an example of BadRequest"
}
Here is the middleware code to log the error message when the response is a BadRequest:
public override async Task Invoke(IOwinContext context)
{
var stream = context.Response.Body;
using (var buffer = new MemoryStream())
{
context.Response.Body = buffer;
await next.Invoke();
buffer.Seek(0, SeekOrigin.Begin);
using (var reader = new StreamReader(buffer))
{
string responseBody = await reader.ReadToEndAsync();
if (context.Response.StatusCode == (int)HttpStatusCode.BadRequest)
{
var definition = new { Message = "" };
var error = JsonConvert.DeserializeAnonymousType(responseBody, definition);
Console.WriteLine(error.Message);
}
buffer.Seek(0, SeekOrigin.Begin);
await buffer.CopyToAsync(stream);
}
}
}
Note that I used an anonymous type to avoid having to declare a class for the error message object.

System.ArgumentException HResult=0x80070057 Message=Stream is now writable Parameter name: stream

I have asp.net core 2 web api endpoint along with custom middleware. In order to accomplish the centralized exception handling and request validation for the project I referred the link: https://www.strathweb.com/2018/07/centralized-exception-handling-and-request-validation-in-asp-net-core/
Based on the reference I implemented the following code :
// Startup
public class Startup
{
public void Configure(IApplicationBuilder app, IHostingEnvironment env)
{
if (env.IsDevelopment())
{
app.UseDeveloperExceptionPage();
}
else
{
app.UseHsts();
}
app.UseGraphiQl();
app.UseHttpsRedirection();
app.UseExceptionHandler(errorApp =>
{
errorApp.Run(async context =>
{
var errorFeature = context.Features.Get<IExceptionHandlerFeature>();
var exception = errorFeature.Error;
// the IsTrusted() extension method doesn't exist and
// you should implement your own as you may want to interpret it differently
// i.e. based on the current principal
var problemDetails = new ProblemDetails{Instance = $"urn:myorganization:error:{Guid.NewGuid()}"};
if (exception is BadHttpRequestException badHttpRequestException)
{
problemDetails.Title = "Invalid request";
problemDetails.Status = (int)typeof (BadHttpRequestException).GetProperty("StatusCode", BindingFlags.NonPublic | BindingFlags.Instance).GetValue(badHttpRequestException);
problemDetails.Detail = badHttpRequestException.Message;
}
else
{
problemDetails.Title = "An unexpected error occurred!";
problemDetails.Status = 500;
problemDetails.Detail = "Testing"; //exception.Demystify().ToString();
}
// log the exception etc..
context.Response.StatusCode = problemDetails.Status.Value;
context.Response.WriteJson(problemDetails, "application/problem+json");
}
);
}
);
app.UseCommonResponser();
app.UseMvc();
}
}
// CommonResponseMiddleware
public class CommonResponseMiddleware
{
private readonly RequestDelegate _next;
private ICommonService _commonService;
public CommonResponseMiddleware(RequestDelegate next, ICommonService commonService)
{
_next = next;
_commonService = commonService;
}
public async Task Invoke(HttpContext context)
{
var currentBody = context.Response.Body;
using (var memoryStream = new MemoryStream())
{
context.Response.Body = memoryStream;
await _next.Invoke(context);
context.Response.Body = currentBody;
memoryStream.Seek(0, SeekOrigin.Begin);
var readToEnd = new StreamReader(memoryStream).ReadToEnd();
}
}
}
// GraphQLController
[Route("[controller]")]
[ApiController]
public class GraphQLController : Controller
{
private readonly IDocumentExecuter _documentExecuter;
private readonly ISchema _schema;
public GraphQLController(ISchema schema, IDocumentExecuter documentExecuter)
{
_schema = schema;
_documentExecuter = documentExecuter;
}
[HttpPost]
public async Task<IActionResult> Post([FromBody] GraphQLQuery query)
{
if (query == null)
{
throw new ArgumentNullException(nameof(query));
}
var inputs = query.Variables.ToInputs();
var executionOptions = new ExecutionOptions{Schema = _schema, Query = query.Query, Inputs = inputs, UserContext = Request.Headers, //UserContext = new GraphQLUserContext { Headers = Request.Headers }
};
throw new Exception("Exception while fetching all the students from the storage.");
var result = await _documentExecuter.ExecuteAsync(executionOptions).ConfigureAwait(false);
if (result.Errors?.Count > 0)
{
return BadRequest(result);
}
return Ok(result);
}
}
// HttpExtensions
public static class HttpExtensions
{
private static readonly JsonSerializer Serializer = new JsonSerializer{NullValueHandling = NullValueHandling.Ignore};
public static void WriteJson<T>(this HttpResponse response, T obj, string contentType = null)
{
response.ContentType = contentType ?? "application/json";
using (var writer = new HttpResponseStreamWriter(response.Body, Encoding.UTF8))
{
using (var jsonWriter = new JsonTextWriter(writer))
{
jsonWriter.CloseOutput = false;
jsonWriter.AutoCompleteOnClose = false;
Serializer.Serialize(jsonWriter, obj);
}
}
}
}
Now on invoking the graphqlcontroller endpoint, I am intentionally throwing an error which I am trying to catch in the global exception logic where I am trying to customize it before passing it to the customer.
I am getting an exception: Stream is now writable Parameter name: stream, while processing it in the HttpExtensions class at the below line:
using (var writer = new HttpResponseStreamWriter(response.Body, Encoding.UTF8))
Can anyone help me to fix this issue?

C# DotNet Core Middleware Wrap Response

I have a simple controller action which looks like:
public Task<IEnumerable<Data>> GetData()
{
IEnumerable<Data> data = new List<Data>();
return data;
}
I want to be able to inspect the return value from within the middleware so the JSON would look something like
{
"data": [
],
"apiVersion": "1.2",
"otherInfoHere": "here"
}
So my payload always is within data. I know I can do this at a controller level but I don't wan to have to do it on every single action. I would rather do it in middleware once for all.
Here is an example of my middleware:
public class NormalResponseWrapper
{
private readonly RequestDelegate next;
public NormalResponseWrapper(RequestDelegate next)
{
this.next = next;
}
public async Task Invoke(HttpContext context)
{
var obj = context;
// DO something to get return value from obj
// Create payload and set data to return value
await context.Response.WriteAsync(/*RETURN NEW PAYLOAD HERE*/);
}
Any ideas?
Got the value now but it's to late to return it
try
{
using (var memStream = new MemoryStream())
{
context.Response.Body = memStream;
await next(context);
memStream.Position = 0;
object responseBody = new StreamReader(memStream).ReadToEnd();
memStream.Position = 0;
await memStream.CopyToAsync(originalBody);
// By now it is to late, above line sets the value that is going to be returned
await context.Response.WriteAsync(new BaseResponse() { data = responseBody }.toJson());
}
}
finally
{
context.Response.Body = originalBody;
}

Review the comments to get an understanding of what you can do to wrap the response.
public async Task Invoke(HttpContext context) {
//Hold on to original body for downstream calls
Stream originalBody = context.Response.Body;
try {
string responseBody = null;
using (var memStream = new MemoryStream()) {
//Replace stream for upstream calls.
context.Response.Body = memStream;
//continue up the pipeline
await next(context);
//back from upstream call.
//memory stream now hold the response data
//reset position to read data stored in response stream
memStream.Position = 0;
responseBody = new StreamReader(memStream).ReadToEnd();
}//dispose of previous memory stream.
//lets convert responseBody to something we can use
var data = JsonConvert.DeserializeObject(responseBody);
//create your wrapper response and convert to JSON
var json = new BaseResponse() {
data = data,
apiVersion = "1.2",
otherInfoHere = "here"
}.toJson();
//convert json to a stream
var buffer = Encoding.UTF8.GetBytes(json);
using(var output = new MemoryStream(buffer)) {
await output.CopyToAsync(originalBody);
}//dispose of output stream
} finally {
//and finally, reset the stream for downstream calls
context.Response.Body = originalBody;
}
}

In .NET Core 3.1 or .NET 5
Create your response envelope object. Example:
internal class ResponseEnvelope<T>
{
public T Data { set; get; }
public string ApiVersion { set; get; }
public string OtherInfoHere { set; get; }
}
Derive a class from ObjectResultExecutor
internal class ResponseEnvelopeResultExecutor : ObjectResultExecutor
{
public ResponseEnvelopeResultExecutor(OutputFormatterSelector formatterSelector, IHttpResponseStreamWriterFactory writerFactory, ILoggerFactory loggerFactory, IOptions<MvcOptions> mvcOptions) : base(formatterSelector, writerFactory, loggerFactory, mvcOptions)
{
}
public override Task ExecuteAsync(ActionContext context, ObjectResult result)
{
var response = new ResponseEnvelope<object>();
response.Data = result.Value;
response.ApiVersion = "v1";
response.OtherInfoHere = "OtherInfo";
TypeCode typeCode = Type.GetTypeCode(result.Value.GetType());
if (typeCode == TypeCode.Object)
result.Value = response;
return base.ExecuteAsync(context, result);
}
}
Inject into the DI like
public void ConfigureServices(IServiceCollection services)
{
services.AddSingleton<IActionResultExecutor<ObjectResult>, ResponseEnvelopeResultExecutor>();
And the responses should have an envelope.
This does not work with primitive types.

How to enrich App Insights ExceptionTelemetry with request body in ASP.NET Core

TL;DR Question: Is there a way to get the request body into an existing ExceptionTelemetry instance, in ASP.NET Core, without copying ALL request bodies?
I would like to be able to include the Request Body in the exception telemetry for application insights. I.e. I only want the request when an exception has occurred.
Browsing around for documentation on both ASP.NET Core and Application Insights, it seems the "right" way to enrich telemetry is using TelemetryProcessors or TelemetryInitializers, so I tried getting the request body in a custom telemetryinitializer, only to discover that the request body stream is closed/disposed when I want to read it (rewinding does not help because apparently it has already been disposed when the App Insights telemetryinitializer is being executed).
I ended up solving it by having a middleware that copies the request stream:
public async Task Invoke(HttpContext context)
{
var stream = context.Request.Body;
try
{
using (var buffer = new MemoryStream())
{
// Copy the request stream and rewind the copy
await stream.CopyToAsync(buffer);
buffer.Position = 0L;
// Create another copy and rewind both
var otherBuffer = new MemoryStream();
await buffer.CopyToAsync(otherBuffer);
buffer.Position = 0L;
otherBuffer.Position = 0L;
// Replace the request stream by the first copy
context.Request.Body = buffer;
// Put a separate copy in items collection for other things to use
context.Items["RequestStreamCopy"] = otherBuffer;
context.Response.RegisterForDispose(otherBuffer);
await next(context);
}
}
finally
{
context.Request.Body = stream;
}
}
And my initializer:
public AiExceptionInitializer(IHttpContextAccessor httpContextAccessor)
{
this.httpContextAccessor = httpContextAccessor ?? throw new ArgumentNullException("httpContextAccessor");
}
public void Initialize(ITelemetry telemetry)
{
var context = this.httpContextAccessor.HttpContext;
if (context == null)
{
return;
}
lock (context)
{
var request = context.Features.Get<RequestTelemetry>();
if (request == null)
{
return;
}
this.OnInitializeTelemetry(context, request, telemetry);
}
}
protected void OnInitializeTelemetry(HttpContext platformContext, RequestTelemetry requestTelemetry, ITelemetry telemetry)
{
if (telemetry is ExceptionTelemetry exceptionTelemetry)
{
var stream = platformContext.Items["RequestStreamCopy"] as MemoryStream;
try
{
if (stream?.Length <= 0)
{
return;
}
// Rewind the stream position just to be on the safe side
stream.Position = 0L;
using (var reader = new StreamReader(stream, Encoding.UTF8, true, 1024, true))
{
string requestBody = reader.ReadToEnd();
exceptionTelemetry.Properties.Add("HttpRequestBody", requestBody);
}
}
finally
{
if (stream != null)
{
// Rewind the stream for others to use.
stream.Position = 0L;
}
}
}
}
However this having to copy the request stream (TWICE) for each request, to only have it used on failures seems rather inefficient to me.
So I am wondering if there is any other way to do something like this where I don't have to copy the stream of each and every request just to serialize the ones failing?
I am aware I could "just" write a middleware that would create new ExceptionTelemetry instances, but as far as I know (I might be wrong) it would leave me with two Exception instances in Application Insights (i.e. the one generated by me and the one generated by the AI extensions), instead of just one exception with the added property I need.

Thanks to the comment from #DmitryMatveev I found an alternate solution. I am not sure its the most effective, but it is better than what i had!
The middleware is "reduced" to only tracking exceptions, and then serialising the body right away (you might still have a stream copy, but I don't need it in my case), something like the following:
using System;
using System.IO;
using System.Text;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Http.Internal;
public class ExceptionBodyTrackingMiddleware
{
public const string ExceptionRequestBodyKey = "ExceptionRequestBody";
private readonly RequestDelegate next;
public ExceptionBodyTrackingMiddleware(RequestDelegate next)
{
this.next = next ?? throw new ArgumentNullException(nameof(next));
}
public async Task Invoke(HttpContext context)
{
try
{
context.Request.EnableRewind();
await this.next.Invoke(context);
}
catch (Exception)
{
RegisterRequestBody(context);
throw;
}
}
private static void RegisterRequestBody(HttpContext context)
{
if (context.Request.Body?.CanSeek == false)
{
return;
}
var body = CopyStreamToString(context.Request.Body);
context.Items[ExceptionRequestBodyKey] = body;
}
private static string CopyStreamToString(Stream stream)
{
var originalPosition = stream.Position;
RewindStream(stream);
string requestBody = null;
using (var reader = new StreamReader(stream, Encoding.UTF8, true, 1024, true))
{
requestBody = reader.ReadToEnd();
}
stream.Position = originalPosition;
return requestBody;
}
private static void RewindStream(Stream stream)
{
if (stream != null)
{
stream.Position = 0L;
}
}
}
Likewise the Initializer becomes a whole lot simpler:
public AiExceptionInitializer(IHttpContextAccessor httpContextAccessor)
{
this.httpContextAccessor = httpContextAccessor ?? throw new ArgumentNullException("httpContextAccessor");
}
public void Initialize(ITelemetry telemetry)
{
var context = this.httpContextAccessor.HttpContext;
if (context == null)
{
return;
}
lock (context)
{
var request = context.Features.Get<RequestTelemetry>();
if (request == null)
{
return;
}
this.OnInitializeTelemetry(context, request, telemetry);
}
}
protected void OnInitializeTelemetry(HttpContext platformContext, RequestTelemetry requestTelemetry, ITelemetry telemetry)
{
if (telemetry is ExceptionTelemetry exceptionTelemetry)
{
var requestBody = platformContext.Items[ExceptionBodyTrackingMiddleware.ExceptionRequestBodyKey] as string;
exceptionTelemetry.Properties.Add("HttpRequestBody", requestBody);
}
}