I am trying to send large file (GB) from one WebAPI (.NET Core) to another WebApi (.Net Core).
I already managed to send smaller file as part of Multipart Request like in last post here: link
To send bigger file I need (I think) send this file as StreamContent, however i am getting Content length = 0 in API which receives request.
Problem occurs even when I am sending (for test) smaller files (10 Mb).
Clientside code:
[HttpPost("UploadFiles")]
public async Task<IActionResult> Post(IFormFile file)
{
var filePath = Path.GetTempFileName();
using (var stream = new FileStream(filePath, FileMode.Create, FileAccess.ReadWrite))
{
await file.CopyToAsync(stream);
using (var formDataContent = new MultipartFormDataContent())
{
using (var httpClient = new HttpClient())
{
formDataContent.Add(CreateFileContent(stream, "myfile.test", "application/octet-stream"));
var response = await httpClient.PostAsync(
"http://localhost:56595/home/upload",
formDataContent);
return Json(response);
}
}
}
}
internal static StreamContent CreateFileContent(Stream stream, string fileName, string contentType)
{
var fileContent = new StreamContent(stream);
fileContent.Headers.ContentDisposition = new System.Net.Http.Headers.ContentDispositionHeaderValue("form-data")
{
Name = "\"file\"",
FileName = "\"" + fileName + "\"",
};
fileContent.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue(contentType);
return fileContent;
}
Serverside code:
[HttpPost]
public ActionResult Upload()
{
IFormFile fileFromRequest = Request.Form.Files.First();
string myFileName = fileFromRequest.Name;
// some code
return Ok();
}
Where is the problem?
To create Multipart request I used advices from:
HttpClient StreamContent append filename twice
POST StreamContent with Multiple Files
Finally I figured it out:
There were 2 problems:
1. Stream pointer position
In client side code, change this:
await file.CopyToAsync(stream);
to that:
await file.CopyToAsync(stream);
stream.Position = 0;
Problem was that file from request was copied to stream and left position of pointer in the end of the stream. That is why request send from client had stream with proper length, but actually when it started to read it, it could not (read 0 bytes).
2. Wrong way of handling request on server.
I used code from dotnetcoretutorials.com
Working code below:
Client side:
[HttpPost("UploadFiles")]
public async Task<IActionResult> Post(IFormFile file)
{
var filePath = Path.GetTempFileName();
using (var stream = new FileStream(filePath, FileMode.Create, FileAccess.ReadWrite))
{
await file.CopyToAsync(stream);
stream.Position = 0;
using (var formDataContent = new MultipartFormDataContent())
{
using (var httpClient = new HttpClient())
{
formDataContent.Add(CreateFileContent(stream, "myfile.test", "application/octet-stream"));
var response = await httpClient.PostAsync(
"http://localhost:56595/home/upload",
formDataContent);
return Json(response);
}
}
}
}
internal static StreamContent CreateFileContent(Stream stream, string fileName, string contentType)
{
var fileContent = new StreamContent(stream);
fileContent.Headers.ContentDisposition = new System.Net.Http.Headers.ContentDispositionHeaderValue("form-data")
{
Name = "\"file\"",
FileName = "\"" + fileName + "\"",
};
fileContent.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue(contentType);
return fileContent;
}
Server side:
Controller:
[HttpPost]
[DisableFormValueModelBinding]
public async Task<IActionResult> Upload()
{
var viewModel = new MyViewModel();
try
{
FormValueProvider formModel;
using (var stream = System.IO.File.Create("c:\\temp\\myfile.temp"))
{
formModel = await Request.StreamFile(stream);
}
var bindingSuccessful = await TryUpdateModelAsync(viewModel, prefix: "",
valueProvider: formModel);
if (!bindingSuccessful)
{
if (!ModelState.IsValid)
{
return BadRequest(ModelState);
}
}
}
catch(Exception exception)
{
throw;
}
return Ok(viewModel);
}
Helper classes for methods from controller:
public static class MultipartRequestHelper
{
// Content-Type: multipart/form-data; boundary="----WebKitFormBoundarymx2fSWqWSd0OxQqq"
// The spec says 70 characters is a reasonable limit.
public static string GetBoundary(MediaTypeHeaderValue contentType, int lengthLimit)
{
var boundary = HeaderUtilities.RemoveQuotes(contentType.Boundary);
if (string.IsNullOrWhiteSpace(boundary.ToString()))
{
throw new InvalidDataException("Missing content-type boundary.");
}
if (boundary.Length > lengthLimit)
{
throw new InvalidDataException(
$"Multipart boundary length limit {lengthLimit} exceeded.");
}
return boundary.ToString();
}
public static bool IsMultipartContentType(string contentType)
{
return !string.IsNullOrEmpty(contentType)
&& contentType.IndexOf("multipart/", StringComparison.OrdinalIgnoreCase) >= 0;
}
public static bool HasFormDataContentDisposition(ContentDispositionHeaderValue contentDisposition)
{
// Content-Disposition: form-data; name="key";
return contentDisposition != null
&& contentDisposition.DispositionType.Equals("form-data")
&& string.IsNullOrEmpty(contentDisposition.FileName.ToString())
&& string.IsNullOrEmpty(contentDisposition.FileNameStar.ToString());
}
public static bool HasFileContentDisposition(ContentDispositionHeaderValue contentDisposition)
{
// Content-Disposition: form-data; name="myfile1"; filename="Misc 002.jpg"
return contentDisposition != null
&& contentDisposition.DispositionType.Equals("form-data")
&& (!string.IsNullOrEmpty(contentDisposition.FileName.ToString())
|| !string.IsNullOrEmpty(contentDisposition.FileNameStar.ToString()));
}
}
public static class FileStreamingHelper
{
private static readonly FormOptions _defaultFormOptions = new FormOptions();
public static async Task<FormValueProvider> StreamFile(this HttpRequest request, Stream targetStream)
{
if (!MultipartRequestHelper.IsMultipartContentType(request.ContentType))
{
throw new Exception($"Expected a multipart request, but got {request.ContentType}");
}
// Used to accumulate all the form url encoded key value pairs in the
// request.
var formAccumulator = new KeyValueAccumulator();
string targetFilePath = null;
var boundary = MultipartRequestHelper.GetBoundary(
MediaTypeHeaderValue.Parse(request.ContentType),
_defaultFormOptions.MultipartBoundaryLengthLimit);
var reader = new MultipartReader(boundary, request.Body);
var section = await reader.ReadNextSectionAsync();
while (section != null)
{
ContentDispositionHeaderValue contentDisposition;
var hasContentDispositionHeader = ContentDispositionHeaderValue.TryParse(section.ContentDisposition, out contentDisposition);
if (hasContentDispositionHeader)
{
if (MultipartRequestHelper.HasFileContentDisposition(contentDisposition))
{
await section.Body.CopyToAsync(targetStream);
}
else if (MultipartRequestHelper.HasFormDataContentDisposition(contentDisposition))
{
// Content-Disposition: form-data; name="key"
//
// value
// Do not limit the key name length here because the
// multipart headers length limit is already in effect.
var key = HeaderUtilities.RemoveQuotes(contentDisposition.Name);
var encoding = GetEncoding(section);
using (var streamReader = new StreamReader(
section.Body,
encoding,
detectEncodingFromByteOrderMarks: true,
bufferSize: 1024,
leaveOpen: true))
{
// The value length limit is enforced by MultipartBodyLengthLimit
var value = await streamReader.ReadToEndAsync();
if (String.Equals(value, "undefined", StringComparison.OrdinalIgnoreCase))
{
value = String.Empty;
}
formAccumulator.Append(key.ToString(), value);
if (formAccumulator.ValueCount > _defaultFormOptions.ValueCountLimit)
{
throw new InvalidDataException($"Form key count limit {_defaultFormOptions.ValueCountLimit} exceeded.");
}
}
}
}
// Drains any remaining section body that has not been consumed and
// reads the headers for the next section.
section = await reader.ReadNextSectionAsync();
}
// Bind form data to a model
var formValueProvider = new FormValueProvider(
BindingSource.Form,
new FormCollection(formAccumulator.GetResults()),
CultureInfo.CurrentCulture);
return formValueProvider;
}
private static Encoding GetEncoding(MultipartSection section)
{
MediaTypeHeaderValue mediaType;
var hasMediaTypeHeader = MediaTypeHeaderValue.TryParse(section.ContentType, out mediaType);
// UTF-7 is insecure and should not be honored. UTF-8 will succeed in
// most cases.
if (!hasMediaTypeHeader || Encoding.UTF7.Equals(mediaType.Encoding))
{
return Encoding.UTF8;
}
return mediaType.Encoding;
}
}
[AttributeUsage(AttributeTargets.Class | AttributeTargets.Method)]
public class DisableFormValueModelBindingAttribute : Attribute, IResourceFilter
{
public void OnResourceExecuting(ResourceExecutingContext context)
{
var formValueProviderFactory = context.ValueProviderFactories
.OfType<FormValueProviderFactory>()
.FirstOrDefault();
if (formValueProviderFactory != null)
{
context.ValueProviderFactories.Remove(formValueProviderFactory);
}
var jqueryFormValueProviderFactory = context.ValueProviderFactories
.OfType<JQueryFormValueProviderFactory>()
.FirstOrDefault();
if (jqueryFormValueProviderFactory != null)
{
context.ValueProviderFactories.Remove(jqueryFormValueProviderFactory);
}
}
public void OnResourceExecuted(ResourceExecutedContext context)
{
}
}
Additional thoughts:
(on clientside) line:
fileContent.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue(contentType);
is not necessary to send the file.
(on clientside) file is send when MediaTypeHeaderValue is one of these:
application/x-msdownload
application/json
application/octet-stream
(on serverside) to use lines with contentDisposition.FileNameStar on serverside you need to change them to contentDisposition.FileNameStar.ToString()
(on serverside) code used in question for serverside will work with smaller files (Mb's) but to send GB file we need code which is pasted in the answer.
some parts of code are taken from aspnet core docs
Related
I have a Web Api controller, that gets file. (Server)
[HttpGet]
[Route("api/FileDownloading/download")]
public HttpResponseMessage GetDocuments()
{
var result = new HttpResponseMessage(HttpStatusCode.OK);
var fileName = "QRimage2.jpg";
var filePath = HttpContext.Current.Server.MapPath("");
var fileBytes = File.ReadAllBytes(#"c:\\TMP\\QRimage2.jpg");
MemoryStream fileMemStream = new MemoryStream(fileBytes);
result.Content = new StreamContent(fileMemStream);
var headers = result.Content.Headers;
headers.ContentDisposition = new ContentDispositionHeaderValue("attachment");
headers.ContentDisposition.FileName = fileName;
headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
headers.ContentLength = fileMemStream.Length;
return result;
}
And Xamarin Android client, that downloading the file using the controller (http://localhost:6100/api/FileDownloading/download)
public void DownloadFile(string url, string folder)
{
string pathToNewFolder = Path.Combine(Android.OS.Environment.ExternalStorageDirectory.AbsolutePath, folder);
Directory.CreateDirectory(pathToNewFolder);
try
{
WebClient webClient = new WebClient();
webClient.DownloadFileCompleted += new AsyncCompletedEventHandler(Completed);
string pathToNewFile = Path.Combine(pathToNewFolder, Path.GetFileName(url));
webClient.DownloadFileAsync(new Uri(url), null);
}
catch (Exception ex)
{
if (OnFileDownloaded != null)
OnFileDownloaded.Invoke(this, new DownloadEventArgs(false));
}
}
Everithing works fine, but on my Android device in file explorer i have file with "download" file name instead of "QRimage2.jpg". How can I get actual file name using this controller?
You will need use the web response to read the content disposition. So, we can't use DownloadFileAsync directly.
public async Task<string> DownloadFileAsync(string url, string folder)
{
var request = WebRequest.Create(url);
var response = await request.GetResponseAsync().ConfigureAwait(false);
var fileName = string.Empty;
if (response.Headers["Content-Disposition"] != null)
{
var contentDisposition = new System.Net.Mime.ContentDisposition(response.Headers["Content-Disposition"]);
if (contentDisposition.DispositionType == "attachment")
{
fileName = contentDisposition.FileName;
}
}
if (string.IsNullOrEmpty(fileName))
{
throw new ArgumentException("Cannot be null or empty.", nameof(fileName));
}
var filePath = Path.Combine(Android.OS.Environment.ExternalStorageDirectory.AbsolutePath, folder, fileName);
using (var contentStream = response.GetResponseStream())
{
using (var fileStream = File.Create(filePath))
{
await contentStream.CopyToAsync(fileStream).ConfigureAwait(false);
}
}
return filePath;
}
Is this always going to be a jpg? If so I'd change the MediaTypeHeaderValue to image/jpeg - By doing that you are telling the browser the exact type of file, instead of a generic file. I'm thinking this is the issue since you are telling the Android Browser it's just a generic binary file.
Do I need Content-Type: application/octet-stream for file download?
I have a Post Web Api Method in a controller and it's already working, I've been testing the method sending file from postman and a my own web application and it works but now I'm trying to send file from console application using Httpclient but always get 404.
Controller Method
public async Task<IHttpActionResult> Post()
{
FileServerConfig config = FileServerConfiguration.ObtenerConfiguracion(ConfigurationManager.AppSettings);
var path = string.Empty;
try
{
if (!Request.Content.IsMimeMultipartContent())
{
return StatusCode(HttpStatusCode.UnsupportedMediaType);
}
var filesReadToProvider = await Request.Content.ReadAsMultipartAsync();
foreach (var stream in filesReadToProvider.Contents)
{
IFileServerWrapper _cliente = FileServerConfiguration.CrearCliente(config);
var name = stream.Headers.ContentDisposition.FileName.Replace("\"", "").Replace("\\", "");
var fileUploaded = await _cliente.FileUpload(path, await stream.ReadAsStreamAsync(), name, false);
}
return Ok();
}
catch (HttpException httpex)
{
if (httpex.GetHttpCode() == (int)HttpStatusCode.Conflict)
{
return Conflict();
}
else
{
return InternalServerError(httpex);
}
}
catch (Exception ex)
{
return InternalServerError(ex);
}
}
Console application
static void Main(string[] args)
{
string filePath = #"C:\Software\itextTifftoPDF.rar";
using (var client = new HttpClient())
using (var content = new MultipartFormDataContent())
{
// Make sure to change API address
//client.BaseAddress = new Uri("http://localhost:80/FileServerAPI/");
client.DefaultRequestHeaders.Add("Accept-Language", "en-GB,en-US;q=0.8,en;q=0.6,ru;q=0.4");
client.DefaultRequestHeaders.CacheControl = CacheControlHeaderValue.Parse("no-cache");
client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
// Add first file content
var fileContent1 = new ByteArrayContent(File.ReadAllBytes(filePath));
fileContent1.Headers.ContentDisposition = new ContentDispositionHeaderValue("form-data")
{
FileName = "itextTifftoPDF.rar"
};
content.Add(fileContent1);
// Make a call to Web API
var result = client.PostAsync("http://localhost/FileServerAPI/api/File", content).Result;
Console.WriteLine(result.StatusCode);
Console.ReadLine();
}
}
Finally I found a solution. the code is ugly because it was made just to be sure it works but I'll hope it may help others with the same problem.
string filePath = #"C:\temp\webdavtest.txt";
using (HttpClient httpClient = new HttpClient())
{
using (MultipartFormDataContent content =
new MultipartFormDataContent())
{
using (FileStream stream = File.Open(
filePath, FileMode.Open, FileAccess.Read))
{
using (StreamContent streamConent =
new StreamContent(stream))
{
content.Add(
streamConent, "webdavtest.txt", "webdavtest.txt");
var result = await httpClient.PostAsync("http://localhost:3983/api/File?path=/nivel5/", content);
return result.StatusCode.ToString();
}
}
}
}
I am struggling with being able to create a file with its data based on the byte array returned from the WebAPI. The following is my code for making the call to the web api
using (var http = new WebClient())
{
string url = string.Format("{0}api/FileUpload/FileServe?FileID=" + fileID, webApiUrl);
http.Headers[HttpRequestHeader.ContentType] = "application/octet-stream";
http.Headers[HttpRequestHeader.Authorization] = "Bearer " + authCookie.Value;
http.DownloadDataCompleted += Http_DownloadDataCompleted;
byte[] json = await http.DownloadDataTaskAsync(url);
}
The api code is
[HttpGet]
[Route("FileServe")]
[Authorize(Roles = "Admin,SuperAdmin,Contractor")]
public async Task<HttpResponseMessage> GetFile(int FileID)
{
using (var repo = new MBHDocRepository())
{
var file = await repo.GetSpecificFile(FileID);
if (file == null)
{
throw new HttpResponseException(HttpStatusCode.BadRequest);
}
var stream = File.Open(file.PathLocator, FileMode.Open);
HttpResponseMessage response = new HttpResponseMessage(HttpStatusCode.OK);
response.Content = new StreamContent(stream);
response.Content.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue(file.FileType);
return response;
}
}
I receive a byte array as a response however am unable to create the corresponding file from that byte array. I have no idea how to convert the byte array into the relevant file type (such as jpg, or pdf based on file type in the web api). any help will be appreciated.
Alright so there are a few ways of solving your problem firstly, on the server side of things you can either simply send the content type and leave it at that or you can also send the complete filename which helps you even further.
I have removed the code that is specific to your stuff with basic test code, please just ignore that stuff and use it in terms of your code.
Some design notes here:
[HttpGet]
[Route("FileServe")]
[Authorize(Roles = "Admin,SuperAdmin,Contractor")]
public async Task<HttpResponseMessage> GetFileAsync(int FileID) //<-- If your method returns Task have it be named with Async in it
{
using (var repo = new MBHDocRepository())
{
var file = await repo.GetSpecificFile(FileID);
if (file == null)
{
throw new HttpResponseException(HttpStatusCode.BadRequest);
}
var stream = File.Open(file.PathLocator, FileMode.Open);
HttpResponseMessage response = new HttpResponseMessage(HttpStatusCode.OK);
response.Content = new StreamContent(stream);
response.Content.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue(file.FileType);
response.Content.Headers.ContentDisposition = new System.Net.Http.Headers.ContentDispositionHeaderValue("attachment") { FileName=Path.GetFileName(file.PathLocator)};
return response;
}
}
Your client side code has two options here:
static void Main(string[] args)
{
using (var http = new WebClient())
{
string url = string.Format("{0}api/FileUpload/FileServe?FileID={1}",webApiUrl, fileId);
http.Headers[HttpRequestHeader.ContentType] = "application/octet-stream";
http.Headers[HttpRequestHeader.Authorization] = "Bearer " + authCookie.Value;
var response = http.OpenRead(url);
var fs = new FileStream(String.Format(#"C:\Users\Bailey Miller\Downloads\{0}", GetName(http.ResponseHeaders)), FileMode.Create);
response.CopyTo(fs); <-- how to move the stream to the actual file, this is not perfect and there are a lot of better examples
fs.Flush();
fs.Close();
}
}
private static object GetName(WebHeaderCollection responseHeaders)
{
var c_type = responseHeaders.GetValues("Content-Type"); //<-- do a switch on this and return a really weird file name with the correct extension for the mime type.
var cd = responseHeaders.GetValues("Content-Disposition")[0].Replace("\"", ""); <-- this gets the attachment type and filename param, also removes illegal character " from filename if present
return cd.Substring(cd.IndexOf("=")+1); <-- extracts the file name
}
I am trying to write code to upload file(s) by WinForm app to WebApi.
The WebApi code is like:
[HttpPost]
[Route("UploadEnvelope")]
[HostAuthentication(DefaultAuthenticationTypes.ExternalBearer)]
public Task<HttpResponseMessage> PostUploadEnvelope()
{
HttpRequestMessage request = this.Request;
if (!request.Content.IsMimeMultipartContent())
{
throw new HttpResponseException(HttpStatusCode.UnsupportedMediaType);
}
string root = System.Web.HttpContext.Current.Server.MapPath("~/App_Data/uploads");
var provider = new MultipartFormDataStreamProvider(root);
var task = request.Content.ReadAsMultipartAsync(provider).ContinueWith<HttpResponseMessage>(o =>
{
foreach (MultipartFileData fileData in provider.FileData)
{
if (string.IsNullOrEmpty(fileData.Headers.ContentDisposition.FileName))
{
return Request.CreateResponse(HttpStatusCode.NotAcceptable, "This request is not properly formatted");
}
string fileName = fileData.Headers.ContentDisposition.FileName;
if (fileName.StartsWith("\"") && fileName.EndsWith("\""))
{
fileName = fileName.Trim('"');
}
if (fileName.Contains(#"/") || fileName.Contains(#"\"))
{
fileName = Path.GetFileName(fileName);
}
File.Move(fileData.LocalFileName, Path.Combine(root, fileName));
}
return new HttpResponseMessage()
{
Content = new StringContent("Files uploaded.")
};
}
);
return task;
}
But I am not sure how to call it and pass file in a client app.
static string UploadEnvelope(string filePath, string token, string url)
{
using (var client = new HttpClient())
{
client.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", token);
// How to pass file here ???
var response = client.GetAsync(url + "/api/Envelope/UploadEnvelope").Result;
return response.Content.ReadAsStringAsync().Result;
}
}
Any help or suggestion is welcome. Thanks in advance!
First you are using Get method which is used for reading. You have to use Post instead.
Try the following:
public static string UploadEnvelope(string filePath,string token, string url)
{
using (var client = new HttpClient())
{
client.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", token);
using (var content = new MultipartFormDataContent("Envelope" + DateTime.Now.ToString(CultureInfo.InvariantCulture)))
{
content.Add(new StreamContent(new MemoryStream(File.ReadAllBytes(filePath))), "filename", "filename.ext");
using (var message = await client.PostAsync(url + "/api/Envelope/UploadEnvelope", content))
{
var input = await message.Content.ReadAsStringAsync();
return "success";
}
}
}
}
Note: For a large file you have to change configuration on IIS web.config.
This is a follow up to a question I had asked previously that was closed for being too broad.Previous Question
In that question I explained that I needed upload a large file (1-3GB) to the database by storing chunks as individual rows. I did this by overriding the MultipartFormDataStreamProvider.GetStream method. That method returned a custom stream that wrote the buffered chunks to the database.
The problem is that the overriden GetStream method is writing the entire request to the database (including the headers). It is successfully writing that data while keeping the Memory levels flat but when I download the file, in addition to the file contents, it's returning all the header information in the downloaded file contents so the file can't be opened.
Is there a way to, in the overriden GetStream method, write just the contents of the file to the database without writing the headers?
API
[HttpPost]
[Route("file")]
[ValidateMimeMultipartContentFilter]
public Task<HttpResponseMessage> PostFormData()
{
var provider = new CustomMultipartFormDataStreamProvider();
// Read the form data and return an async task.
var task = Request.Content.ReadAsMultipartAsync(provider).ContinueWith<HttpResponseMessage>(t =>
{
if (t.IsFaulted || t.IsCanceled)
{
Request.CreateErrorResponse(HttpStatusCode.InternalServerError, t.Exception);
}
return Request.CreateResponse(HttpStatusCode.OK);
});
return task;
}
[HttpGet]
[Route("file/{id}")]
public async Task<HttpResponseMessage> GetFile(string id)
{
var result = new HttpResponseMessage()
{
Content = new PushStreamContent(async (outputStream, httpContent, transportContext) =>
{
await WriteDataChunksFromDBToStream(outputStream, httpContent, transportContext, id);
}),
StatusCode = HttpStatusCode.OK
};
result.Content.Headers.ContentType = new MediaTypeHeaderValue("application/zipx");
result.Content.Headers.ContentDisposition = new ContentDispositionHeaderValue("attachment") { FileName = "test response.zipx" };
return result;
}
return new HttpResponseMessage(HttpStatusCode.BadRequest);
}
private async Task WriteDataChunksFromDBToStream(Stream responseStream, HttpContent httpContent, TransportContext transportContext, string fileIdentifier)
{
// PushStreamContent requires the responseStream to be closed
// for signaling it that you have finished writing the response.
using (responseStream)
{
using (var myConn = new SqlConnection(System.Configuration.ConfigurationManager.ConnectionStrings["TestDB"].ConnectionString))
{
await myConn.OpenAsync();
using (var myCmd = new SqlCommand("ReadAttachmentChunks", myConn))
{
myCmd.CommandType = System.Data.CommandType.StoredProcedure;
var fileName = new SqlParameter("#Identifier", fileIdentifier);
myCmd.Parameters.Add(fileName);
// Read data back from db in async call to avoid OutOfMemoryException when sending file back to user
using (var reader = await myCmd.ExecuteReaderAsync(CommandBehavior.SequentialAccess))
{
while (await reader.ReadAsync())
{
if (!(await reader.IsDBNullAsync(3)))
{
using (var data = reader.GetStream(3))
{
// Asynchronously copy the stream from the server to the response stream
await data.CopyToAsync(responseStream);
}
}
}
}
}
}
}// close response stream
}
Custom MultipartFormDataStreamProvider GetStream method implementation
public override Stream GetStream(HttpContent parent, HttpContentHeaders headers)
{
// For form data, Content-Disposition header is a requirement
ContentDispositionHeaderValue contentDisposition = headers.ContentDisposition;
if (contentDisposition != null)
{
// If we have a file name then write contents out to AWS stream. Otherwise just write to MemoryStream
if (!String.IsNullOrEmpty(contentDisposition.FileName))
{
var identifier = Guid.NewGuid().ToString();
var fileName = contentDisposition.FileName;// GetLocalFileName(headers);
if (fileName.Contains("\\"))
{
fileName = fileName.Substring(fileName.LastIndexOf("\\") + 1).Replace("\"", "");
}
// We won't post process files as form data
_isFormData.Add(false);
var stream = new CustomSqlStream();
stream.Filename = fileName;
stream.Identifier = identifier;
stream.ContentType = headers.ContentType.MediaType;
stream.Description = (_formData.AllKeys.Count() > 0 && _formData["description"] != null) ? _formData["description"] : "";
return stream;
//return new CustomSqlStream(contentDisposition.Name);
}
// We will post process this as form data
_isFormData.Add(true);
// If no filename parameter was found in the Content-Disposition header then return a memory stream.
return new MemoryStream();
}
throw new InvalidOperationException("Did not find required 'Content-Disposition' header field in MIME multipart body part..");
#endregion
}
Implemented Write method of Stream called by CustomSqlStream
public override void Write(byte[] buffer, int offset, int count)
{
//write buffer to database
using (var myConn = new SqlConnection(System.Configuration.ConfigurationManager.ConnectionStrings["TestDB"].ConnectionString)) {
using (var myCmd = new SqlCommand("WriteAttachmentChunk", myConn)) {
myCmd.CommandType = System.Data.CommandType.StoredProcedure;
var pContent = new SqlParameter("#Content", buffer);
myCmd.Parameters.Add(pContent);
myConn.Open();
myCmd.ExecuteNonQuery();
if (myConn.State == System.Data.ConnectionState.Open)
{
myConn.Close();
}
}
}
((ManualResetEvent)_dataAddedEvent).Set();
}
The "ReadAttachmentChunks" stored procedure gets the rows respective to the file from the db ordered by the time they are inserted into the database. So, the way the code works is it pulls those chunks back and then async writes it back to the PushStreamContent to go back to the user.
So my question is:
Is there a way to write ONLY the content of the file being uploaded as opposed to the headers in addition to the content?
Any help would be greatly appreciated. Thank you.
I finally figured it out. I over-complicated the write process which brought about most of the struggle. Here is my solution to my initial issue:
To keep .net from buffering the file in memory (so that you can handle large file uploads), you first need to override the WebHostBufferPolicySelector so that it doesnt buffer the input stream for your controller and then replace the BufferPolicy Selector.
public class NoBufferPolicySelector : WebHostBufferPolicySelector
{
public override bool UseBufferedInputStream(object hostContext)
{
var context = hostContext as HttpContextBase;
if (context != null)
{
if (context.Request.RequestContext.RouteData.Values["controller"] != null)
{
if (string.Equals(context.Request.RequestContext.RouteData.Values["controller"].ToString(), "upload", StringComparison.InvariantCultureIgnoreCase))
return false;
}
}
return true;
}
public override bool UseBufferedOutputStream(HttpResponseMessage response)
{
return base.UseBufferedOutputStream(response);
}
}
then for replacing the BufferPolicy Selector
GlobalConfiguration.Configuration.Services.Replace(typeof(IHostBufferPolicySelector), new NoBufferPolicySelector());
Then to avoid the default behavior of having the file stream written to disk, you need to provide a stream provider that will write to the database instead. To do this you inherit MultipartStreamProvider and override the GetStream method to return the stream that will write to your database.
public override Stream GetStream(HttpContent parent, HttpContentHeaders headers)
{
// For form data, Content-Disposition header is a requirement
ContentDispositionHeaderValue contentDisposition = headers.ContentDisposition;
if (contentDisposition != null && !String.IsNullOrEmpty(contentDisposition.FileName))
{
// We won't post process files as form data
_isFormData.Add(false);
//create unique identifier for this file upload
var identifier = Guid.NewGuid();
var fileName = contentDisposition.FileName;
var boundaryObj = parent.Headers.ContentType.Parameters.SingleOrDefault(a => a.Name == "boundary");
var boundary = (boundaryObj != null) ? boundaryObj.Value : "";
if (fileName.Contains("\\"))
{
fileName = fileName.Substring(fileName.LastIndexOf("\\") + 1).Replace("\"", "");
}
//write parent container for the file chunks that are being stored
WriteLargeFileContainer(fileName, identifier, headers.ContentType.MediaType, boundary);
//create an instance of the custom stream that will write the chunks to the database
var stream = new CustomSqlStream();
stream.Filename = fileName;
stream.FullFilename = contentDisposition.FileName.Replace("\"", "");
stream.Identifier = identifier.ToString();
stream.ContentType = headers.ContentType.MediaType;
stream.Boundary = (!string.IsNullOrEmpty(boundary)) ? boundary : "";
return stream;
}
else
{
// We will post process this as form data
_isFormData.Add(true);
// If no filename parameter was found in the Content-Disposition header then return a memory stream.
return new MemoryStream();
}
}
The custom stream you create needs to inherit Stream and override the Write method. This is where I overthought the problem and thought I needed to parse out the boundary headers that were passed via the buffer parameter. But this is actually done for you by leveraging the offset and count parameters.
public override void Write(byte[] buffer, int offset, int count)
{
//no boundary is inluded in buffer
byte[] fileData = new byte[count];
Buffer.BlockCopy(buffer, offset, fileData, 0, count);
WriteData(fileData);
}
From there, it's just plugging in the api methods for upload and download.
For upload:
public Task<HttpResponseMessage> PostFormData()
{
var provider = new CustomMultipartLargeFileStreamProvider();
// Read the form data and return an async task.
var task = Request.Content.ReadAsMultipartAsync(provider).ContinueWith<HttpResponseMessage>(t =>
{
if (t.IsFaulted || t.IsCanceled)
{
Request.CreateErrorResponse(HttpStatusCode.InternalServerError, t.Exception);
}
return Request.CreateResponse(HttpStatusCode.OK);
});
return task;
}
For download, and in order to keep the memory footprint low, I leveraged the PushStreamContent to push the chunks back to the user:
[HttpGet]
[Route("file/{id}")]
public async Task<HttpResponseMessage> GetFile(string id)
{
string mimeType = string.Empty;
string filename = string.Empty;
if (!string.IsNullOrEmpty(id))
{
//get the headers for the file being sent back to the user
using (var myConn = new SqlConnection(System.Configuration.ConfigurationManager.ConnectionStrings["PortalBetaConnectionString"].ConnectionString))
{
using (var myCmd = new SqlCommand("ReadLargeFileInfo", myConn))
{
myCmd.CommandType = System.Data.CommandType.StoredProcedure;
var pIdentifier = new SqlParameter("#Identifier", id);
myCmd.Parameters.Add(pIdentifier);
myConn.Open();
var dataReader = myCmd.ExecuteReader();
if (dataReader.HasRows)
{
while (dataReader.Read())
{
mimeType = dataReader.GetString(0);
filename = dataReader.GetString(1);
}
}
}
}
var result = new HttpResponseMessage()
{
Content = new PushStreamContent(async (outputStream, httpContent, transportContext) =>
{
//pull the data back from the db and stream the data back to the user
await WriteDataChunksFromDBToStream(outputStream, httpContent, transportContext, id);
}),
StatusCode = HttpStatusCode.OK
};
result.Content.Headers.ContentType = new MediaTypeHeaderValue(mimeType);// "application/octet-stream");
result.Content.Headers.ContentDisposition = new ContentDispositionHeaderValue("attachment") { FileName = filename };
return result;
}
return new HttpResponseMessage(HttpStatusCode.BadRequest);
}
private async Task WriteDataChunksFromDBToStream(Stream responseStream, HttpContent httpContent, TransportContext transportContext, string fileIdentifier)
{
// PushStreamContent requires the responseStream to be closed
// for signaling it that you have finished writing the response.
using (responseStream)
{
using (var myConn = new SqlConnection(System.Configuration.ConfigurationManager.ConnectionStrings["PortalBetaConnectionString"].ConnectionString))
{
await myConn.OpenAsync();
//stored proc to pull the data back from the db
using (var myCmd = new SqlCommand("ReadAttachmentChunks", myConn))
{
myCmd.CommandType = System.Data.CommandType.StoredProcedure;
var fileName = new SqlParameter("#Identifier", fileIdentifier);
myCmd.Parameters.Add(fileName);
// The reader needs to be executed with the SequentialAccess behavior to enable network streaming
// Otherwise ReadAsync will buffer the entire BLOB into memory which can cause scalability issues or even OutOfMemoryExceptions
using (var reader = await myCmd.ExecuteReaderAsync(CommandBehavior.SequentialAccess))
{
while (await reader.ReadAsync())
{
//confirm the column that has the binary data of the file returned is not null
if (!(await reader.IsDBNullAsync(0)))
{
//read the binary data of the file into a stream
using (var data = reader.GetStream(0))
{
// Asynchronously copy the stream from the server to the response stream
await data.CopyToAsync(responseStream);
await data.FlushAsync();
}
}
}
}
}
}
}// close response stream
}
Ugh. This is nasty. With the upload, you have to make sure to
separate the headers from the content portion - you must follow the requirements RFC documents for HTTP.
Allow for chunked transfers
Of course, the content portion (unless you are transmitting text) will be binary encoded into strings.
Allow for transfers that are compressed, i.e. GZIP or DEFLATE.
Maybe - just maybe - take the encoding into account (ASCII, Unicode, UTF8, etc).
You can't really ensure that you're persisting the right info to the database without looking at all of these. For the latter items, all of your metadata as to what to do will be somewhere in the header, so it's not just a throwaway.