I have the following code, which essentially reads an HTTP Response into a byte array and uses unsafe operations to copy data from the byte buffer into a specified region of a memory mapped file:
private static readonly ArrayPool<byte> Pool = ArrayPool<byte>.Shared;
[Obsolete("Obsolete")]
private static async Task GetStream(string url, long start, long end, MemoryMappedFile mmf, int bufferSize, CancellationToken cancellationToken)
{
var r = (HttpWebRequest)WebRequest.Create(url);
r.Method = "GET";
r.AddRange(start,end);
r.Pipelined = true;
r.KeepAlive = true;
using var response = await r.GetResponseAsync().ConfigureAwait(false);
var buffer = Pool.Rent(bufferSize);
try
{
await using var stream = response.GetResponseStream();
var accessor = mmf.CreateViewAccessor(start, response.ContentLength, MemoryMappedFileAccess.ReadWrite);
var bytesRead = 0;
var brtot = 0;
var total = 0;
int i = -1;
do
{
Array.Clear(buffer);
var offset = 0;
do {
bytesRead = await stream.ReadAsync(buffer, offset, bufferSize - offset, cancellationToken).ConfigureAwait(false);
brtot += bytesRead;
offset += bytesRead;
} while (bytesRead != 0 && offset < bufferSize);
if (offset == 0) {continue;}
i = i + 1;
if (brtot - total > 1)
{
CopyBytes(buffer, total, accessor, i == 0 ? brtot : brtot - total);
Console.WriteLine(String.Format("{0} - {1} ==== {2}", total, brtot - bytesRead, i));
total = brtot + 1;
}
} while (bytesRead != 0);
await stream.FlushAsync(cancellationToken);
}
finally
{
Pool.Return(buffer);
response.Close();
}
}
private static void CopyBytes(byte[] buff, int offset, MemoryMappedViewAccessor s, int length)
{
unsafe
{
var pointer = (byte*)0;
RuntimeHelpers.PrepareConstrainedRegions();
try {
s.SafeMemoryMappedViewHandle.AcquirePointer(ref pointer);
Marshal.Copy(buff, 0, IntPtr.Add(new IntPtr(pointer), offset), length);
}
finally {
if (pointer != null)
s.SafeMemoryMappedViewHandle.ReleasePointer();
}
}
}
Then I have this code that calls it:
public static async Task DownloadFile(string url, String outFile, int parts, int bufferSize)
{
var responseLength = (await WebRequest.Create(url).GetResponseAsync()).ContentLength;
var partSize = (long)Math.Floor(responseLength / (parts + 0.0));
var pieces = new List<ValueTuple<long, long>>();
var uri = new Uri(url);
var filename = outFile ?? Path.GetFileName(uri.LocalPath);
Console.WriteLine(responseLength);
Console.WriteLine(partSize);
ServicePointManager.Expect100Continue = false;
ServicePointManager.DefaultConnectionLimit = 10000;
ServicePointManager.FindServicePoint(new Uri(url)).ConnectionLimit = parts;
#if NET6_0_OR_GREATER
ServicePointManager.ReusePort = true;
#endif
//Loop to add all the events to the queue
for (long i = 0; i < responseLength; i += partSize) {
pieces.Insert(0,(i + partSize < responseLength ? new ValueTuple<long, long>(i, i + partSize) : new ValueTuple<long, long>(i, responseLength)));
}
var cs = new CancellationToken(false);
File.Delete(filename);
var mmf = MemoryMappedFile.CreateFromFile(filename, FileMode.CreateNew,null, responseLength, MemoryMappedFileAccess.ReadWrite);
GCSettings.LatencyMode = GCLatencyMode.SustainedLowLatency;
await Parallel.ForEachAsync(pieces, new ParallelOptions() { MaxDegreeOfParallelism = Environment.ProcessorCount}, async (tuple, token) =>
{
await GetStream(url,tuple.Item1, tuple.Item2, mmf, bufferSize, cs);
});
}
The only issue with the code is that it is corrupting whatever file I download with no errors. I am sure this is due to my parameters specified to CopyBytes(). But unfortunately, I can't get the right calculations for the parameters, can someone help me with this or at least point me on the right direction? I know some of the methods are obsolete, and I will address this, but I am currently more focused on getting the unsafe memory copying to work. Thank you!
So I want to be able to seek to a point in a fileStream, then read forward using a StreamReader. Then seek forward again, and use the StreamReader to read another chunk of data.
const int BufferSize = 4096;
var buffer = new char[BufferSize];
var endpoints = new List<long>();
using (var fileStream = new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.Read))
{
var fileLength = fileStream.Length;
var seekPositionCount = fileLength / concurrentReads;
long currentOffset = 0;
for (var i = 0; i < concurrentReads; i++)
{
var seekPosition = seekPositionCount + currentOffset;
// seek the file forward
fileStream.Seek(seekPosition, SeekOrigin.Current);
// setting true at the end is very important, keeps the underlying fileStream open.
using (var streamReader = new StreamReader(fileStream, Encoding.UTF8, true, BufferSize, true))
{
// this also seeks the file forward the amount in the buffer...
int bytesRead;
var totalBytesRead = 0;
while ((bytesRead = await streamReader.ReadAsync(buffer, 0, buffer.Length)) > 0)
{
totalBytesRead += bytesRead;
var found = false;
var gotR = false;
for (var j = 0; j < buffer.Length; j++)
{
if (buffer[j] == '\r')
{
gotR = true;
continue;
}
if (buffer[j] == '\n' && gotR)
{
// so we add the total bytes read, minus the current buffer amount read, then add how far into the buffer we actually read.
seekPosition += totalBytesRead - BufferSize + j;
endpoints.Add(seekPosition);
found = true;
break;
}
}
if (found) break;
}
}
// we need to seek to the position we got to in the StreamReader (but not going by how much was read).
fileStream.Seek(seekPosition, SeekOrigin.Current);
currentOffset += seekPosition;
}
}
return endpoints;
However, I get to two entries in endpoints and it exits out.
(bytesRead = await streamReader.ReadAsync(buffer, 0, buffer.Length)) > 0
The arguments you pass to ReadAsync I thought are solely to do with the buffer, so the index argument I thought was to say, fill the buffer at index.
I can't make out from Reference Source how this value is used.
I assumed (and can't find the evidence to back up) that, when you opened a StreamReader it uses the underlying Stream as it's guide, so when you ask to read some bytes, it will start from the position the underlying Stream is at...
But the results of what I'm doing aren't showing that, they seem to be showing that the StreamReader is starting at the beginning of the Stream each time - however, I can't find the evidence to support that is how it does it either...
Seeking
Is my understanding of seeking correct, in the sense that if I call seek
fileStream.Seek(seekPosition, SeekOrigin.Current);
If the file is at 300, I want to seek to 600, the above variable seekPosition should be 600??
ReferenceSource would say otherwise:
else if (origin == SeekOrigin.Current) {
// Don't call FlushRead here, which would have caused an infinite
// loop. Simply adjust the seek origin. This isn't necessary
// if we're seeking relative to the beginning or end of the stream.
offset -= (_readLen - _readPos);
}
So thanks to Hans Passant, I have got the answer:
var buffer = new char[BufferSize];
var endpoints = new List<long>();
using (var fileStream = this.CreateMultipleReadAccessFileStream(fileName))
{
var fileLength = fileStream.Length;
var seekPositionCount = fileLength / concurrentReads;
long currentOffset = 0;
for (var i = 0; i < concurrentReads; i++)
{
var seekPosition = seekPositionCount + currentOffset;
// seek the file forward
// fileStream.Seek(seekPosition, SeekOrigin.Current);
// setting true at the end is very important, keeps the underlying fileStream open.
using (var streamReader = this.CreateTemporaryStreamReader(fileStream))
{
// this is poor on performance, hence why you split the file here and read in new threads.
streamReader.DiscardBufferedData();
// you have to advance the fileStream here, because of the previous line
streamReader.BaseStream.Seek(seekPosition, SeekOrigin.Begin);
// this also seeks the file forward the amount in the buffer...
int bytesRead;
var totalBytesRead = 0;
while ((bytesRead = await streamReader.ReadAsync(buffer, 0, buffer.Length)) > 0)
{
totalBytesRead += bytesRead;
var found = false;
var gotR = false;
for (var j = 0; j < buffer.Length; j++)
{
if (buffer[j] == '\r')
{
gotR = true;
continue;
}
if (buffer[j] == '\n' && gotR)
{
// so we add the total bytes read, minus the current buffer amount read, then add how far into the buffer we actually read.
seekPosition += totalBytesRead - BufferSize + j;
endpoints.Add(seekPosition);
found = true;
break;
}
// if we have found new line then move the position to
}
if (found) break;
}
}
currentOffset = seekPosition;
}
}
return endpoints;
Note the new part, rather than doing this twice:
fileStream.Seek(seekPosition, SeekOrigin.Current);
I now use SeekOrigin.Begin and use the StreamReader to progress the underlying base stream:
// this is poor on performance, hence why you split the file here and read in new threads.
streamReader.DiscardBufferedData();
// you have to advance the fileStream here, because of the previous line
streamReader.BaseStream.Seek(seekPosition, SeekOrigin.Begin);
The DiscardBufferedData will mean that I'm always using the underlying stream position.
I am trying to retrieve an uploaded file from my webserver. As the client sends its files through a webform (random files), I need to parse the request to get the file out and to process it further on.
Basically, the code goes as:
HttpListenerContext context = listener.GetContext();
HttpListenerRequest request = context.Request;
StreamReader r = new StreamReader(request.InputStream, System.Text.Encoding.Default);
// this is the retrieved file from streamreader
string file = null;
while ((line = r.ReadLine()) != null){
// i read the stream till i retrieve the filename
// get the file data out and break the loop
}
// A byststream is created by converting the string,
Byte[] bytes = request.ContentEncoding.GetBytes(file);
MemoryStream mstream = new MemoryStream(bytes);
// do the rest
As a result, i am able to retrieve textfiles, but for all other files, they are corrupted.
Could someone tell me how to parse these HttplistnerRequests properly (or providing a lightweighted alternative)?
I think you are making things harder on yourself than necessary by doing this with an HttpListener rather than using the built in facilities of ASP.Net. But if you must do it this way here is some sample code. Note: 1) I'm assuming you're using enctype="multipart/form-data" on your <form>. 2) This code is designed to be used with a form containing only your <input type="file" /> if you want to post other fields or multiple files you'll have to change the code. 3) This is meant to be a proof of concept/example, it may have bugs, and is not particularly flexible.
static void Main(string[] args)
{
HttpListener listener = new HttpListener();
listener.Prefixes.Add("http://localhost:8080/ListenerTest/");
listener.Start();
HttpListenerContext context = listener.GetContext();
SaveFile(context.Request.ContentEncoding, GetBoundary(context.Request.ContentType), context.Request.InputStream);
context.Response.StatusCode = 200;
context.Response.ContentType = "text/html";
using (StreamWriter writer = new StreamWriter(context.Response.OutputStream, Encoding.UTF8))
writer.WriteLine("File Uploaded");
context.Response.Close();
listener.Stop();
}
private static String GetBoundary(String ctype)
{
return "--" + ctype.Split(';')[1].Split('=')[1];
}
private static void SaveFile(Encoding enc, String boundary, Stream input)
{
Byte[] boundaryBytes = enc.GetBytes(boundary);
Int32 boundaryLen = boundaryBytes.Length;
using (FileStream output = new FileStream("data", FileMode.Create, FileAccess.Write))
{
Byte[] buffer = new Byte[1024];
Int32 len = input.Read(buffer, 0, 1024);
Int32 startPos = -1;
// Find start boundary
while (true)
{
if (len == 0)
{
throw new Exception("Start Boundaray Not Found");
}
startPos = IndexOf(buffer, len, boundaryBytes);
if (startPos >= 0)
{
break;
}
else
{
Array.Copy(buffer, len - boundaryLen, buffer, 0, boundaryLen);
len = input.Read(buffer, boundaryLen, 1024 - boundaryLen);
}
}
// Skip four lines (Boundary, Content-Disposition, Content-Type, and a blank)
for (Int32 i = 0; i < 4; i++)
{
while (true)
{
if (len == 0)
{
throw new Exception("Preamble not Found.");
}
startPos = Array.IndexOf(buffer, enc.GetBytes("\n")[0], startPos);
if (startPos >= 0)
{
startPos++;
break;
}
else
{
len = input.Read(buffer, 0, 1024);
}
}
}
Array.Copy(buffer, startPos, buffer, 0, len - startPos);
len = len - startPos;
while (true)
{
Int32 endPos = IndexOf(buffer, len, boundaryBytes);
if (endPos >= 0)
{
if (endPos > 0) output.Write(buffer, 0, endPos-2);
break;
}
else if (len <= boundaryLen)
{
throw new Exception("End Boundaray Not Found");
}
else
{
output.Write(buffer, 0, len - boundaryLen);
Array.Copy(buffer, len - boundaryLen, buffer, 0, boundaryLen);
len = input.Read(buffer, boundaryLen, 1024 - boundaryLen) + boundaryLen;
}
}
}
}
private static Int32 IndexOf(Byte[] buffer, Int32 len, Byte[] boundaryBytes)
{
for (Int32 i = 0; i <= len - boundaryBytes.Length; i++)
{
Boolean match = true;
for (Int32 j = 0; j < boundaryBytes.Length && match; j++)
{
match = buffer[i + j] == boundaryBytes[j];
}
if (match)
{
return i;
}
}
return -1;
}
To help you better understand what the code above is doing, here is what the body of the HTTP POST looks like:
Content-Type: multipart/form-data; boundary=----WebKitFormBoundary9lcB0OZVXSqZLbmv
------WebKitFormBoundary9lcB0OZVXSqZLbmv
Content-Disposition: form-data; name="my_file"; filename="Test.txt"
Content-Type: text/plain
Test
------WebKitFormBoundary9lcB0OZVXSqZLbmv--
I've left out the irrelevant headers. As you can see, you need to parse the body by scanning through to find the beginning and ending boundary sequences, and drop the sub headers that come before the content of your file. Unfortunately you cannot use StreamReader because of the potential for binary data. Also unfortunate is the fact that there is no per file Content-Length (the Content-Length header for the request specifies the total length of the body including boundaries, sub-headers, and spacing.
You can't use StreamReader because it is meant to read streams in which the bytes are in the UTF8. Instead, you want to read the contents of the stream to a receive buffer, remove all the stuff you don't need, get the file extension of the uploaded file, extract the contents of the uploaded file, then save the file contents to a new file. The code I show in this post assumes your form looks like this:
<form enctype="multipart/form-data" method="POST" action="/uploader">
<input type="file" name="file">
<input type="submit">
</form>
As you can see, the code is only meant to handle a form that only has the file. Since there is no way to extract the contents of a file on the server from a application/x-www-form-urlencoded form, so you have to include the "multipart/form-data".
First, for this method of handling uploaded files, you will first need this little bit of code:
using System;
using System.IO;
using System.Text;
using System.Net;
using System.Collections.Generic;
Second, you need to read the contents of the request.InputStream to a receive buffer, or a byte[]. We do this by making a byte[] buffer with the length of the Content-Length header sent by the browser. Then, we read the contents of the request.InputStream to the buffer. The code would look like this:
int len = int.Parse(request.Headers["Content-Length"]);
byte[] buffer = new byte[len];
request.InputStream.Read(buffer, 0, len);
The stream will look somewhat like this:
------WebKitFormBoundary9lcB0OZVXSqZLbmv
Content-Disposition: form-data; name="file"; filename="example-file.txt"
Content-Type: text/plain
file contents here
------WebKitFormBoundary9lcB0OZVXSqZLbmv--
Next, you need to get the file extension of the uploaded file. We can do this using this code:
string fileExtension = Encoding.UTF8.GetString(bytes).Split("\r\n")[1].Split("filename=\"")[1].Replace("\"", "").Split('.')[^1];
Then, we need to get the contents of the file. We do this by removing the stuff at the beginning (the -----WebKitFormBoundary, the Content-Disposition, the Content-Type, and a blank line), then removing the last line of the request body, plus an extra \r\n at the end. Here is the code that does just that:
// note that the variable buffer is the byte[], and the variable bytes is the List<byte>
string stringBuffer = Encoding.UTF8.GetString(buffer);
List<byte> bytes = new List<byte>(buffer);
string[] splitString = stringBuffer.Split('\n');
int lengthOfFourLines = splitString[0].Length + splitString[1].Length +
splitString[2].Length + splitString[3].Length + 4;
bytes.RemoveRange(0, lengthOfFourLines);
int lengthOfLastLine = splitString[^2].Length+2;
bytes.RemoveRange(bytes.Count - lengthOfLastLine, lengthOfLastLine);
buffer = bytes.ToArray();
Finally, we need to save the contents to a file. The code below generates a random file name with the user-specified file extension, and saves the files contents to it.
string fname = "";
string[] chars = "q w e r t y u i o p a s d f g h j k l z x c v b n m Q W E R T Y U I O P A S D F G H J K L Z X C V B N M 1 2 3 4 5 6 7 8 9 0".Split(" ");
for (int i = 0; i < 10; i++)
{
fname += chars[new Random().Next(chars.Length)];
}
fname += fileExtension;
FileStream file = File.Create(fname);
file.Write(buffer);
file.Close();
Here is the whole code put together:
public static void Main()
{
var listener = new HttpListener();
listener.Prefixes.Add("http://localhost:8080/");
listener.Start();
while(true)
{
HttpListenerContext context = listener.GetContext();
HttpListenerRequest request = context.Request;
HttpListenerResponse response = context.Response;
if(request.HttpMethod=="POST") SaveFile(request);
response.OutputStream.Write(Encoding.UTF8.GetBytes("file successfully uploaded"));
response.OutputStream.Close();
}
}
void SaveFile(HttpListenerRequest request)
{
int len = (int)request.ContentLength64;
Console.WriteLine(len);
byte[] buffer = new byte[len];
request.InputStream.Read(buffer, 0, len);
string stringBuffer = Encoding.UTF8.GetString(buffer);
Console.WriteLine(stringBuffer.Replace("\r\n","\\r\\n\n"));
string fileExtension = stringBuffer.Split("\r\n")[1]
.Split("filename=\"")[1]
.Replace("\"", "")
.Split('.')[^1]
;
List<byte> bytes = new List<byte>(buffer);
string[] splitString = stringBuffer.Split('\n');
int lengthOfFourLines = splitString[0].Length + splitString[1].Length + splitString[2].Length + splitString[3].Length + 4;
bytes.RemoveRange(0, lengthOfFourLines);
int lengthOfLastLine = splitString[^2].Length+2;
bytes.RemoveRange(bytes.Count - lengthOfLastLine, lengthOfLastLine);
buffer = bytes.ToArray();
string fname = "";
string[] chars = "q w e r t y u i o p a s d f g h j k l z x c v b n m Q W E R T Y U I O P A S D F G H J K L Z X C V B N M 1 2 3 4 5 6 7 8 9 0".Split(" ");
for (int i = 0; i < 10; i++)
{
fname += chars[new Random().Next(chars.Length)];
}
fname += "." + fileExtension;
FileStream file = File.Create(fname);
file.Write(buffer);
file.Close();
}
Also, if you want to send an uploaded file to the client, here is a useful function that sends the file to the client.
// Make sure you are using System.IO, and System.Net when making this function.
// Also make sure you set the content type of the response before calling this function.
// fileName is the name of the file you want to send to the client, and output is the response.OutputStream.
public static void SendFile(string fileName, Stream output)
{
FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read);
fs.CopyTo(output);
fs.Close();
output.Close();
}
The problem is you are reading the file as text.
You need to read the file as a bytearray instead and using the BinaryReader is better and easier to use than StreamReader:
Byte[] bytes;
using (System.IO.BinaryReader r = new System.IO.BinaryReader(request.InputStream))
{
// Read the data from the stream into the byte array
bytes = r.ReadBytes(Convert.ToInt32(request.InputStream.Length));
}
MemoryStream mstream = new MemoryStream(bytes);
May have bugs, test thoroughly. This one gets all post, get, and files.
using System.Collections.Generic;
using System.Collections.Specialized;
using System.IO;
using System.Net;
using System.Text;
using System.Web;
namespace DUSTLauncher
{
class HttpNameValueCollection
{
public class File
{
private string _fileName;
public string FileName { get { return _fileName ?? (_fileName = ""); } set { _fileName = value; } }
private string _fileData;
public string FileData { get { return _fileData ?? (_fileName = ""); } set { _fileData = value; } }
private string _contentType;
public string ContentType { get { return _contentType ?? (_contentType = ""); } set { _contentType = value; } }
}
private NameValueCollection _get;
private Dictionary<string, File> _files;
private readonly HttpListenerContext _ctx;
public NameValueCollection Get { get { return _get ?? (_get = new NameValueCollection()); } set { _get = value; } }
public NameValueCollection Post { get { return _ctx.Request.QueryString; } }
public Dictionary<string, File> Files { get { return _files ?? (_files = new Dictionary<string, File>()); } set { _files = value; } }
private void PopulatePostMultiPart(string post_string)
{
var boundary_index = _ctx.Request.ContentType.IndexOf("boundary=") + 9;
var boundary = _ctx.Request.ContentType.Substring(boundary_index, _ctx.Request.ContentType.Length - boundary_index);
var upper_bound = post_string.Length - 4;
if (post_string.Substring(2, boundary.Length) != boundary)
throw (new InvalidDataException());
var raw_post_strings = new List<string>();
var current_string = new StringBuilder();
for (var x = 4 + boundary.Length; x < upper_bound; ++x)
{
if (post_string.Substring(x, boundary.Length) == boundary)
{
x += boundary.Length + 1;
raw_post_strings.Add(current_string.ToString().Remove(current_string.Length - 3, 3));
current_string.Clear();
continue;
}
current_string.Append(post_string[x]);
var post_variable_string = current_string.ToString();
var end_of_header = post_variable_string.IndexOf("\r\n\r\n");
if (end_of_header == -1) throw (new InvalidDataException());
var filename_index = post_variable_string.IndexOf("filename=\"", 0, end_of_header);
var filename_starts = filename_index + 10;
var content_type_starts = post_variable_string.IndexOf("Content-Type: ", 0, end_of_header) + 14;
var name_starts = post_variable_string.IndexOf("name=\"") + 6;
var data_starts = end_of_header + 4;
if (filename_index == -1) continue;
var filename = post_variable_string.Substring(filename_starts, post_variable_string.IndexOf("\"", filename_starts) - filename_starts);
var content_type = post_variable_string.Substring(content_type_starts, post_variable_string.IndexOf("\r\n", content_type_starts) - content_type_starts);
var file_data = post_variable_string.Substring(data_starts, post_variable_string.Length - data_starts);
var name = post_variable_string.Substring(name_starts, post_variable_string.IndexOf("\"", name_starts) - name_starts);
Files.Add(name, new File() { FileName = filename, ContentType = content_type, FileData = file_data });
continue;
}
}
private void PopulatePost()
{
if (_ctx.Request.HttpMethod != "POST" || _ctx.Request.ContentType == null) return;
var post_string = new StreamReader(_ctx.Request.InputStream, _ctx.Request.ContentEncoding).ReadToEnd();
if (_ctx.Request.ContentType.StartsWith("multipart/form-data"))
PopulatePostMultiPart(post_string);
else
Get = HttpUtility.ParseQueryString(post_string);
}
public HttpNameValueCollection(ref HttpListenerContext ctx)
{
_ctx = ctx;
PopulatePost();
}
}
}
I like #paul-wheeler answer. However I needed to modify their code to include some additional data (In this case, the directory structure).
I'm using this code to upload files:
var myDropzone = $("#fileDropZone");
myDropzone.dropzone(
{
url: "http://" + self.location.hostname + "/Path/Files.html,
method: "post",
createImageThumbnails: true,
previewTemplate: document.querySelector('#previewTemplateId').innerHTML,
clickable: false,
init: function () {
this.on('sending', function(file, xhr, formData){
// xhr is XMLHttpRequest
var name = file.fullPath;
if (typeof (file.fullPath) === "undefined") {
name = file.name;
}
formData.append('fileNameWithPath', name);
});
}
});
Here is #paul-wheeler modified code. Thanks #paul-wheeler.
public class FileManager
{
public static void SaveFile(HttpListenerRequest request, string savePath)
{
var tempFileName = Path.Combine(savePath, $"{DateTime.Now.Ticks}.tmp");
if (!Directory.Exists(savePath))
{
Directory.CreateDirectory(savePath);
}
var (res, fileName) = SaveTmpFile(request, tempFileName);
if (res)
{
var filePath = Path.Combine(savePath, fileName);
var fileDir = filePath.Substring(0, filePath.LastIndexOf(Path.DirectorySeparatorChar));
if (!Directory.Exists(fileDir))
{
Directory.CreateDirectory(fileDir);
}
if (File.Exists(filePath))
{
File.Delete(filePath);
}
File.Move(tempFileName, filePath);
}
}
private static (bool, string) SaveTmpFile(HttpListenerRequest request, string tempFileName)
{
var enc = request.ContentEncoding;
var boundary = GetBoundary(request.ContentType);
var input = request.InputStream;
byte[] boundaryBytes = enc.GetBytes(boundary);
var boundaryLen = boundaryBytes.Length;
using (FileStream output = new FileStream(tempFileName, FileMode.Create, FileAccess.Write))
{
var buffer = new byte[1024];
var len = input.Read(buffer, 0, 1024);
var startPos = -1;
// Get file name and relative path
var strBuffer = Encoding.Default.GetString(buffer);
var strStart = strBuffer.IndexOf("fileNameWithPath") + 21;
if (strStart < 21)
{
Logger.LogError("File name not found");
return (false, null);
}
var strEnd = strBuffer.IndexOf(boundary, strStart) - 2;
var fileName = strBuffer.Substring(strStart, strEnd - strStart);
fileName = fileName.Replace('/', Path.DirectorySeparatorChar);
// Find start boundary
while (true)
{
if (len == 0)
{
Logger.LogError("Find start boundary not found");
return (false, null);
}
startPos = IndexOf(buffer, len, boundaryBytes);
if (startPos >= 0)
{
break;
}
else
{
Array.Copy(buffer, len - boundaryLen, buffer, 0, boundaryLen);
len = input.Read(buffer, boundaryLen, 1024 - boundaryLen);
}
}
// Advance to data
var foundData = false;
while (!foundData)
{
while (true)
{
if (len == 0)
{
Logger.LogError("Preamble not Found");
return (false, null);
}
startPos = Array.IndexOf(buffer, enc.GetBytes("\n")[0], startPos);
if (startPos >= 0)
{
startPos++;
break;
}
else
{
// In case read in line is longer than buffer
len = input.Read(buffer, 0, 1024);
}
}
var currStr = Encoding.Default.GetString(buffer).Substring(startPos);
if (currStr.StartsWith("Content-Type:"))
{
// Go past the last carriage-return\line-break. (\r\n)
startPos = Array.IndexOf(buffer, enc.GetBytes("\n")[0], startPos) + 3;
break;
}
}
Array.Copy(buffer, startPos, buffer, 0, len - startPos);
len = len - startPos;
while (true)
{
var endPos = IndexOf(buffer, len, boundaryBytes);
if (endPos >= 0)
{
if (endPos > 0) output.Write(buffer, 0, endPos - 2);
break;
}
else if (len <= boundaryLen)
{
Logger.LogError("End Boundaray Not Found");
return (false, null);
}
else
{
output.Write(buffer, 0, len - boundaryLen);
Array.Copy(buffer, len - boundaryLen, buffer, 0, boundaryLen);
len = input.Read(buffer, boundaryLen, 1024 - boundaryLen) + boundaryLen;
}
}
return (true, fileName);
}
}
private static int IndexOf(byte[] buffer, int len, byte[] boundaryBytes)
{
for (int i = 0; i <= len - boundaryBytes.Length; i++)
{
var match = true;
for (var j = 0; j < boundaryBytes.Length && match; j++)
{
match = buffer[i + j] == boundaryBytes[j];
}
if (match)
{
return i;
}
}
return -1;
}
private static string GetBoundary(string ctype)
{
return "--" + ctype.Split(';')[1].Split('=')[1];
}
}
using Microsoft.AspNetCore.WebUtilities;
using Microsoft.Net.Http.Headers;
var contentType = MediaTypeHeaderValue.Parse(context.Request.ContentType);
var boundary = HeaderUtilities.RemoveQuotes(contentType.Boundary).Value;
var multipartReader = new MultipartReader(boundary, context.Request.InputStream);
var section = (await multipartReader.ReadNextSectionAsync()).AsFileSection();
var fileName = section.FileName;
var fileStream = section.FileStream;
for a given HttpListenerRequest req and a string path the following code saves a file (worked for text file but also png file)
int len = (int)req.ContentLength64;
byte[] buffer = new byte[len];
int totalRead = 0;
while(totalRead < len){
// InputStream.Read does not read always read full stream (so loop until it has)
totalRead += req.InputStream.Read(buffer, totalRead, len - totalRead);
}
string stringBuffer = Encoding.UTF8.GetString(buffer);
string startTag = stringBuffer.Substring(0, stringBuffer.IndexOf("\r\n\r\n") + 4);
string endTag = stringBuffer.Substring(stringBuffer.IndexOf("\r\n------WebKitFormBoundary"));
List<byte> bytes = new List<byte>(buffer);
bytes = bytes.GetRange(startTag.Length, len - (startTag.Length + endTag.Length));
buffer = bytes.ToArray();
File.WriteAllBytes(path, buffer);
(build on top of copee moo solution)
I'm making a simple file transfer sender and receiver app through the wire. What I have so far is that the sender converts the file into a byte array and sends chunks of that array to the receiver.
This works with file of up to 256mb, but this line throws a "System out of memory" exception for anything above:
byte[] buffer = StreamFile(fileName); //This is where I convert the file
I'm looking for a way to read the file in chunks then write that chunk instead of loading the whole file into a byte. How can I do this with a FileStream?
EDIT:
Sorry, heres my crappy code so far:
private void btnSend(object sender, EventArgs e)
{
Socket clientSock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
byte[] fileName = Encoding.UTF8.GetBytes(fName); //file name
byte[] fileData = null;
try
{
fileData = StreamFile(textBox1.Text); //file
}
catch (OutOfMemoryException ex)
{
MessageBox.Show("Out of memory");
return;
}
byte[] fileNameLen = BitConverter.GetBytes(fileName.Length); //length of file name
clientData = new byte[4 + fileName.Length + fileData.Length];
fileNameLen.CopyTo(clientData, 0);
fileName.CopyTo(clientData, 4);
fileData.CopyTo(clientData, 4 + fileName.Length);
clientSock.Connect("172.16.12.91", 9050);
clientSock.Send(clientData, 0, 4 + fileName.Length, SocketFlags.None);
for (int i = 4 + fileName.Length; i < clientData.Length; i++)
{
clientSock.Send(clientData, i, 1 , SocketFlags.None);
}
clientSock.Close();
}
And here's how I receive (the code was from a tutorial)
public void ReadCallback(IAsyncResult ar)
{
int fileNameLen = 1;
String content = String.Empty;
StateObject state = (StateObject)ar.AsyncState;
Socket handler = state.workSocket;
int bytesRead = handler.EndReceive(ar);
if (bytesRead > 0)
{
if (flag == 0)
{
Thread.Sleep(1000);
fileNameLen = BitConverter.ToInt32(state.buffer, 0);
string fileName = Encoding.UTF8.GetString(state.buffer, 4, fileNameLen);
receivedPath = fileName;
flag++;
}
if (flag >= 1)
{
BinaryWriter writer = new BinaryWriter(File.Open(receivedPath, FileMode.Append));
if (flag == 1)
{
writer.Write(state.buffer, 4 + fileNameLen, bytesRead - (4 + fileNameLen));
flag++;
}
else
writer.Write(state.buffer, 0, bytesRead);
writer.Close();
handler.BeginReceive(state.buffer, 0, StateObject.BufferSize, 0,
new AsyncCallback(ReadCallback), state);
}
}
else
{
Invoke(new MyDelegate(LabelWriter));
}
}
I just really want to know how I can read the file in chunks so that I dont need to convert it to a byte.
Thanks for the responses so far, I think I'm starting to get it :D
Just call Read repeatedly with a small buffer (I tend to use something like 16K). Note that the call to Read may end up reading a smaller amount than you request. If you're using a fixed chunk size and need the whole chunk in memory, you could just use an array of that size of course.
Without knowing how you're sending the file, it's hard to give much advice about how to structure your code, but it could be something like this:
byte[] chunk = new byte[MaxChunkSize];
while (true)
{
int index = 0;
// There are various different ways of structuring this bit of code.
// Fundamentally we're trying to keep reading in to our chunk until
// either we reach the end of the stream, or we've read everything we need.
while (index < chunk.Length)
{
int bytesRead = stream.Read(chunk, index, chunk.Length - index);
if (bytesRead == 0)
{
break;
}
index += bytesRead;
}
if (index != 0) // Our previous chunk may have been the last one
{
SendChunk(chunk, index); // index is the number of bytes in the chunk
}
if (index != chunk.Length) // We didn't read a full chunk: we're done
{
return;
}
}
If I was more awake I'd probably find a more readable way of writing this, but it'll do for now. One option is to extract another method from the middle section:
// Attempts to read an entire chunk into the given array; returns the size of
// chunk actually read.
int ReadChunk(Stream stream, byte[] chunk)
{
int index = 0;
while (index < chunk.Length)
{
int bytesRead = stream.Read(chunk, index, chunk.Length - index);
if (bytesRead == 0)
{
break;
}
index += bytesRead;
}
return index;
}
var b = new byte[1<<15]; // 32k
while((count = inStream.Read(b, 0, b.Length)) > 0)
{
outStream.Write(b, 0, count);
}
public static IEnumerable<byte[]> SplitStreamIntoChunks(Stream stream, int chunkSize)
{
var bytesRemaining = stream.Length;
while (bytesRemaining > 0)
{
var size = Math.Min((int) bytesRemaining, chunkSize);
var buffer = new byte[size];
var bytesRead = stream.Read(buffer, 0, size);
if (bytesRead <= 0)
break;
yield return buffer;
bytesRemaining -= bytesRead;
}
}
int bufferlength = 12488;
int pointer = 1;
int offset = 0;
int length = 0;
FileStream fstwrite = new FileStream("D:\\Movie.wmv", FileMode.Create);
while (pointer != 0)
{
byte[] buff = new byte[bufferlength];
FileStream fst = new FileStream("E:\\Movie.wmv", FileMode.Open);
pointer = fst.Read(buff, 0, bufferlength);
fst.Close();
fstwrite.Write(buff, offset , pointer);
offset += pointer;
}
I used the above code for splitting a file and place it in other drive.Im not able to set the correct offset and length for this routine can anyone help me to fix this
splitting in the sense ,i split it in "x" kbs and pass it somewhere make the same file in some other location
I find it atlast ,thanks to evry one who gave their valueble responses.
Currently you're always reading from the start of the file... and even if you weren't you'd just be copying the whole file.
Here's some code which will actually split a single file into multiple files:
public static void SplitFile(string inputFile,
string outputPrefix,
int chunkSize)
{
byte[] buffer = new byte[chunkSize];
using (Stream input = File.OpenRead(inputFile))
{
int index = 0;
while (input.Position < input.Length)
{
using (Stream output = File.Create(outputPrefix + index))
{
int chunkBytesRead = 0;
while (chunkBytesRead < chunkSize)
{
int bytesRead = input.Read(buffer,
chunkBytesRead,
chunkSize - chunkBytesRead);
// End of input
if (bytesRead == 0)
{
break;
}
chunkBytesRead += bytesRead;
}
output.Write(buffer, 0, chunkBytesRead);
}
index++;
}
}
}
Your reading bufferlength of bytes. Shouldn't you set the offset like this then?
offset += bufferlength;
Don't open your source file inside the loop, or you'll always read the first chunk.
Open it before the loop, then make sure your offset is applied to the read.