Related
After looking at multiple questions/answers I couldn't find a solution for my problem. I remember I got this code from some question here at StackOverflow and it works perfectly but just for one file. What I want is multiple files.
This is the original CopyTo Function:
public static void CopyTo(this FileInfo file, FileInfo destination, Action<int> progressCallback)
{
const int bufferSize = 1024 * 1024; //1MB
byte[] buffer = new byte[bufferSize], buffer2 = new byte[bufferSize];
bool swap = false;
int progress = 0, reportedProgress = 0, read = 0;
long len = file.Length;
float flen = len;
Task writer = null;
using (var source = file.OpenRead())
using (var dest = destination.OpenWrite())
{
//dest.SetLength(source.Length);
for (long size = 0; size < len; size += read)
{
if ((progress = ((int)((size / flen) * 100))) != reportedProgress)
progressCallback(reportedProgress = progress);
read = source.Read(swap ? buffer : buffer2, 0, bufferSize);
writer?.Wait(); // if < .NET4 // if (writer != null) writer.Wait();
writer = dest.WriteAsync(swap ? buffer : buffer2, 0, read);
swap = !swap;
}
writer?.Wait(); //Fixed - Thanks #sam-hocevar
}
}
So here is how I start the file copy process:
var ficheiro = ficheirosCopia.ElementAt(x);
var _source = new FileInfo(ficheiro.Key);
var _destination = new FileInfo(ficheiro.Value);
if (_destination.Exists)
{
_destination.Delete();
}
Task.Run(() =>
{
_source.CopyTo(_destination, perc => Dispatcher.Invoke(() => progressBar.SetProgress(perc)));
}).GetAwaiter().OnCompleted(() => MessageBox.Show("File Copied!"));
This works very well when I copy only one file but I need to copy multiple files. So I've started to change things a bit:
public static void CopyTo(Dictionary<string, string> files, Action<int> progressCallback)
{
int globalProgress = 0, globalReportedProgress = 0, globalRead = 0;
for (var x = 0; x < files.Count; x++)
{
var item = files.ElementAt(x);
var file = new FileInfo(item.Key);
var destination = new FileInfo(item.Value);
const int bufferSize = 1024 * 1024; //1MB
byte[] buffer = new byte[bufferSize], buffer2 = new byte[bufferSize];
bool swap = false;
int progress = 0, reportedProgress = 0, read = 0;
long len = file.Length;
float flen = len;
Task writer = null;
using (var source = file.OpenRead())
using (var dest = destination.OpenWrite())
{
for (long size = 0; size < len; size += read)
{
if ((progress = ((int)((size / flen) * 100))) != reportedProgress)
progressCallback(reportedProgress = progress);
read = source.Read(swap ? buffer : buffer2, 0, bufferSize);
writer?.Wait(); // if < .NET4 // if (writer != null) writer.Wait();
writer = dest.WriteAsync(swap ? buffer : buffer2, 0, read);
swap = !swap;
}
writer?.Wait(); //Fixed - Thanks #sam-hocevar
}
}
}
Of course this code has a lot of errors but I can't understand how this should be done.
The main goal would be to Start a single task for multiple tiles and having progresscallback for global copy. Receiving a Dictionary (it's already created on other part of the code) as a parameter.
I came up with two approaches to this, one reporting progress after each file, and the other reporting progress every n bytes.
namespace StackOverflow41750117CopyProgress
{
using System;
using System.Collections.Generic;
using System.IO;
public class Batch
{
private bool _overwrite;
/// <summary>
/// Initializes a new instance of the <see cref="Batch"/> class.
/// </summary>
/// <param name="overwrite">
/// True to overwrite the destination file if it already exists (default),
/// false to throw an exception if the destination file already exists.
/// </param>
public Batch(bool overwrite = true)
{
this._overwrite = overwrite;
}
/// <summary>
/// Copies the files, reporting progress once per file.
/// </summary>
/// <param name="filesToCopy">
/// A dictionary with the paths of the source files as its keys, and the path to the destination file as its values.
/// </param>
/// <param name="progressCallback">
/// A callback which accepts two Int64 parameters - the number of bytes copied so far, and the total number of bytes to copy.
/// </param>
public void CopyReportingPerFile(Dictionary<string, string> filesToCopy, Action<long, long> progressCallback)
{
var bytesToCopy = this.GetTotalFileSize(filesToCopy);
long totalBytesCopied = 0;
foreach (var copy in filesToCopy)
{
File.Copy(copy.Key, copy.Value, this._overwrite);
totalBytesCopied += new FileInfo(copy.Key).Length;
progressCallback(totalBytesCopied, bytesToCopy);
}
}
/// <summary>
/// Copies the files, reporting progress once per read/write operation.
/// </summary>
/// <param name="filesToCopy">
/// A dictionary with the paths of the source files as its keys, and the path to the destination file as its values.
/// </param>
/// <param name="progressCallback">
/// A callback which accepts two Int64 parameters - the number of bytes copied so far, and the total number of bytes to copy.
/// </param>
public void CopyReportingPerBuffer(Dictionary<string, string> filesToCopy, Action<long, long> progressCalllback)
{
var bytesToCopy = this.GetTotalFileSize(filesToCopy);
var bufferSize = 1024 * 1024 * 50;
var buffer = new byte[bufferSize];
var span = new Span<byte>(buffer);
long totalBytesCopied = 0;
foreach (var copy in filesToCopy)
{
using (var source = File.OpenRead(copy.Key))
using (var destination = File.OpenWrite(copy.Value))
{
int bytesRead = 0;
do
{
// The Read method returns 0 once we've reached the end of the file
bytesRead = source.Read(span);
destination.Write(span);
totalBytesCopied += bytesRead;
progressCalllback(totalBytesCopied, bytesToCopy);
} while (bytesRead > 0);
source.Close();
destination.Close();
}
}
}
private long GetTotalFileSize(Dictionary<string, string> filesToCopy)
{
long bytesToCopy = 0;
foreach (var filename in filesToCopy.Keys)
{
var fileInfo = new FileInfo(filename);
bytesToCopy += fileInfo.Length;
}
return bytesToCopy;
}
}
}
Usage:
namespace StackOverflow41750117CopyProgress
{
using System;
using System.Collections.Generic;
using System.IO;
public class Program
{
public static void Main(string[] args)
{
var filesToCopy = new Dictionary<string, string>();
filesToCopy.Add(#"C:\temp\1.mp4", #"C:\temp\1copy.mp4");
filesToCopy.Add(#"C:\temp\2.mp4", #"C:\temp\2copy.mp4");
filesToCopy.Add(#"C:\temp\3.mp4", #"C:\temp\3copy.mp4");
filesToCopy.Add(#"C:\temp\4.mp4", #"C:\temp\4copy.mp4");
filesToCopy.Add(#"C:\temp\5.mp4", #"C:\temp\5copy.mp4");
filesToCopy.Add(#"C:\temp\6.mp4", #"C:\temp\6copy.mp4");
filesToCopy.Add(#"C:\temp\7.mp4", #"C:\temp\7copy.mp4");
// Make sure the destination files don't already exist
foreach (var copy in filesToCopy)
{
File.Delete(copy.Value);
}
var batch = new Batch();
Console.WriteLine($"Started {DateTime.Now}");
batch.CopyReportingPerFile(filesToCopy, (bytesCopied, bytesToCopy) => Console.WriteLine($"Copied {bytesCopied} bytes of {bytesToCopy}"));
//batch.CopyReportingPerBuffer(filesToCopy, (bytesCopied, bytesToCopy) => Console.WriteLine($"Copied {bytesCopied} bytes of {bytesToCopy}"));
Console.WriteLine($"Finished {DateTime.Now}");
}
}
}
A few observations...
Reporting progress once per file was easier to implement but doesn't meet the requirements of the question, and isn't very responsive if you're copying a small number of large files.
Using File.Copy preserves the original file's modified date, reading the files into memory and then writing them does not.
Increasing the buffer size from 1MB to 10MB and them 50MB increased the memory usage and also improved the performance, although most of that performance improvement seems to be as a result of calling Console.Writeline in my progressCallback less often, rather than increasing the speed of the disk I/O.
The optimum balance between performance and frequency of progress reports will depend on your circumstances and the spec of the machine running the process, but I found a 50MB buffer results in a progress report roughly once per second.
Note the use of Span<byte> rather than byte[] for the buffer that the data is read into and written from - this removes the need for my code to keep track of the current position in the file (and that's something new I learned today).
I know I'm rather late to this question, but hopefully someone will find this useful.
I am currently working on a networking project where I worked out a binary protocol. My packets look like this:
[1 byte TYPE][2 bytes INDEX][2 bytes LENGTH][LENGTH bytes DATA]
And here's the code where I am receiving the packets:
NetworkStream clientStream= Client.GetStream();
while (Client.Connected)
{
Thread.Sleep(10);
try
{
if (clientStream.DataAvailable)
{
byte[] infobuffer = new byte[5];
int inforead = clientStream.Read(infobuffer, 0, 5);
if (inforead < 5) { continue; }
byte[] rawclient = new byte[2];
Array.Copy(infobuffer, 1, rawclient, 0, 2);
PacketType type = (PacketType)Convert.ToSByte(infobuffer[0]);
int clientIndex = BitConverter.ToInt16(rawclient, 0);
int readLength = BitConverter.ToInt16(infobuffer, 3);
byte[] readbuffer = new byte[readLength];
int count_read = clientStream.Read(readbuffer, 0, readLength);
byte[] read_data = new byte[count_read];
Array.Copy(readbuffer, read_data, count_read);
HandleData(read_data, type, clientIndex);
}
}
catch (Exception ex)
{
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine("[E] " + ex.GetType().ToString());
Console.ResetColor();
break;
}
}
Well, and everything works fine... as long as I run it on 127.0.0.1. As soon as I try testing it over long distance, packets somehow get lost, and I am getting an overflow-exception on the line where I convert the first byte to PacketType. Also, if I try to convert the other values to int16, I get very strange values.
I assume the stream somehow looses some bytes on its way to the server, but can this be? Or is it just a little mistake of mine somewhere in the code?
edit:
I now edited the code, now it reads till it gets its 5 bytes. But I still get the same exception over long distance...
NetworkStream clientStream = Client.GetStream();
while (Client.Connected)
{
Thread.Sleep(10);
try
{
if (clientStream.DataAvailable)
{
int totalread = 0;
byte[] infobuffer = new byte[5];
while (totalread < 5)
{
int inforead = clientStream.Read(infobuffer, totalread, 5 - totalread);
if (inforead == 0)
{ break; }
totalread += inforead;
}
byte[] rawclient = new byte[2];
Array.Copy(infobuffer, 1, rawclient, 0, 2);
PacketType type = (PacketType)Convert.ToSByte(infobuffer[0]);
int clientIndex = BitConverter.ToInt16(rawclient, 0);
int readLength = BitConverter.ToInt16(infobuffer, 3);
byte[] readbuffer = new byte[readLength];
int count_read = clientStream.Read(readbuffer, 0, readLength);
byte[] read_data = new byte[count_read];
Array.Copy(readbuffer, read_data, count_read);
HandleData(read_data, type, clientIndex);
}
}
catch (Exception ex)
{
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine("[E] " + ex.GetType().ToString());
Console.ResetColor();
break;
}
}
PacketType is an enum:
public enum PacketType
{
AddressSocks5 = 0,
Status = 1,
Data = 2,
Disconnect = 3,
AddressSocks4 = 4
}
So many things you're doing wrong here... so many bugs... where to even start...
First Network polling? Really? That's just a naïve way of doing network activity in this day and age.. but I won't harp on that.
Second, with this type of protocol, it's pretty easy to get "out of sync" and once you do, you have no way to get back in sync. This is typically accomplished with some kind of "framing protocol" which provides a unique sequence of bytes that you can use to indicate the start and end of a frame, so that if you ever find yourself out of sync you can read data until you get back in sync. Yes, you will lose data, but you've already lost it if you're out of sync.
Third, you're not really doing anything huge here, so I shamelessly stole the "ReadWholeArray" code from here, it's not the most efficient, but it works and there is other code there that might help:
http://www.yoda.arachsys.com/csharp/readbinary.html
Note: you don't mention how you are serializing the length, type and index values on the other side. So using the BitConverter may be the wrong thing depending on how that was done.
if (clientStream.DataAvailable)
{
byte[] data = new byte[5];
// if it can't read all 5 bytes, it throws an exception
ReadWholeArray(clientStream, data);
PacketType type = (PacketType)Convert.ToSByte(data[0]);
int clientIndex = BitConverter.ToInt16(data, 1);
int readLength = BitConverter.ToInt16(data, 3);
byte[] rawdata = new byte[readLength];
ReadWholeArray(clientStream, rawdata);
HandleData(rawdata, type, clientIndex);
}
/// <summary>
/// Reads data into a complete array, throwing an EndOfStreamException
/// if the stream runs out of data first, or if an IOException
/// naturally occurs.
/// </summary>
/// <param name="stream">The stream to read data from</param>
/// <param name="data">The array to read bytes into. The array
/// will be completely filled from the stream, so an appropriate
/// size must be given.</param>
public static void ReadWholeArray (Stream stream, byte[] data)
{
int offset=0;
int remaining = data.Length;
while (remaining > 0)
{
int read = stream.Read(data, offset, remaining);
if (read <= 0)
throw new EndOfStreamException
(String.Format("End of stream reached with {0} bytes left to read", remaining));
remaining -= read;
offset += read;
}
}
I think the problem is in these lines
int inforead = clientStream.Read(infobuffer, 0, 5);
if (inforead < 5) { continue; }
what happen to your previously read data if the length is under 5 byte? you should save the bytes you have read so far and append next bytes so you can have the header completely
You Read 5 - totalRead.
let totalRead equal 5 or more. When that happens you read nothing, and in cases of 1 - 4 you read that many arbitrary bytes. Not 5. You also then discard any result of less then 5.
You also copy at a offset 1 or another offset without really knowing the offset.
BitConverter.ToInt16(infobuffer, 3);
Is an example of this, what is at off 2?
So if it's not that (decoding error) and and not the structure of your data then unless you change the structure of your loop its you who's losing the bytes not the NetworkStream.
Calculate totalRead by increments of justRead when you recieve so you can handle any size of data as well as receiving it at the correct offset.
So, it would seem that a blocking Read() can return before it is done receiving all of the data being sent to it. In turn we wrap the Read() with a loop that is controlled by the DataAvailable value from the stream in question. The problem is that you can receive more data while in this while loop, but there is no behind the scenes processing going on to let the system know this. Most of the solutions I have found to this on the net have not been applicable in one way or another to me.
What I have ended up doing is as the last step in my loop, I do a simple Thread.Sleep(1) after reading each block from the stream. This appears to give the system time to update and I am not getting accurate results but this seems a bit hacky and quite a bit 'circumstantial' for a solution.
Here is a list of the circumstances I am dealing with: Single TCP Connection between an IIS Application and a standalone application, both written in C# for send/receive communication. It sends a request and then waits for a response. This request is initiated by an HTTP request, but I am not having this issue reading data from the HTTP Request, it is after the fact.
Here is the basic code for handling an incoming connection
protected void OnClientCommunication(TcpClient oClient)
{
NetworkStream stream = oClient.GetStream();
MemoryStream msIn = new MemoryStream();
byte[] aMessage = new byte[4096];
int iBytesRead = 0;
while ( stream.DataAvailable )
{
int iRead = stream.Read(aMessage, 0, aMessage.Length);
iBytesRead += iRead;
msIn.Write(aMessage, 0, iRead);
Thread.Sleep(1);
}
MemoryStream msOut = new MemoryStream();
// .. Do some processing adding data to the msOut stream
msOut.WriteTo(stream);
stream.Flush();
oClient.Close();
}
All feedback welcome for a better solution or just a thumbs up on needing to give that Sleep(1) a go to allow things to update properly before we check the DataAvailable value.
Guess I am hoping after 2 years that the answer to this question isn't how things still are :)
You have to know how much data you need to read; you cannot simply loop reading data until there is no more data, because you can never be sure that no more is going to come.
This is why HTTP GET results have a byte count in the HTTP headers: so the client side will know when it has received all the data.
Here are two solutions for you depending on whether you have control over what the other side is sending:
Use "framing" characters: (SB)data(EB), where SB and EB are start-block and end-block characters (of your choosing) but which CANNOT occur inside the data. When you "see" EB, you know you are done.
Implement a length field in front of each message to indicate how much data follows: (len)data. Read (len), then read (len) bytes; repeat as necessary.
This isn't like reading from a file where a zero-length read means end-of-data (that DOES mean the other side has disconnected, but that's another story).
A third (not recommended) solution is that you can implement a timer. Once you start getting data, set the timer. If the receive loop is idle for some period of time (say a few seconds, if data doesn't come often), you can probably assume no more data is coming. This last method is a last resort... it's not very reliable, hard to tune, and it's fragile.
I'm seeing a problem with this.
You're expecting that the communication will be faster than the while() loop, which is very unlikely.
The while() loop will finish as soon as there is no more data, which may not be the case a few milliseconds just after it exits.
Are you expecting a certain amount of bytes?
How often is OnClientCommunication() fired? Who triggers it?
What do you do with the data after the while() loop? Do you keep appending to previous data?
DataAvailable WILL return false because you're reading faster than the communication, so that's fine only if you keep coming back to this code block to process more data coming in.
I was trying to check DataAvailable before reading data from a network stream and it would return false, although after reading a single byte it would return true. So I checked the MSDN documentation and they also read before checking. I would re-arrange the while loop to a do while loop to follow this pattern.
http://msdn.microsoft.com/en-us/library/system.net.sockets.networkstream.dataavailable.aspx
// Check to see if this NetworkStream is readable.
if(myNetworkStream.CanRead){
byte[] myReadBuffer = new byte[1024];
StringBuilder myCompleteMessage = new StringBuilder();
int numberOfBytesRead = 0;
// Incoming message may be larger than the buffer size.
do{
numberOfBytesRead = myNetworkStream.Read(myReadBuffer, 0, myReadBuffer.Length);
myCompleteMessage.AppendFormat("{0}", Encoding.ASCII.GetString(myReadBuffer, 0, numberOfBytesRead));
}
while(myNetworkStream.DataAvailable);
// Print out the received message to the console.
Console.WriteLine("You received the following message : " +
myCompleteMessage);
}
else{
Console.WriteLine("Sorry. You cannot read from this NetworkStream.");
}
When I have this code:
var readBuffer = new byte[1024];
using (var memoryStream = new MemoryStream())
{
do
{
int numberOfBytesRead = networkStream.Read(readBuffer, 0, readBuffer.Length);
memoryStream.Write(readBuffer, 0, numberOfBytesRead);
}
while (networkStream.DataAvailable);
}
From what I can observe:
When sender sends 1000 bytes and reader wants to read them. Then I suspect that NetworkStream somehow "knows" that it should receive 1000 bytes.
When I call .Read before any data arrives from NetworkStream then .Read should be blocking until it gets more than 0 bytes (or more if .NoDelay is false on networkStream)
Then when I read first batch of data I suspect that .Read is somehow updating from its result the counter of those 1000 bytes at NetworkStream and before this happens I suspect, that in this time the .DataAvailable is set to false and after the counter is updated then the .DataAvailable is then set to correct value if the counter data is less than 1000 bytes. It makes sense when you think about it. Because otherwise it would go to the next cycle before checking that 1000 bytes arrived and the .Read method would be blocking indefinitely, because reader could have already read 1000 bytes and no more data would arrive.
This I think is the point of failure here as already James said:
Yes, this is just the way these libraries work. They need to be given time to run to fully validate the data incoming. – James Apr 20 '16 at 5:24
I suspect that the update of internal counter between end of .Read and before accessing .DataAvailable is not as atomic operation (transaction) so the TcpClient needs more time to properly set the DataAvailable.
When I have this code:
var readBuffer = new byte[1024];
using (var memoryStream = new MemoryStream())
{
do
{
int numberOfBytesRead = networkStream.Read(readBuffer, 0, readBuffer.Length);
memoryStream.Write(readBuffer, 0, numberOfBytesRead);
if (!networkStream.DataAvailable)
System.Threading.Thread.Sleep(1); //Or 50 for non-believers ;)
}
while (networkStream.DataAvailable);
}
Then the NetworkStream have enough time to properly set .DataAvailable and this method should function correctly.
Fun fact... This seems to be somehow OS Version dependent. Because the first function without sleep worked for me on Win XP and Win 10, but was failing to receive whole 1000 bytes on Win 7. Don't ask me why, but I tested it quite thoroughly and it was easily reproducible.
Using TcpClient.Available will allow this code to read exactly what is available each time. TcpClient.Available is automatically set to TcpClient.ReceiveBufferSize when the amount of data remaining to be read is greater than or equal to TcpClient.ReceiveBufferSize. Otherwise it is set to the size of the remaining data.
Hence, you can indicate the maximum amount of data that is available for each read by setting TcpClient.ReceiveBufferSize (e.g., oClient.ReceiveBufferSize = 4096;).
protected void OnClientCommunication(TcpClient oClient)
{
NetworkStream stream = oClient.GetStream();
MemoryStream msIn = new MemoryStream();
byte[] aMessage;
oClient.ReceiveBufferSize = 4096;
int iBytesRead = 0;
while (stream.DataAvailable)
{
int myBufferSize = (oClient.Available < 1) ? 1 : oClient.Available;
aMessage = new byte[oClient.Available];
int iRead = stream.Read(aMessage, 0, aMessage.Length);
iBytesRead += iRead;
msIn.Write(aMessage, 0, iRead);
}
MemoryStream msOut = new MemoryStream();
// .. Do some processing adding data to the msOut stream
msOut.WriteTo(stream);
stream.Flush();
oClient.Close();
}
public class NetworkStream
{
private readonly Socket m_Socket;
public NetworkStream(Socket socket)
{
m_Socket = socket ?? throw new ArgumentNullException(nameof(socket));
}
public void Send(string message)
{
if (message is null)
{
throw new ArgumentNullException(nameof(message));
}
byte[] data = Encoding.UTF8.GetBytes(message);
SendInternal(data);
}
public string Receive()
{
byte[] buffer = ReceiveInternal();
string message = Encoding.UTF8.GetString(buffer);
return message;
}
private void SendInternal(byte[] message)
{
int size = message.Length;
if (size == 0)
{
m_Socket.Send(BitConverter.GetBytes(size), 0, sizeof(int), SocketFlags.None);
}
else
{
m_Socket.Send(BitConverter.GetBytes(size), 0, sizeof(int), SocketFlags.None);
m_Socket.Send(message, 0, size, SocketFlags.None);
}
}
private byte[] ReceiveInternal()
{
byte[] sizeData = CommonReceiveMessage(sizeof(int));
int size = BitConverter.ToInt32(sizeData);
if (size == 0)
{
return Array.Empty<byte>();
}
return CommonReceiveMessage(size);
}
private byte[] CommonReceiveMessage(int messageLength)
{
if (messageLength < 0)
{
throw new ArgumentOutOfRangeException(nameof(messageLength), messageLength, "Размер сообщения не может быть меньше нуля.");
}
if (messageLength == 0)
{
return Array.Empty<byte>();
}
byte[] buffer = new byte[m_Socket.ReceiveBufferSize];
int currentLength = 0;
int receivedDataLength;
using (MemoryStream memoryStream = new())
{
do
{
receivedDataLength = m_Socket.Receive(buffer, 0, m_Socket.ReceiveBufferSize, SocketFlags.None);
currentLength += receivedDataLength;
memoryStream.Write(buffer, 0, receivedDataLength);
}
while (currentLength < messageLength);
return memoryStream.ToArray();
}
}
}
This example presents an algorithm for sending and receiving data, namely text messages. You can also send files.
using System;
using System.IO;
using System.Net.Sockets;
using System.Text;
namespace Network
{
/// <summary>
/// Represents a network stream for transferring data.
/// </summary>
public class NetworkStream
{
#region Fields
private static readonly byte[] EmptyArray = Array.Empty<byte>();
private readonly Socket m_Socket;
#endregion
#region Constructors
/// <summary>
/// Initializes a new instance of the class <seealso cref="NetworkStream"/>.
/// </summary>
/// <param name="socket">
/// Berkeley socket interface.
/// </param>
public NetworkStream(Socket socket)
{
m_Socket = socket ?? throw new ArgumentNullException(nameof(socket));
}
#endregion
#region Properties
#endregion
#region Methods
/// <summary>
/// Sends a message.
/// </summary>
/// <param name="message">
/// Message text.
/// </param>
/// <exception cref="ArgumentNullException"/>
public void Send(string message)
{
if (message is null)
{
throw new ArgumentNullException(nameof(message));
}
byte[] data = Encoding.UTF8.GetBytes(message);
Write(data);
}
/// <summary>
/// Receives the sent message.
/// </summary>
/// <returns>
/// Sent message.
/// </returns>
public string Receive()
{
byte[] data = Read();
return Encoding.UTF8.GetString(data);
}
/// <summary>
/// Receives the specified number of bytes from a bound <seealso cref="Socket"/>.
/// </summary>
/// <param name="socket">
/// <seealso cref="Socket"/> for receiving data.
/// </param>
/// <param name="size">
/// The size of the received data.
/// </param>
/// <returns>
/// Returns an array of received data.
/// </returns>
private byte[] Read(int size)
{
if (size < 0)
{
// You can throw an exception.
return null;
}
if (size == 0)
{
// Don't throw an exception here, just return an empty data array.
return EmptyArray;
}
// There are many examples on the Internet where the
// Socket.Available property is used, this is WRONG!
// Important! The Socket.Available property is not working as expected.
// Data packages may be in transit, but the Socket.Available property may indicate otherwise.
// Therefore, we use a counter that will allow us to receive all data packets, no more and no less.
// The cycle will continue until we receive all the data packets or the timeout is triggered.
// Note. This algorithm is not designed to work with big data.
SimpleCounter counter = new(size, m_Socket.ReceiveBufferSize);
byte[] buffer = new byte[counter.BufferSize];
int received;
using MemoryStream storage = new();
// The cycle will run until we get all the data.
while (counter.IsExpected)
{
received = m_Socket.Receive(buffer, 0, counter.Available, SocketFlags.None);
// Pass the size of the received data to the counter.
counter.Count(received);
// Write data to memory.
storage.Write(buffer, 0, received);
}
return storage.ToArray();
}
/// <summary>
/// Receives the specified number of bytes from a bound <seealso cref="Socket"/>.
/// </summary>
/// <returns>
/// Returns an array of received data.
/// </returns>
private byte[] Read()
{
byte[] sizeData;
// First, we get the size of the master data.
sizeData = Read(sizeof(int));
// We convert the received data into a number.
int size = BitConverter.ToInt32(sizeData);
// If the data size is less than 0 then throws an exception.
// We inform the recipient that an error occurred while reading the data.
if (size < 0)
{
// Or return the value null.
throw new SocketException();
}
// If the data size is 0, then we will return an empty array.
// Do not allow an exception here.
if (size == 0)
{
return EmptyArray;
}
// Here we read the master data.
byte[] data = Read(size);
return data;
}
/// <summary>
/// Writes data to the stream.
/// </summary>
/// <param name="data"></param>
private void Write(byte[] data)
{
if (data is null)
{
// Throw an exception.
// Or send a negative number that will represent the value null.
throw new ArgumentNullException(nameof(data));
}
byte[] sizeData = BitConverter.GetBytes(data.Length);
// In any case, we inform the recipient about the size of the data.
m_Socket.Send(sizeData, 0, sizeof(int), SocketFlags.None);
if (data.Length != 0)
{
// We send data whose size is greater than zero.
m_Socket.Send(data, 0, data.Length, SocketFlags.None);
}
}
#endregion
#region Classes
/// <summary>
/// Represents a simple counter of received data over the network.
/// </summary>
private class SimpleCounter
{
#region Fields
private int m_Received;
private int m_Available;
private bool m_IsExpected;
#endregion
#region Constructors
/// <summary>
/// Initializes a new instance of the class <seealso cref="SimpleCounter"/>.
/// </summary>
/// <param name="dataSize">
/// Data size.
/// </param>
/// <param name="bufferSize">
/// Buffer size.
/// </param>
/// <exception cref="ArgumentOutOfRangeException"/>
public SimpleCounter(int dataSize, int bufferSize)
{
if (dataSize < 0)
{
throw new ArgumentOutOfRangeException(nameof(dataSize), dataSize, "Data size cannot be less than 0");
}
if (bufferSize < 0)
{
throw new ArgumentOutOfRangeException(nameof(dataSize), bufferSize, "Buffer size cannot be less than 0");
}
DataSize = dataSize;
BufferSize = bufferSize;
// Update the counter data.
UpdateCounter();
}
#endregion
#region Properties
/// <summary>
/// Returns the size of the expected data.
/// </summary>
/// <value>
/// Size of expected data.
/// </value>
public int DataSize { get; }
/// <summary>
/// Returns the size of the buffer.
/// </summary>
/// <value>
/// Buffer size.
/// </value>
public int BufferSize { get; }
/// <summary>
/// Returns the available buffer size for receiving data.
/// </summary>
/// <value>
/// Available buffer size.
/// </value>
public int Available
{
get
{
return m_Available;
}
}
/// <summary>
/// Returns a value indicating whether the thread should wait for data.
/// </summary>
/// <value>
/// <see langword="true"/> if the stream is waiting for data; otherwise, <see langword="false"/>.
/// </value>
public bool IsExpected
{
get
{
return m_IsExpected;
}
}
#endregion
#region Methods
// Updates the counter.
private void UpdateCounter()
{
int unreadDataSize = DataSize - m_Received;
m_Available = unreadDataSize < BufferSize ? unreadDataSize : BufferSize;
m_IsExpected = m_Available > 0;
}
/// <summary>
/// Specifies the size of the received data.
/// </summary>
/// <param name="bytes">
/// The size of the received data.
/// </param>
public void Count(int bytes)
{
// NOTE: Counter cannot decrease.
if (bytes > 0)
{
int received = m_Received += bytes;
// NOTE: The value of the received data cannot exceed the size of the expected data.
m_Received = (received < DataSize) ? received : DataSize;
// Update the counter data.
UpdateCounter();
}
}
/// <summary>
/// Resets counter data.
/// </summary>
public void Reset()
{
m_Received = 0;
UpdateCounter();
}
#endregion
}
#endregion
}
}
Use a do-while loop. This will make sure the memory stream pointers have moved. The first Read or ReadAsync will cause the memorystream pointer to move and then onwards the ".DataAvailable" property will continue to return true until we hit the end of the stream.
An example from microsoft docs:
// Check to see if this NetworkStream is readable.
if(myNetworkStream.CanRead){
byte[] myReadBuffer = new byte[1024];
StringBuilder myCompleteMessage = new StringBuilder();
int numberOfBytesRead = 0;
// Incoming message may be larger than the buffer size.
do{
numberOfBytesRead = myNetworkStream.Read(myReadBuffer, 0, myReadBuffer.Length);
myCompleteMessage.AppendFormat("{0}", Encoding.ASCII.GetString(myReadBuffer, 0, numberOfBytesRead));
}
while(myNetworkStream.DataAvailable);
// Print out the received message to the console.
Console.WriteLine("You received the following message : " +
myCompleteMessage);
}
else{
Console.WriteLine("Sorry. You cannot read from this NetworkStream.");
}
Original Micorosoft Doc
I want to compare two binary files. One of them is already stored on the server with a pre-calculated CRC32 in the database from when I stored it originally.
I know that if the CRC is different, then the files are definitely different. However, if the CRC is the same, I don't know that the files are. So, I'm looking for a nice efficient way of comparing the two streams: one from the posted file and one from the file system.
I'm not an expert on streams, but I'm well aware that I could easily shoot myself in the foot here as far as memory usage is concerned.
static bool FileEquals(string fileName1, string fileName2)
{
// Check the file size and CRC equality here.. if they are equal...
using (var file1 = new FileStream(fileName1, FileMode.Open))
using (var file2 = new FileStream(fileName2, FileMode.Open))
return FileStreamEquals(file1, file2);
}
static bool FileStreamEquals(Stream stream1, Stream stream2)
{
const int bufferSize = 2048;
byte[] buffer1 = new byte[bufferSize]; //buffer size
byte[] buffer2 = new byte[bufferSize];
while (true) {
int count1 = stream1.Read(buffer1, 0, bufferSize);
int count2 = stream2.Read(buffer2, 0, bufferSize);
if (count1 != count2)
return false;
if (count1 == 0)
return true;
// You might replace the following with an efficient "memcmp"
if (!buffer1.Take(count1).SequenceEqual(buffer2.Take(count2)))
return false;
}
}
I sped up the "memcmp" by using a Int64 compare in a loop over the read stream chunks. This reduced time to about 1/4.
private static bool StreamsContentsAreEqual(Stream stream1, Stream stream2)
{
const int bufferSize = 2048 * 2;
var buffer1 = new byte[bufferSize];
var buffer2 = new byte[bufferSize];
while (true)
{
int count1 = stream1.Read(buffer1, 0, bufferSize);
int count2 = stream2.Read(buffer2, 0, bufferSize);
if (count1 != count2)
{
return false;
}
if (count1 == 0)
{
return true;
}
int iterations = (int)Math.Ceiling((double)count1 / sizeof(Int64));
for (int i = 0; i < iterations; i++)
{
if (BitConverter.ToInt64(buffer1, i * sizeof(Int64)) != BitConverter.ToInt64(buffer2, i * sizeof(Int64)))
{
return false;
}
}
}
}
This is how I would do it if you didn't want to rely on crc:
/// <summary>
/// Binary comparison of two files
/// </summary>
/// <param name="fileName1">the file to compare</param>
/// <param name="fileName2">the other file to compare</param>
/// <returns>a value indicateing weather the file are identical</returns>
public static bool CompareFiles(string fileName1, string fileName2)
{
FileInfo info1 = new FileInfo(fileName1);
FileInfo info2 = new FileInfo(fileName2);
bool same = info1.Length == info2.Length;
if (same)
{
using (FileStream fs1 = info1.OpenRead())
using (FileStream fs2 = info2.OpenRead())
using (BufferedStream bs1 = new BufferedStream(fs1))
using (BufferedStream bs2 = new BufferedStream(fs2))
{
for (long i = 0; i < info1.Length; i++)
{
if (bs1.ReadByte() != bs2.ReadByte())
{
same = false;
break;
}
}
}
}
return same;
}
The accepted answer had an error that was pointed out, but never corrected: stream read calls are not guaranteed to return all bytes requested.
BinaryReader ReadBytes calls are guaranteed to return as many bytes as requested unless the end of the stream is reached first.
The following code takes advantage of BinaryReader to do the comparison:
static private bool FileEquals(string file1, string file2)
{
using (FileStream s1 = new FileStream(file1, FileMode.Open, FileAccess.Read, FileShare.Read))
using (FileStream s2 = new FileStream(file2, FileMode.Open, FileAccess.Read, FileShare.Read))
using (BinaryReader b1 = new BinaryReader(s1))
using (BinaryReader b2 = new BinaryReader(s2))
{
while (true)
{
byte[] data1 = b1.ReadBytes(64 * 1024);
byte[] data2 = b2.ReadBytes(64 * 1024);
if (data1.Length != data2.Length)
return false;
if (data1.Length == 0)
return true;
if (!data1.SequenceEqual(data2))
return false;
}
}
}
if you change that crc to a sha1 signature the chances of it being different but with the same signature are astronomicly small
You can check the length and dates of the two files even before checking the CRC to possibly avoid the CRC check.
But if you have to compare the entire file contents, one neat trick I've seen is reading the bytes in strides equal to the bitness of the CPU. For example, on a 32 bit PC, read 4 bytes at a time and compare them as int32's. On a 64 bit PC you can read 8 bytes at a time. This is roughly 4 or 8 times as fast as doing it byte by byte. You also would probably wanna use an unsafe code block so that you could use pointers instead of doing a bunch of bit shifting and OR'ing to get the bytes into the native int sizes.
You can use IntPtr.Size to determine the ideal size for the current processor architecture.
I need to process a large file, around 400K lines and 200 M. But sometimes I have to process from bottom up. How can I use iterator (yield return) here? Basically I don't like to load everything in memory. I know it is more efficient to use iterator in .NET.
Reading text files backwards is really tricky unless you're using a fixed-size encoding (e.g. ASCII). When you've got variable-size encoding (such as UTF-8) you will keep having to check whether you're in the middle of a character or not when you fetch data.
There's nothing built into the framework, and I suspect you'd have to do separate hard coding for each variable-width encoding.
EDIT: This has been somewhat tested - but that's not to say it doesn't still have some subtle bugs around. It uses StreamUtil from MiscUtil, but I've included just the necessary (new) method from there at the bottom. Oh, and it needs refactoring - there's one pretty hefty method, as you'll see:
using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Text;
namespace MiscUtil.IO
{
/// <summary>
/// Takes an encoding (defaulting to UTF-8) and a function which produces a seekable stream
/// (or a filename for convenience) and yields lines from the end of the stream backwards.
/// Only single byte encodings, and UTF-8 and Unicode, are supported. The stream
/// returned by the function must be seekable.
/// </summary>
public sealed class ReverseLineReader : IEnumerable<string>
{
/// <summary>
/// Buffer size to use by default. Classes with internal access can specify
/// a different buffer size - this is useful for testing.
/// </summary>
private const int DefaultBufferSize = 4096;
/// <summary>
/// Means of creating a Stream to read from.
/// </summary>
private readonly Func<Stream> streamSource;
/// <summary>
/// Encoding to use when converting bytes to text
/// </summary>
private readonly Encoding encoding;
/// <summary>
/// Size of buffer (in bytes) to read each time we read from the
/// stream. This must be at least as big as the maximum number of
/// bytes for a single character.
/// </summary>
private readonly int bufferSize;
/// <summary>
/// Function which, when given a position within a file and a byte, states whether
/// or not the byte represents the start of a character.
/// </summary>
private Func<long,byte,bool> characterStartDetector;
/// <summary>
/// Creates a LineReader from a stream source. The delegate is only
/// called when the enumerator is fetched. UTF-8 is used to decode
/// the stream into text.
/// </summary>
/// <param name="streamSource">Data source</param>
public ReverseLineReader(Func<Stream> streamSource)
: this(streamSource, Encoding.UTF8)
{
}
/// <summary>
/// Creates a LineReader from a filename. The file is only opened
/// (or even checked for existence) when the enumerator is fetched.
/// UTF8 is used to decode the file into text.
/// </summary>
/// <param name="filename">File to read from</param>
public ReverseLineReader(string filename)
: this(filename, Encoding.UTF8)
{
}
/// <summary>
/// Creates a LineReader from a filename. The file is only opened
/// (or even checked for existence) when the enumerator is fetched.
/// </summary>
/// <param name="filename">File to read from</param>
/// <param name="encoding">Encoding to use to decode the file into text</param>
public ReverseLineReader(string filename, Encoding encoding)
: this(() => File.OpenRead(filename), encoding)
{
}
/// <summary>
/// Creates a LineReader from a stream source. The delegate is only
/// called when the enumerator is fetched.
/// </summary>
/// <param name="streamSource">Data source</param>
/// <param name="encoding">Encoding to use to decode the stream into text</param>
public ReverseLineReader(Func<Stream> streamSource, Encoding encoding)
: this(streamSource, encoding, DefaultBufferSize)
{
}
internal ReverseLineReader(Func<Stream> streamSource, Encoding encoding, int bufferSize)
{
this.streamSource = streamSource;
this.encoding = encoding;
this.bufferSize = bufferSize;
if (encoding.IsSingleByte)
{
// For a single byte encoding, every byte is the start (and end) of a character
characterStartDetector = (pos, data) => true;
}
else if (encoding is UnicodeEncoding)
{
// For UTF-16, even-numbered positions are the start of a character.
// TODO: This assumes no surrogate pairs. More work required
// to handle that.
characterStartDetector = (pos, data) => (pos & 1) == 0;
}
else if (encoding is UTF8Encoding)
{
// For UTF-8, bytes with the top bit clear or the second bit set are the start of a character
// See http://www.cl.cam.ac.uk/~mgk25/unicode.html
characterStartDetector = (pos, data) => (data & 0x80) == 0 || (data & 0x40) != 0;
}
else
{
throw new ArgumentException("Only single byte, UTF-8 and Unicode encodings are permitted");
}
}
/// <summary>
/// Returns the enumerator reading strings backwards. If this method discovers that
/// the returned stream is either unreadable or unseekable, a NotSupportedException is thrown.
/// </summary>
public IEnumerator<string> GetEnumerator()
{
Stream stream = streamSource();
if (!stream.CanSeek)
{
stream.Dispose();
throw new NotSupportedException("Unable to seek within stream");
}
if (!stream.CanRead)
{
stream.Dispose();
throw new NotSupportedException("Unable to read within stream");
}
return GetEnumeratorImpl(stream);
}
private IEnumerator<string> GetEnumeratorImpl(Stream stream)
{
try
{
long position = stream.Length;
if (encoding is UnicodeEncoding && (position & 1) != 0)
{
throw new InvalidDataException("UTF-16 encoding provided, but stream has odd length.");
}
// Allow up to two bytes for data from the start of the previous
// read which didn't quite make it as full characters
byte[] buffer = new byte[bufferSize + 2];
char[] charBuffer = new char[encoding.GetMaxCharCount(buffer.Length)];
int leftOverData = 0;
String previousEnd = null;
// TextReader doesn't return an empty string if there's line break at the end
// of the data. Therefore we don't return an empty string if it's our *first*
// return.
bool firstYield = true;
// A line-feed at the start of the previous buffer means we need to swallow
// the carriage-return at the end of this buffer - hence this needs declaring
// way up here!
bool swallowCarriageReturn = false;
while (position > 0)
{
int bytesToRead = Math.Min(position > int.MaxValue ? bufferSize : (int)position, bufferSize);
position -= bytesToRead;
stream.Position = position;
StreamUtil.ReadExactly(stream, buffer, bytesToRead);
// If we haven't read a full buffer, but we had bytes left
// over from before, copy them to the end of the buffer
if (leftOverData > 0 && bytesToRead != bufferSize)
{
// Buffer.BlockCopy doesn't document its behaviour with respect
// to overlapping data: we *might* just have read 7 bytes instead of
// 8, and have two bytes to copy...
Array.Copy(buffer, bufferSize, buffer, bytesToRead, leftOverData);
}
// We've now *effectively* read this much data.
bytesToRead += leftOverData;
int firstCharPosition = 0;
while (!characterStartDetector(position + firstCharPosition, buffer[firstCharPosition]))
{
firstCharPosition++;
// Bad UTF-8 sequences could trigger this. For UTF-8 we should always
// see a valid character start in every 3 bytes, and if this is the start of the file
// so we've done a short read, we should have the character start
// somewhere in the usable buffer.
if (firstCharPosition == 3 || firstCharPosition == bytesToRead)
{
throw new InvalidDataException("Invalid UTF-8 data");
}
}
leftOverData = firstCharPosition;
int charsRead = encoding.GetChars(buffer, firstCharPosition, bytesToRead - firstCharPosition, charBuffer, 0);
int endExclusive = charsRead;
for (int i = charsRead - 1; i >= 0; i--)
{
char lookingAt = charBuffer[i];
if (swallowCarriageReturn)
{
swallowCarriageReturn = false;
if (lookingAt == '\r')
{
endExclusive--;
continue;
}
}
// Anything non-line-breaking, just keep looking backwards
if (lookingAt != '\n' && lookingAt != '\r')
{
continue;
}
// End of CRLF? Swallow the preceding CR
if (lookingAt == '\n')
{
swallowCarriageReturn = true;
}
int start = i + 1;
string bufferContents = new string(charBuffer, start, endExclusive - start);
endExclusive = i;
string stringToYield = previousEnd == null ? bufferContents : bufferContents + previousEnd;
if (!firstYield || stringToYield.Length != 0)
{
yield return stringToYield;
}
firstYield = false;
previousEnd = null;
}
previousEnd = endExclusive == 0 ? null : (new string(charBuffer, 0, endExclusive) + previousEnd);
// If we didn't decode the start of the array, put it at the end for next time
if (leftOverData != 0)
{
Buffer.BlockCopy(buffer, 0, buffer, bufferSize, leftOverData);
}
}
if (leftOverData != 0)
{
// At the start of the final buffer, we had the end of another character.
throw new InvalidDataException("Invalid UTF-8 data at start of stream");
}
if (firstYield && string.IsNullOrEmpty(previousEnd))
{
yield break;
}
yield return previousEnd ?? "";
}
finally
{
stream.Dispose();
}
}
IEnumerator IEnumerable.GetEnumerator()
{
return GetEnumerator();
}
}
}
// StreamUtil.cs:
public static class StreamUtil
{
public static void ReadExactly(Stream input, byte[] buffer, int bytesToRead)
{
int index = 0;
while (index < bytesToRead)
{
int read = input.Read(buffer, index, bytesToRead - index);
if (read == 0)
{
throw new EndOfStreamException
(String.Format("End of stream reached with {0} byte{1} left to read.",
bytesToRead - index,
bytesToRead - index == 1 ? "s" : ""));
}
index += read;
}
}
}
Feedback very welcome. This was fun :)
Attention: this approach doesn't work (explained in EDIT)
You could use File.ReadLines to get lines iterator
foreach (var line in File.ReadLines(#"C:\temp\ReverseRead.txt").Reverse())
{
if (noNeedToReadFurther)
break;
// process line here
Console.WriteLine(line);
}
EDIT:
After reading applejacks01's comment, I run some tests and it does look like .Reverse() actually loads whole file.
I used File.ReadLines() to print first line of a 40MB file - memory usage of console app was 5MB. Then, used File.ReadLines().Reverse() to print last line of same file - memory usage was 95MB.
Conclusion
Whatever `Reverse()' is doing, it is not a good choice for reading bottom of a big file.
Very fast solution for huge files: From C#, use PowerShell's Get-Content with the Tail parameter.
using System.Management.Automation;
using (PowerShell powerShell = PowerShell.Create())
{
string lastLine = powerShell.AddCommand("Get-Content")
.AddParameter("Path", #"c:\a.txt")
.AddParameter("Tail", 1)
.Invoke().FirstOrDefault()?.ToString();
}
Required reference: 'System.Management.Automation.dll' - may be somewhere like 'C:\Program Files (x86)\Reference Assemblies\Microsoft\WindowsPowerShell\3.0'
Using PowerShell incurs a small overhead but is worth it for huge files.
To create a file iterator you can do this:
EDIT:
This is my fixed version of a fixed-width reverse file reader:
public static IEnumerable<string> readFile()
{
using (FileStream reader = new FileStream(#"c:\test.txt",FileMode.Open,FileAccess.Read))
{
int i=0;
StringBuilder lineBuffer = new StringBuilder();
int byteRead;
while (-i < reader.Length)
{
reader.Seek(--i, SeekOrigin.End);
byteRead = reader.ReadByte();
if (byteRead == 10 && lineBuffer.Length > 0)
{
yield return Reverse(lineBuffer.ToString());
lineBuffer.Remove(0, lineBuffer.Length);
}
lineBuffer.Append((char)byteRead);
}
yield return Reverse(lineBuffer.ToString());
reader.Close();
}
}
public static string Reverse(string str)
{
char[] arr = new char[str.Length];
for (int i = 0; i < str.Length; i++)
arr[i] = str[str.Length - 1 - i];
return new string(arr);
}
I also add my solution. After reading some answers, nothing really fit to my case.
I'm reading byte by byte from from behind until I find a LineFeed, then I'm returing the collected bytes as string, without using buffering.
Usage:
var reader = new ReverseTextReader(path);
while (!reader.EndOfStream)
{
Console.WriteLine(reader.ReadLine());
}
Implementation:
public class ReverseTextReader
{
private const int LineFeedLf = 10;
private const int LineFeedCr = 13;
private readonly Stream _stream;
private readonly Encoding _encoding;
public bool EndOfStream => _stream.Position == 0;
public ReverseTextReader(Stream stream, Encoding encoding)
{
_stream = stream;
_encoding = encoding;
_stream.Position = _stream.Length;
}
public string ReadLine()
{
if (_stream.Position == 0) return null;
var line = new List<byte>();
var endOfLine = false;
while (!endOfLine)
{
var b = _stream.ReadByteFromBehind();
if (b == -1 || b == LineFeedLf)
{
endOfLine = true;
}
line.Add(Convert.ToByte(b));
}
line.Reverse();
return _encoding.GetString(line.ToArray());
}
}
public static class StreamExtensions
{
public static int ReadByteFromBehind(this Stream stream)
{
if (stream.Position == 0) return -1;
stream.Position = stream.Position - 1;
var value = stream.ReadByte();
stream.Position = stream.Position - 1;
return value;
}
}
I put the file into a list line by line, then used List.Reverse();
StreamReader objReader = new StreamReader(filename);
string sLine = "";
ArrayList arrText = new ArrayList();
while (sLine != null)
{
sLine = objReader.ReadLine();
if (sLine != null)
arrText.Add(sLine);
}
objReader.Close();
arrText.Reverse();
foreach (string sOutput in arrText)
{
...
You can read the file one character at a time backwards and cache all characters until you reach a carriage return and/or line feed.
You then reverse the collected string and yeld it as a line.
There are good answers here already, and here's another LINQ-compatible class you can use which focuses on performance and support for large files. It assumes a "\r\n" line terminator.
Usage:
var reader = new ReverseTextReader(#"C:\Temp\ReverseTest.txt");
while (!reader.EndOfStream)
Console.WriteLine(reader.ReadLine());
ReverseTextReader Class:
/// <summary>
/// Reads a text file backwards, line-by-line.
/// </summary>
/// <remarks>This class uses file seeking to read a text file of any size in reverse order. This
/// is useful for needs such as reading a log file newest-entries first.</remarks>
public sealed class ReverseTextReader : IEnumerable<string>
{
private const int BufferSize = 16384; // The number of bytes read from the uderlying stream.
private readonly Stream _stream; // Stores the stream feeding data into this reader
private readonly Encoding _encoding; // Stores the encoding used to process the file
private byte[] _leftoverBuffer; // Stores the leftover partial line after processing a buffer
private readonly Queue<string> _lines; // Stores the lines parsed from the buffer
#region Constructors
/// <summary>
/// Creates a reader for the specified file.
/// </summary>
/// <param name="filePath"></param>
public ReverseTextReader(string filePath)
: this(new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read), Encoding.Default)
{ }
/// <summary>
/// Creates a reader using the specified stream.
/// </summary>
/// <param name="stream"></param>
public ReverseTextReader(Stream stream)
: this(stream, Encoding.Default)
{ }
/// <summary>
/// Creates a reader using the specified path and encoding.
/// </summary>
/// <param name="filePath"></param>
/// <param name="encoding"></param>
public ReverseTextReader(string filePath, Encoding encoding)
: this(new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read), encoding)
{ }
/// <summary>
/// Creates a reader using the specified stream and encoding.
/// </summary>
/// <param name="stream"></param>
/// <param name="encoding"></param>
public ReverseTextReader(Stream stream, Encoding encoding)
{
_stream = stream;
_encoding = encoding;
_lines = new Queue<string>(128);
// The stream needs to support seeking for this to work
if(!_stream.CanSeek)
throw new InvalidOperationException("The specified stream needs to support seeking to be read backwards.");
if (!_stream.CanRead)
throw new InvalidOperationException("The specified stream needs to support reading to be read backwards.");
// Set the current position to the end of the file
_stream.Position = _stream.Length;
_leftoverBuffer = new byte[0];
}
#endregion
#region Overrides
/// <summary>
/// Reads the next previous line from the underlying stream.
/// </summary>
/// <returns></returns>
public string ReadLine()
{
// Are there lines left to read? If so, return the next one
if (_lines.Count != 0) return _lines.Dequeue();
// Are we at the beginning of the stream? If so, we're done
if (_stream.Position == 0) return null;
#region Read and Process the Next Chunk
// Remember the current position
var currentPosition = _stream.Position;
var newPosition = currentPosition - BufferSize;
// Are we before the beginning of the stream?
if (newPosition < 0) newPosition = 0;
// Calculate the buffer size to read
var count = (int)(currentPosition - newPosition);
// Set the new position
_stream.Position = newPosition;
// Make a new buffer but append the previous leftovers
var buffer = new byte[count + _leftoverBuffer.Length];
// Read the next buffer
_stream.Read(buffer, 0, count);
// Move the position of the stream back
_stream.Position = newPosition;
// And copy in the leftovers from the last buffer
if (_leftoverBuffer.Length != 0)
Array.Copy(_leftoverBuffer, 0, buffer, count, _leftoverBuffer.Length);
// Look for CrLf delimiters
var end = buffer.Length - 1;
var start = buffer.Length - 2;
// Search backwards for a line feed
while (start >= 0)
{
// Is it a line feed?
if (buffer[start] == 10)
{
// Yes. Extract a line and queue it (but exclude the \r\n)
_lines.Enqueue(_encoding.GetString(buffer, start + 1, end - start - 2));
// And reset the end
end = start;
}
// Move to the previous character
start--;
}
// What's left over is a portion of a line. Save it for later.
_leftoverBuffer = new byte[end + 1];
Array.Copy(buffer, 0, _leftoverBuffer, 0, end + 1);
// Are we at the beginning of the stream?
if (_stream.Position == 0)
// Yes. Add the last line.
_lines.Enqueue(_encoding.GetString(_leftoverBuffer, 0, end - 1));
#endregion
// If we have something in the queue, return it
return _lines.Count == 0 ? null : _lines.Dequeue();
}
#endregion
#region IEnumerator<string> Interface
public IEnumerator<string> GetEnumerator()
{
string line;
// So long as the next line isn't null...
while ((line = ReadLine()) != null)
// Read and return it.
yield return line;
}
IEnumerator IEnumerable.GetEnumerator()
{
throw new NotImplementedException();
}
#endregion
}
I know this post is very old but as I couldn't find how to use the most voted solution, I finally found this:
here is the best answer I found with a low memory cost in VB and C#
http://www.blakepell.com/2010-11-29-backward-file-reader-vb-csharp-source
Hope, I'll help others with that because it tooks me hours to finally find this post!
[Edit]
Here is the c# code :
//*********************************************************************************************************************************
//
// Class: BackwardReader
// Initial Date: 11/29/2010
// Last Modified: 11/29/2010
// Programmer(s): Original C# Source - the_real_herminator
// http://social.msdn.microsoft.com/forums/en-US/csharpgeneral/thread/9acdde1a-03cd-4018-9f87-6e201d8f5d09
// VB Converstion - Blake Pell
//
//*********************************************************************************************************************************
using System.Text;
using System.IO;
public class BackwardReader
{
private string path;
private FileStream fs = null;
public BackwardReader(string path)
{
this.path = path;
fs = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
fs.Seek(0, SeekOrigin.End);
}
public string Readline()
{
byte[] line;
byte[] text = new byte[1];
long position = 0;
int count;
fs.Seek(0, SeekOrigin.Current);
position = fs.Position;
//do we have trailing rn?
if (fs.Length > 1)
{
byte[] vagnretur = new byte[2];
fs.Seek(-2, SeekOrigin.Current);
fs.Read(vagnretur, 0, 2);
if (ASCIIEncoding.ASCII.GetString(vagnretur).Equals("rn"))
{
//move it back
fs.Seek(-2, SeekOrigin.Current);
position = fs.Position;
}
}
while (fs.Position > 0)
{
text.Initialize();
//read one char
fs.Read(text, 0, 1);
string asciiText = ASCIIEncoding.ASCII.GetString(text);
//moveback to the charachter before
fs.Seek(-2, SeekOrigin.Current);
if (asciiText.Equals("n"))
{
fs.Read(text, 0, 1);
asciiText = ASCIIEncoding.ASCII.GetString(text);
if (asciiText.Equals("r"))
{
fs.Seek(1, SeekOrigin.Current);
break;
}
}
}
count = int.Parse((position - fs.Position).ToString());
line = new byte[count];
fs.Read(line, 0, count);
fs.Seek(-count, SeekOrigin.Current);
return ASCIIEncoding.ASCII.GetString(line);
}
public bool SOF
{
get
{
return fs.Position == 0;
}
}
public void Close()
{
fs.Close();
}
}
I wanted to do the similar thing.
Here is my code. This class will create temporary files containing chunks of the big file. This will avoid memory bloating. User can specify whether s/he wants the file reversed. Accordingly it will return the content in reverse manner.
This class can also be used to write big data in a single file without bloating memory.
Please provide feedback.
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace BigFileService
{
public class BigFileDumper
{
/// <summary>
/// Buffer that will store the lines until it is full.
/// Then it will dump it to temp files.
/// </summary>
public int CHUNK_SIZE = 1000;
public bool ReverseIt { get; set; }
public long TotalLineCount { get { return totalLineCount; } }
private long totalLineCount;
private int BufferCount = 0;
private StreamWriter Writer;
/// <summary>
/// List of files that would store the chunks.
/// </summary>
private List<string> LstTempFiles;
private string ParentDirectory;
private char[] trimchars = { '/', '\\'};
public BigFileDumper(string FolderPathToWrite)
{
this.LstTempFiles = new List<string>();
this.ParentDirectory = FolderPathToWrite.TrimEnd(trimchars) + "\\" + "BIG_FILE_DUMP";
this.totalLineCount = 0;
this.BufferCount = 0;
this.Initialize();
}
private void Initialize()
{
// Delete existing directory.
if (Directory.Exists(this.ParentDirectory))
{
Directory.Delete(this.ParentDirectory, true);
}
// Create a new directory.
Directory.CreateDirectory(this.ParentDirectory);
}
public void WriteLine(string line)
{
if (this.BufferCount == 0)
{
string newFile = "DumpFile_" + LstTempFiles.Count();
LstTempFiles.Add(newFile);
Writer = new StreamWriter(this.ParentDirectory + "\\" + newFile);
}
// Keep on adding in the buffer as long as size is okay.
if (this.BufferCount < this.CHUNK_SIZE)
{
this.totalLineCount++; // main count
this.BufferCount++; // Chunk count.
Writer.WriteLine(line);
}
else
{
// Buffer is full, time to create a new file.
// Close the existing file first.
Writer.Close();
// Make buffer count 0 again.
this.BufferCount = 0;
this.WriteLine(line);
}
}
public void Close()
{
if (Writer != null)
Writer.Close();
}
public string GetFullFile()
{
if (LstTempFiles.Count <= 0)
{
Debug.Assert(false, "There are no files created.");
return "";
}
string returnFilename = this.ParentDirectory + "\\" + "FullFile";
if (File.Exists(returnFilename) == false)
{
// Create a consolidated file from the existing small dump files.
// Now this is interesting. We will open the small dump files one by one.
// Depending on whether the user require inverted file, we will read them in descending order & reverted,
// or ascending order in normal way.
if (this.ReverseIt)
this.LstTempFiles.Reverse();
foreach (var fileName in LstTempFiles)
{
string fullFileName = this.ParentDirectory + "\\" + fileName;
// FileLines will use small memory depending on size of CHUNK. User has control.
var fileLines = File.ReadAllLines(fullFileName);
// Time to write in the writer.
if (this.ReverseIt)
fileLines = fileLines.Reverse().ToArray();
// Write the lines
File.AppendAllLines(returnFilename, fileLines);
}
}
return returnFilename;
}
}
}
This service can be used as follows -
void TestBigFileDump_File(string BIG_FILE, string FOLDER_PATH_FOR_CHUNK_FILES)
{
// Start processing the input Big file.
StreamReader reader = new StreamReader(BIG_FILE);
// Create a dump file class object to handle efficient memory management.
var bigFileDumper = new BigFileDumper(FOLDER_PATH_FOR_CHUNK_FILES);
// Set to reverse the output file.
bigFileDumper.ReverseIt = true;
bigFileDumper.CHUNK_SIZE = 100; // How much at a time to keep in RAM before dumping to local file.
while (reader.EndOfStream == false)
{
string line = reader.ReadLine();
bigFileDumper.WriteLine(line);
}
bigFileDumper.Close();
reader.Close();
// Get back full reversed file.
var reversedFilename = bigFileDumper.GetFullFile();
Console.WriteLine("Check output file - " + reversedFilename);
}
In case anyone else comes across this, I solved it with the following PowerShell script which can easily be modified into a C# script with a small amount of effort.
[System.IO.FileStream]$fileStream = [System.IO.File]::Open("C:\Name_of_very_large_file.log", [System.IO.FileMode]::Open, [System.IO.FileAccess]::Read, [System.IO.FileShare]::ReadWrite)
[System.IO.BufferedStream]$bs = New-Object System.IO.BufferedStream $fileStream;
[System.IO.StreamReader]$sr = New-Object System.IO.StreamReader $bs;
$buff = New-Object char[] 20;
$seek = $bs.Seek($fileStream.Length - 10000, [System.IO.SeekOrigin]::Begin);
while(($line = $sr.ReadLine()) -ne $null)
{
$line;
}
This basically starts reading from the last 10,000 characters of a file, outputting each line.