We Keep Coding

Sending multiple files from C# to C

I have a wpf gui that I want to upload files from and send to a C client .
I want to send 3 files and for some reason 1 of them is being sent and written (but it adds 8 nulls in the end and removes 4 of the first letters in the file)
and in the other two when I try to receive the size it says their size is 0
I've been stuck on this problem for a while now and i'm becoming depserate as i'm probably missing a small thing , if any of u could give a hand that'll mean a lot ! I really wanna know whats the problem in my code.
I have the files paths in an array and sending it in main like so
C# Main
IPEndPoint ipPoint = new IPEndPoint(IPAddress.Parse("127.0.0.1"), 5555);//switch the port
Socket listenSocket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
listenSocket.Bind(ipPoint);
listenSocket.Listen(1);
Socket clientSocket = listenSocket.Accept();
for (int i = 0; i < 1; i++)
{
SendFile(clientSocket, filePaths[i]);
}
clientSocket.Shutdown(SocketShutdown.Both);
clientSocket.Close();
SendFile(C# side)
public static void SendFile(Socket clientSocket, string filePath)
{
if (File.Exists(filePath))
{
FileInfo fi = new FileInfo(filePath);
long file_size = fi.Length;
byte[] preBuffer;
using (var memoryStream = new MemoryStream())
{
using (BinaryWriter writer = new BinaryWriter(memoryStream))
{
writer.Write(file_size);
}
preBuffer = memoryStream.ToArray();
byte[] fixedBuffer = new byte[4];
Array.Copy(preBuffer, 0, fixedBuffer, 0, 4);
Console.WriteLine(BitConverter.ToString(preBuffer));
Console.WriteLine(BitConverter.ToString(fixedBuffer)); //fixing the problem i had with the converting to array that it added 4 useless zeros.
clientSocket.Send(fixedBuffer); // sending size
}
byte[] data = new Byte[4096];
using (FileStream fs = new FileStream(filePath, FileMode.Open))
{
int actualRead;
do
{
actualRead = fs.Read(data, 0, data.Length);
clientSocket.Send(data);
file_size -= actualRead;
} while (file_size - filePath.Length > 0);
}
}
else
{
MessageBox.Show("File for the program is missing! lua/pcap/csv");
}
}
C Receive(built from 3 functions)
/*
============================================
General : function is responsible for receiving a length of data from the client
Parameters : sock - client socket to receive the data from
*buf - holds a pointer to the buffer that needs to update
bufsize - the length of the buffer
Return Value : returns TRUE when the data is read correctly
else, FALSE when there was a socket error or no bytes are received.
============================================
*/
bool recv_raw(SOCKET sock, void* buf, int bufsize)
{
unsigned char* pbuf = (unsigned char*)buf;
while (bufsize > 0) {
int num = recv(sock, pbuf, bufsize, 0);
if (num <= 0) { return false; }
pbuf += num;
bufsize -= num;
}
return true;
}
/*
===================================================
General : receives the length of the file and updates it
Parameters : sock - client socket to receive the data from
*filesize - holds a pointer to the size of the buffer that needs to update
filesize_len - the length of the file size pointer
Return Value : returns TRUE when the size is read correctly
else, FALSE when there was a socket error or no bytes are received.
===================================================
*/
bool recv_file_len(SOCKET sock, long* filesize)
{
if (!recv_raw(sock, filesize, sizeof(*filesize))) { return false; }
return true;
}
/*
================================================== =
General : writes to the lua file the data from the file
that was received in the socket
Parameters : sock - the socket between the client and server
*f - the file to write the data received to
Return Value : returns TRUE when everything was written to the file.
returns FALSE if there's no data received or detected a socket problem.
================================================== =
*/
bool write_data(SOCKET sock, FILE *f)
{
long filesize;//size of address
char buffer[BUFFER_SIZE];
if (!recv_file_len(sock, &filesize)) { return false; }
printf("file size (From C#) : %ld\n", filesize);
int n = recv_raw(sock, buffer, 8); // need to get the size of the name
if (filesize > 0)
{
do {
int num = min(filesize, BUFFER_SIZE);
if (!recv_raw(sock, buffer, num)) {
return false;
}
int offset = 0;
do
{
size_t written = fwrite(&buffer[offset], 1, num - offset, f);
if (written < 1) { return false; }
offset += written;
} while (offset < num);
filesize -= num;
} while (filesize > 0);
}
return true;
}
C Main
FILE* luafhandler = fopen("test.lua", "wb");//the new lua file
if (luafhandler == NULL)
{
fclose(luafhandler);
printf("GUI CONNECT lua file failed to open!\n");
}
FILE* pcapfhandler = fopen("test.pcap", "wb");//the new lua file
if (pcapfhandler == NULL)
{
fclose(pcapfhandler);
printf("GUI CONNECT pcap file failed to open!\n");
}
FILE* csvfhandler = fopen("AlgoTest.csv", "wb");//the new lua file
if (csvfhandler == NULL)
{
fclose(csvfhandler);
printf("GUI CONNECT csv file failed to open!\n");
}
else {
SOCKET sock1 = open_socket(5555, SERVER_IP);
bool check = write_data(sock1, luafhandler);
bool check1 = write_data(sock1, pcapfhandler);
bool check2 = write_data(sock1, csvfhandler);
fclose(luafhandler);
fclose(pcapfhandler);
fclose(csvfhandler);
}

Read a fixed number of bytes when reading file using StreamReader

I'm using this port of the Mozilla character set detector to determine a file's encoding and then using that to construct a StreamReader. So far, so good.
However, the file format I am reading is an odd one and from time to time it is necessary to skip a number of bytes. That is, a file that is otherwise text, in one or other encoding, will have some raw bytes embedded in it.
I would like to read the stream as text, up to the point that I hit some text that indicates a byte stream follows, then I would like to read the byte stream, then resume reading as text. What is the best way of doing this (balance of simplicity and performance)?
I can't rely on seeking against the FileStream underlying the the StreamReader (and then discarding the buffered data in the latter) because I don't know how many bytes were used in reading the characters up to that point. I might abandon using StreamReader and switch to a bespoke class that uses parallel arrays of bytes and chars, populates the latter from the former using a decoder, and tracks the position in the byte array every time a character is read by using the encoding to calculate the number of bytes used for the character. Yuk.
To further clarify, the file has this format:
[encoded chars][embedded bytes indicator + len][len bytes][encoded chars]...
Where there many be zero one or many blocks of embedded bytes and the blocks of embedded chars may be any length.
So, for example:
ABC:123:DEF:456:$0099[0x00,0x01,0x02,... x 99]GHI:789:JKL:...
There are no line delimiters. I may have any number of fields (ABC, 123, ...) delimited by some character (in this case a colon). These fields may be in various codepages, including UTF-8 (not guaranteed to be single byte). When I hit a $ I know that the next 4 bytes contain a length (call it n), the next n bytes are to be read raw, and byte n + 1 will be another text field (GHI).

Proof of concept. This class works with UTF-16 string data, and ':' delimiters per OP. It expects binary length as a 4-byte, little-endian binary integer. It should be easy to adjust to more specific details of your (odd) file format. For example, any Decoder class should drop in to ReadString() and "just work".
To use it, construct it with a Stream class. For each individual data element, call ReportNextData(), which will tell you what kind of data is next, and then call the appropriate Read*() method. For binary data, call ReadBinaryLength() and then ReadBinaryData().
Note that ReadBinaryData() follows the stream contract; it is not guaranteed to return as many bytes as you asked for, so you may need to call it several times. However, if you ask for too many bytes, it will throw EndOfStreamException.
I tested it with this data (hex format):
410042004300240A0000000102030405060708090024050000000504030201580059005A003A310032003300
Which is:
ABC$[10][1234567890]$[5][54321]XYZ:123
Scan the data like so:
OddFileReader.NextData nextData;
while ((nextData = reader.ReportNextData()) != OddFileReader.NextData.Eof)
{
// Call appropriate Read*() here.
}
public class OddFileReader : IDisposable
{
public enum NextData
{
Unknown,
Eof,
String,
BinaryLength,
BinaryData
}
private Stream source;
private byte[] byteBuffer;
private int bufferOffset;
private int bufferEnd;
private NextData nextData;
private int binaryOffset;
private int binaryEnd;
private char[] characterBuffer;
public OddFileReader(Stream source)
{
this.source = source;
}
public NextData ReportNextData()
{
if (nextData != NextData.Unknown)
{
return nextData;
}
if (!PopulateBufferIfNeeded(1))
{
return (nextData = NextData.Eof);
}
if (byteBuffer[bufferOffset] == '$')
{
return (nextData = NextData.BinaryLength);
}
else
{
return (nextData = NextData.String);
}
}
public string ReadString()
{
ReportNextData();
if (nextData == NextData.Eof)
{
throw new EndOfStreamException();
}
else if (nextData != NextData.String)
{
throw new InvalidOperationException("Attempt to read non-string data as string");
}
if (characterBuffer == null)
{
characterBuffer = new char[1];
}
StringBuilder stringBuilder = new StringBuilder();
Decoder decoder = Encoding.Unicode.GetDecoder();
while (nextData == NextData.String)
{
byte b = byteBuffer[bufferOffset];
if (b == '$')
{
nextData = NextData.BinaryLength;
break;
}
else if (b == ':')
{
nextData = NextData.Unknown;
bufferOffset++;
break;
}
else
{
if (decoder.GetChars(byteBuffer, bufferOffset++, 1, characterBuffer, 0) == 1)
{
stringBuilder.Append(characterBuffer[0]);
}
if (bufferOffset == bufferEnd && !PopulateBufferIfNeeded(1))
{
nextData = NextData.Eof;
break;
}
}
}
return stringBuilder.ToString();
}
public int ReadBinaryLength()
{
ReportNextData();
if (nextData == NextData.Eof)
{
throw new EndOfStreamException();
}
else if (nextData != NextData.BinaryLength)
{
throw new InvalidOperationException("Attempt to read non-binary-length data as binary length");
}
bufferOffset++;
if (!PopulateBufferIfNeeded(sizeof(Int32)))
{
nextData = NextData.Eof;
throw new EndOfStreamException();
}
binaryEnd = BitConverter.ToInt32(byteBuffer, bufferOffset);
binaryOffset = 0;
bufferOffset += sizeof(Int32);
nextData = NextData.BinaryData;
return binaryEnd;
}
public int ReadBinaryData(byte[] buffer, int offset, int count)
{
ReportNextData();
if (nextData == NextData.Eof)
{
throw new EndOfStreamException();
}
else if (nextData != NextData.BinaryData)
{
throw new InvalidOperationException("Attempt to read non-binary data as binary data");
}
if (count > binaryEnd - binaryOffset)
{
throw new EndOfStreamException();
}
int bytesRead;
if (bufferOffset < bufferEnd)
{
bytesRead = Math.Min(count, bufferEnd - bufferOffset);
Array.Copy(byteBuffer, bufferOffset, buffer, offset, bytesRead);
bufferOffset += bytesRead;
}
else if (count < byteBuffer.Length)
{
if (!PopulateBufferIfNeeded(1))
{
throw new EndOfStreamException();
}
bytesRead = Math.Min(count, bufferEnd - bufferOffset);
Array.Copy(byteBuffer, bufferOffset, buffer, offset, bytesRead);
bufferOffset += bytesRead;
}
else
{
bytesRead = source.Read(buffer, offset, count);
}
binaryOffset += bytesRead;
if (binaryOffset == binaryEnd)
{
nextData = NextData.Unknown;
}
return bytesRead;
}
private bool PopulateBufferIfNeeded(int minimumBytes)
{
if (byteBuffer == null)
{
byteBuffer = new byte[8192];
}
if (bufferEnd - bufferOffset < minimumBytes)
{
int shiftCount = bufferEnd - bufferOffset;
if (shiftCount > 0)
{
Array.Copy(byteBuffer, bufferOffset, byteBuffer, 0, shiftCount);
}
bufferOffset = 0;
bufferEnd = shiftCount;
while (bufferEnd - bufferOffset < minimumBytes)
{
int bytesRead = source.Read(byteBuffer, bufferEnd, byteBuffer.Length - bufferEnd);
if (bytesRead == 0)
{
return false;
}
bufferEnd += bytesRead;
}
}
return true;
}
public void Dispose()
{
Stream source = this.source;
this.source = null;
if (source != null)
{
source.Dispose();
}
}
}

How can I count the occurences of a certain character combination in a text file in the fastest possible way?

I would like to make a method that counts the occurences of a series of characters in a .txt file (C#). I've found some related questions here that have valid answers. However, there are certain circumstances that restrict the possible solutions:
The method has to work quite fast, because I have to use it more hundred times in the program.
The text in the file is overlong to be read in a string.
Thank you for your help.

The method has to work quite fast, because I have to use it more hundred times in the program.
According to recent benchmarks, SequenceEqual of Span<T> tends to be the fastest way to compare array slices in .NET nowadays (except for unsafe or P/Invoke approaches).
The text in the file is overlong to be read in a string.
This issue can easily be tackled using FileStream or StreamReader.
In a nutshell, you need to read the file chunked: read a fixed size part from the file, look for occurences in it, read the next part, look for occurences, and so on. This can be coded without moving back the cursor, just the leftover of each part needs to be taken into account when dealing with the next part.
Here is my approach using FileStream and Span<T>:
public static int CountOccurences(Stream stream, string searchString, Encoding encoding = null, int bufferSize = 4096)
{
if (stream == null)
throw new ArgumentNullException(nameof(stream));
if (searchString == null)
throw new ArgumentNullException(nameof(searchString));
if (!stream.CanRead)
throw new ArgumentException("Stream must be readable.", nameof(stream));
if (bufferSize <= 0)
throw new ArgumentException("Buffer size must be a positive number.", nameof(bufferSize));
// detecting encoding
Span<byte> bom = stackalloc byte[4];
var actualLength = stream.Read(bom);
if (actualLength == 0)
return 0;
bom = bom.Slice(0, actualLength);
Encoding detectedEncoding;
if (bom.StartsWith(Encoding.UTF8.GetPreamble()))
detectedEncoding = Encoding.UTF8;
else if (bom.StartsWith(Encoding.UTF32.GetPreamble()))
detectedEncoding = Encoding.UTF32;
else if (bom.StartsWith(Encoding.Unicode.GetPreamble()))
detectedEncoding = Encoding.Unicode;
else if (bom.StartsWith(Encoding.BigEndianUnicode.GetPreamble()))
detectedEncoding = Encoding.BigEndianUnicode;
else
detectedEncoding = null;
if (detectedEncoding != null)
{
if (encoding == null)
encoding = detectedEncoding;
if (encoding == detectedEncoding)
bom = bom.Slice(detectedEncoding.GetPreamble().Length);
}
else if (encoding == null)
encoding = Encoding.ASCII;
// acquiring a buffer
ReadOnlySpan<byte> searchBytes = encoding.GetBytes(searchString);
bufferSize = Math.Max(Math.Max(bufferSize, searchBytes.Length), 128);
var bufferArray = ArrayPool<byte>.Shared.Rent(bufferSize);
try
{
var buffer = new Span<byte>(bufferArray, 0, bufferSize);
// looking for occurences
bom.CopyTo(buffer);
actualLength = bom.Length + stream.Read(buffer.Slice(bom.Length));
var occurrences = 0;
do
{
var index = 0;
var endIndex = actualLength - searchBytes.Length;
for (; index <= endIndex; index++)
if (buffer.Slice(index, searchBytes.Length).SequenceEqual(searchBytes))
occurrences++;
if (actualLength < buffer.Length)
break;
ReadOnlySpan<byte> leftover = buffer.Slice(index);
leftover.CopyTo(buffer);
actualLength = leftover.Length + stream.Read(buffer.Slice(leftover.Length));
}
while (true);
return occurrences;
}
finally { ArrayPool<byte>.Shared.Return(bufferArray); }
}
This code requires C# 7.2 to compile. You may have to include the System.Buffers and System.Memory NuGet packages, as well. If you use .NET Core version lower than 2.1 or another platform than .NET Core, you need to include this "polyfill", as well:
static class Compatibility
{
public static int Read(this Stream stream, Span<byte> buffer)
{
// copied over from corefx sources (https://github.com/dotnet/corefx/blob/master/src/Common/src/CoreLib/System/IO/Stream.cs)
byte[] sharedBuffer = ArrayPool<byte>.Shared.Rent(buffer.Length);
try
{
int numRead = stream.Read(sharedBuffer, 0, buffer.Length);
if ((uint)numRead > buffer.Length)
throw new IOException("Stream was too long.");
new Span<byte>(sharedBuffer, 0, numRead).CopyTo(buffer);
return numRead;
}
finally { ArrayPool<byte>.Shared.Return(sharedBuffer); }
}
}
Usage:
using (var fs = new FileStream(#"path-to-file", FileMode.Open, FileAccess.Read, FileShare.Read))
Console.WriteLine(CountOccurences(fs, "string to search"));
When you don't specify the encoding argument, the encoding will be auto-detected by examining the BOM of the file. If BOM is not present, ASCII encoding is assumed as a fallback.

FileStream delimiter?

I have a large file with (text/Binary) format.
file format: (0 represent a byte)
00000FileName0000000Hello
World
world1
...
0000000000000000000000
Currently i'm using FileStream and i want to read the Hello.
I Know where Hello start, and it ends with a 0x0D 0x0A.
I also need to go back if the words is not equal to Hello.
How can i read until a carriage return?
is there any PEEK like function in FileStream so i can move back the read pointer`?
is FileStream even a good choice in this case?

You can use the method FileStream.Seek to change the read/write position.

You can use BinaryReader for reading binary content; however, it uses an inner buffer so you cannot rely the underlying Stream.Position anymore, because it can read more bytes in the background than you want. But you can re-implement its needed methods:
private byte[] ReadBytes(Stream s, int count)
{
buffer = new byte[count];
if (count == 0)
{
return buffer;
}
// reading one byte
if (count == 1)
{
int value = s.ReadByte();
if (value == -1)
threw new IOException("Out of stream");
buffer[0] = (byte)value;
return buffer;
}
// reading multiple bytes
int offset = 0;
do
{
int readBytes = s.Read(buffer, offset, count - offset);
if (readBytes == 0)
threw new IOException("Out of stream");
offset += readBytes;
}
while (offset < count);
return buffer;
}
public int ReadInt32(Stream s)
{
byte[] buffer = ReadBytes(s, 4);
return BitConverter.ToInt32(buffer, 0);
}
// similarly, write ReadInt16/64, etc, whatever you need
Assuming that you are on the start position, you can write a ReadString, too:
private string ReadString(Stream s, char delimiter)
{
var result = new List<char>();
int c;
while ((c = s.ReadByte()) != -1 && (char)c != delimiter)
{
result.Add((char)c);
}
return new string(result.ToArray());
}
Usage:
FileStream fs = GetMyFile(); // todo
if (!fs.CanSeek)
throw new NotSupportedException("sorry");
long posCurrent = fs.Position; // save current position
int posHello = ReadInt32(fs); // read position of "hello"
fs.Seek(posHello, SeekOrigin.Begin); // seeking to hello
string hello = ReadString(fs, '\n'); // reading hello
fs.Seek(posCurrent, SeekOrigin.Begin); // seeking back

How to split a large file into chunks in c#?

I'm making a simple file transfer sender and receiver app through the wire. What I have so far is that the sender converts the file into a byte array and sends chunks of that array to the receiver.
This works with file of up to 256mb, but this line throws a "System out of memory" exception for anything above:
byte[] buffer = StreamFile(fileName); //This is where I convert the file
I'm looking for a way to read the file in chunks then write that chunk instead of loading the whole file into a byte. How can I do this with a FileStream?
EDIT:
Sorry, heres my crappy code so far:
private void btnSend(object sender, EventArgs e)
{
Socket clientSock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
byte[] fileName = Encoding.UTF8.GetBytes(fName); //file name
byte[] fileData = null;
try
{
fileData = StreamFile(textBox1.Text); //file
}
catch (OutOfMemoryException ex)
{
MessageBox.Show("Out of memory");
return;
}
byte[] fileNameLen = BitConverter.GetBytes(fileName.Length); //length of file name
clientData = new byte[4 + fileName.Length + fileData.Length];
fileNameLen.CopyTo(clientData, 0);
fileName.CopyTo(clientData, 4);
fileData.CopyTo(clientData, 4 + fileName.Length);
clientSock.Connect("172.16.12.91", 9050);
clientSock.Send(clientData, 0, 4 + fileName.Length, SocketFlags.None);
for (int i = 4 + fileName.Length; i < clientData.Length; i++)
{
clientSock.Send(clientData, i, 1 , SocketFlags.None);
}
clientSock.Close();
}
And here's how I receive (the code was from a tutorial)
public void ReadCallback(IAsyncResult ar)
{
int fileNameLen = 1;
String content = String.Empty;
StateObject state = (StateObject)ar.AsyncState;
Socket handler = state.workSocket;
int bytesRead = handler.EndReceive(ar);
if (bytesRead > 0)
{
if (flag == 0)
{
Thread.Sleep(1000);
fileNameLen = BitConverter.ToInt32(state.buffer, 0);
string fileName = Encoding.UTF8.GetString(state.buffer, 4, fileNameLen);
receivedPath = fileName;
flag++;
}
if (flag >= 1)
{
BinaryWriter writer = new BinaryWriter(File.Open(receivedPath, FileMode.Append));
if (flag == 1)
{
writer.Write(state.buffer, 4 + fileNameLen, bytesRead - (4 + fileNameLen));
flag++;
}
else
writer.Write(state.buffer, 0, bytesRead);
writer.Close();
handler.BeginReceive(state.buffer, 0, StateObject.BufferSize, 0,
new AsyncCallback(ReadCallback), state);
}
}
else
{
Invoke(new MyDelegate(LabelWriter));
}
}
I just really want to know how I can read the file in chunks so that I dont need to convert it to a byte.
Thanks for the responses so far, I think I'm starting to get it :D

Just call Read repeatedly with a small buffer (I tend to use something like 16K). Note that the call to Read may end up reading a smaller amount than you request. If you're using a fixed chunk size and need the whole chunk in memory, you could just use an array of that size of course.
Without knowing how you're sending the file, it's hard to give much advice about how to structure your code, but it could be something like this:
byte[] chunk = new byte[MaxChunkSize];
while (true)
{
int index = 0;
// There are various different ways of structuring this bit of code.
// Fundamentally we're trying to keep reading in to our chunk until
// either we reach the end of the stream, or we've read everything we need.
while (index < chunk.Length)
{
int bytesRead = stream.Read(chunk, index, chunk.Length - index);
if (bytesRead == 0)
{
break;
}
index += bytesRead;
}
if (index != 0) // Our previous chunk may have been the last one
{
SendChunk(chunk, index); // index is the number of bytes in the chunk
}
if (index != chunk.Length) // We didn't read a full chunk: we're done
{
return;
}
}
If I was more awake I'd probably find a more readable way of writing this, but it'll do for now. One option is to extract another method from the middle section:
// Attempts to read an entire chunk into the given array; returns the size of
// chunk actually read.
int ReadChunk(Stream stream, byte[] chunk)
{
int index = 0;
while (index < chunk.Length)
{
int bytesRead = stream.Read(chunk, index, chunk.Length - index);
if (bytesRead == 0)
{
break;
}
index += bytesRead;
}
return index;
}

var b = new byte[1<<15]; // 32k
while((count = inStream.Read(b, 0, b.Length)) > 0)
{
outStream.Write(b, 0, count);
}

public static IEnumerable<byte[]> SplitStreamIntoChunks(Stream stream, int chunkSize)
{
var bytesRemaining = stream.Length;
while (bytesRemaining > 0)
{
var size = Math.Min((int) bytesRemaining, chunkSize);
var buffer = new byte[size];
var bytesRead = stream.Read(buffer, 0, size);
if (bytesRead <= 0)
break;
yield return buffer;
bytesRemaining -= bytesRead;
}
}